Refactored the command line config system and other aspects of config.

* Removed ``NoConfigDefault`` from our config logic entirely. It turns out
  that argparse has a nice ``argparse.SUPPRESS`` object that handles this 
  nicely (r1283, r1284).
* Completely cleared the previous config when :meth:`load_config` is called
  to allow it to be called more than once.
* Added some additional tests and docs for the config system.
* Removed the ``override_config`` and ``constructor_config`` config stages
  from :class:`~IPython.core.application.Applications` (r1293).
* Completely refactored the command line config parser classes. We are now
  using a more object oriented design that allows for fine grained 
  inheritance and good encapsulation.

IPython/config/loader.py

@@ -1,386 +1,370 b''
 # coding: utf-8
 """A simple configuration system.
 
 Authors
 -------
 * Brian Granger
 * Fernando Perez
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008-2009  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import __builtin__
 import os
 import sys
 
 from IPython.external import argparse
 from IPython.utils.path import filefind
 
 #-----------------------------------------------------------------------------
 # Exceptions
 #-----------------------------------------------------------------------------
 
 
 class ConfigError(Exception):
     pass
 
 
 class ConfigLoaderError(ConfigError):
     pass
 
 #-----------------------------------------------------------------------------
 # Argparse fix
 #-----------------------------------------------------------------------------
 
 # Unfortunately argparse by default prints help messages to stderr instead of
 # stdout.  This makes it annoying to capture long help screens at the command
 # line, since one must know how to pipe stderr, which many users don't know how
 # to do.  So we override the print_help method with one that defaults to
 # stdout and use our class instead.
 
 class ArgumentParser(argparse.ArgumentParser):
     """Simple argparse subclass that prints help to stdout by default."""
     
     def print_help(self, file=None):
         if file is None:
             file = sys.stdout
         return super(ArgumentParser, self).print_help(file)
     
     print_help.__doc__ = argparse.ArgumentParser.print_help.__doc__
     
 #-----------------------------------------------------------------------------
 # Config class for holding config information
 #-----------------------------------------------------------------------------
 
 
 class Config(dict):
     """An attribute based dict that can do smart merges."""
 
     def __init__(self, *args, **kwds):
         dict.__init__(self, *args, **kwds)
         # This sets self.__dict__ = self, but it has to be done this way
         # because we are also overriding __setattr__.
         dict.__setattr__(self, '__dict__', self)
 
     def _merge(self, other):
         to_update = {}
         for k, v in other.items():
             if not self.has_key(k):
                 to_update[k] = v
             else: # I have this key
                 if isinstance(v, Config):
                     # Recursively merge common sub Configs
                     self[k]._merge(v)
                 else:
                     # Plain updates for non-Configs
                     to_update[k] = v
 
         self.update(to_update)
 
     def _is_section_key(self, key):
         if key[0].upper()==key[0] and not key.startswith('_'):
             return True
         else:
             return False
 
     def has_key(self, key):
         if self._is_section_key(key):
             return True
         else:
             return dict.has_key(self, key)
 
     def _has_section(self, key):
         if self._is_section_key(key):
             if dict.has_key(self, key):
                 return True
         return False
 
     def copy(self):
         return type(self)(dict.copy(self))
 
     def __copy__(self):
         return self.copy()
 
     def __deepcopy__(self, memo):
         import copy
         return type(self)(copy.deepcopy(self.items()))
 
     def __getitem__(self, key):
         # Because we use this for an exec namespace, we need to delegate
         # the lookup of names in __builtin__ to itself.  This means
         # that you can't have section or attribute names that are 
         # builtins.
         try:
             return getattr(__builtin__, key)
         except AttributeError:
             pass
         if self._is_section_key(key):
             try:
                 return dict.__getitem__(self, key)
             except KeyError:
                 c = Config()
                 dict.__setitem__(self, key, c)
                 return c
         else:
             return dict.__getitem__(self, key)
 
     def __setitem__(self, key, value):
         # Don't allow names in __builtin__ to be modified.
         if hasattr(__builtin__, key):
             raise ConfigError('Config variable names cannot have the same name '
                               'as a Python builtin: %s' % key)
         if self._is_section_key(key):
             if not isinstance(value, Config):
                 raise ValueError('values whose keys begin with an uppercase '
                                  'char must be Config instances: %r, %r' % (key, value))
         else:
             dict.__setitem__(self, key, value)
 
     def __getattr__(self, key):
         try:
             return self.__getitem__(key)
         except KeyError, e:
             raise AttributeError(e)
 
     def __setattr__(self, key, value):
         try:
             self.__setitem__(key, value)
         except KeyError, e:
             raise AttributeError(e)
 
     def __delattr__(self, key):
         try:
             dict.__delitem__(self, key)
         except KeyError, e:
             raise AttributeError(e)
 
 
 #-----------------------------------------------------------------------------
 # Config loading classes
 #-----------------------------------------------------------------------------
 
 
 class ConfigLoader(object):
     """A object for loading configurations from just about anywhere.
     
     The resulting configuration is packaged as a :class:`Struct`.
     
     Notes
     -----
     A :class:`ConfigLoader` does one thing: load a config from a source 
     (file, command line arguments) and returns the data as a :class:`Struct`.
     There are lots of things that :class:`ConfigLoader` does not do.  It does
     not implement complex logic for finding config files.  It does not handle
     default values or merge multiple configs.  These things need to be 
     handled elsewhere.
     """
 
     def __init__(self):
         """A base class for config loaders.
         
         Examples
         --------
         
         >>> cl = ConfigLoader()
         >>> config = cl.load_config()
         >>> config
         {}
         """
         self.clear()
 
     def clear(self):
         self.config = Config()
 
     def load_config(self):
         """Load a config from somewhere, return a :class:`Config` instance.
         
         Usually, this will cause self.config to be set and then returned.
         However, in most cases, :meth:`ConfigLoader.clear` should be called
         to erase any previous state.
         """
         self.clear()
         return self.config
 
 
 class FileConfigLoader(ConfigLoader):
     """A base class for file based configurations.
 
     As we add more file based config loaders, the common logic should go
     here.
     """
     pass
 
 
 class PyFileConfigLoader(FileConfigLoader):
     """A config loader for pure python files.
     
     This calls execfile on a plain python file and looks for attributes
     that are all caps.  These attribute are added to the config Struct.
     """
 
     def __init__(self, filename, path=None):
         """Build a config loader for a filename and path.
 
         Parameters
         ----------
         filename : str
             The file name of the config file.
         path : str, list, tuple
             The path to search for the config file on, or a sequence of
             paths to try in order.
         """
         super(PyFileConfigLoader, self).__init__()
         self.filename = filename
         self.path = path
         self.full_filename = ''
         self.data = None
 
     def load_config(self):
         """Load the config from a file and return it as a Struct."""
         self.clear()
         self._find_file()
         self._read_file_as_dict()
         self._convert_to_config()
         return self.config
 
     def _find_file(self):
         """Try to find the file by searching the paths."""
         self.full_filename = filefind(self.filename, self.path)
 
     def _read_file_as_dict(self):
         """Load the config file into self.config, with recursive loading."""
         # This closure is made available in the namespace that is used
         # to exec the config file.  This allows users to call
         # load_subconfig('myconfig.py') to load config files recursively.
         # It needs to be a closure because it has references to self.path
         # and self.config.  The sub-config is loaded with the same path
         # as the parent, but it uses an empty config which is then merged
         # with the parents.
         def load_subconfig(fname):
             loader = PyFileConfigLoader(fname, self.path)
             try:
                 sub_config = loader.load_config()
             except IOError:
                 # Pass silently if the sub config is not there. This happens
                 # when a user us using a profile, but not the default config.
                 pass
             else:
                 self.config._merge(sub_config)
 
         # Again, this needs to be a closure and should be used in config
         # files to get the config being loaded.
         def get_config():
             return self.config
 
         namespace = dict(load_subconfig=load_subconfig, get_config=get_config)
         execfile(self.full_filename, namespace)
 
     def _convert_to_config(self):
         if self.data is None:
             ConfigLoaderError('self.data does not exist')
 
 
 class CommandLineConfigLoader(ConfigLoader):
     """A config loader for command line arguments.
 
     As we add more command line based loaders, the common logic should go
     here.
     """
 
 
 class ArgParseConfigLoader(CommandLineConfigLoader):
 
-    def __init__(self, argv=None, arguments=(), *parser_args, **parser_kw):
+    def __init__(self, argv=None, *parser_args, **parser_kw):
         """Create a config loader for use with argparse.
 
         Parameters
         ----------
 
         argv : optional, list
           If given, used to read command-line arguments from, otherwise
           sys.argv[1:] is used.
 
-        arguments : optional, tuple
-          A tuple of two element tuples each having the form (args, kwargs).
-          Each such pair is passed to parser.add_argument(*args, **kwargs)
-          in sequence to configure the parser.
-
         parser_args : tuple
           A tuple of positional arguments that will be passed to the
           constructor of :class:`argparse.ArgumentParser`.
 
         parser_kw : dict
           A tuple of keyword arguments that will be passed to the
           constructor of :class:`argparse.ArgumentParser`.
         """
         super(CommandLineConfigLoader, self).__init__()
         if argv == None:
             argv = sys.argv[1:]
         self.argv = argv
-        self.arguments = arguments
         self.parser_args = parser_args
         kwargs = dict(argument_default=argparse.SUPPRESS)
         kwargs.update(parser_kw)
         self.parser_kw = kwargs
 
     def load_config(self, args=None):
         """Parse command line arguments and return as a Struct.
 
         Parameters
         ----------
 
         args : optional, list
           If given, a list with the structure of sys.argv[1:] to parse
           arguments from. If not given, the instance's self.argv attribute
           (given at construction time) is used."""
         self.clear()
         if args is None:
             args = self.argv
         self._create_parser()
         self._parse_args(args)
         self._convert_to_config()
         return self.config
 
     def get_extra_args(self):
         if hasattr(self, 'extra_args'):
             return self.extra_args
         else:
             return []
 
     def _create_parser(self):
         self.parser = ArgumentParser(*self.parser_args, **self.parser_kw)
         self._add_arguments()
-        self._add_other_arguments()
 
     def _add_arguments(self):
-        for argument in self.arguments:
-            # Remove any defaults in case people add them. We can't have 
-            # command line default because all default are determined by
-            # traited class attributes.
-            argument[1].pop('default', None)
-            self.parser.add_argument(*argument[0], **argument[1])
-
-    def _add_other_arguments(self):
-        """Meant for subclasses to add their own arguments."""
-        pass
+        raise NotImplementedError("subclasses must implement _add_arguments")
 
     def _parse_args(self, args):
         """self.parser->self.parsed_data""" 
         self.parsed_data, self.extra_args = self.parser.parse_known_args(args)
 
     def _convert_to_config(self):
         """self.parsed_data->self.config"""
         for k, v in vars(self.parsed_data).items():
             exec_str = 'self.config.' + k + '= v'
             exec exec_str in locals(), globals()
 
 

IPython/config/tests/test_loader.py

@@ -1,178 +1,174 b''
 #!/usr/bin/env python
 # encoding: utf-8
 """
 Tests for IPython.config.loader
 
 Authors:
 
 * Brian Granger
 * Fernando Perez (design help)
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008-2009  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import os
 from tempfile import mkstemp
 from unittest import TestCase
 
 from IPython.config.loader import (
     Config,
     PyFileConfigLoader, 
     ArgParseConfigLoader,
     ConfigError
 )
 
 #-----------------------------------------------------------------------------
 # Actual tests
 #-----------------------------------------------------------------------------
 
 
 pyfile = """
 c = get_config()
 c.a = 10
 c.b = 20
 c.Foo.Bar.value = 10
 c.Foo.Bam.value = range(10)
 c.D.C.value = 'hi there'
 """
 
 class TestPyFileCL(TestCase):
 
     def test_basic(self):
         fd, fname = mkstemp('.py')
         f = os.fdopen(fd, 'w')
         f.write(pyfile)
         f.close()
         # Unlink the file
         cl = PyFileConfigLoader(fname)
         config = cl.load_config()
         self.assertEquals(config.a, 10)
         self.assertEquals(config.b, 20)
         self.assertEquals(config.Foo.Bar.value, 10)
         self.assertEquals(config.Foo.Bam.value, range(10))
         self.assertEquals(config.D.C.value, 'hi there')
 
-
-arguments = (
-        (('-f','--foo'), dict(dest='Global.foo', type=str)),
-        (('-b',), dict(dest='MyClass.bar', type=int)),
-        (('-n',), dict(dest='n', action='store_true')),
-        (('Global.bam',), dict(type=str))
-    )
+class MyLoader1(ArgParseConfigLoader):
+    def _add_arguments(self):
+        p = self.parser
+        p.add_argument('-f', '--foo', dest='Global.foo', type=str)
+        p.add_argument('-b', dest='MyClass.bar', type=int)
+        p.add_argument('-n', dest='n', action='store_true')
+        p.add_argument('Global.bam', type=str)
+
+class MyLoader2(ArgParseConfigLoader):
+    def _add_arguments(self):
+        subparsers = self.parser.add_subparsers(dest='subparser_name')
+        subparser1 = subparsers.add_parser('1')
+        subparser1.add_argument('-x',dest='Global.x')
+        subparser2 = subparsers.add_parser('2')
+        subparser2.add_argument('y')
 
 class TestArgParseCL(TestCase):
 
     def test_basic(self):
-        cl = ArgParseConfigLoader(arguments=arguments)
+        cl = MyLoader1()
         config = cl.load_config('-f hi -b 10 -n wow'.split())
         self.assertEquals(config.Global.foo, 'hi')
         self.assertEquals(config.MyClass.bar, 10)
         self.assertEquals(config.n, True)
         self.assertEquals(config.Global.bam, 'wow')
         config = cl.load_config(['wow'])
         self.assertEquals(config.keys(), ['Global'])
         self.assertEquals(config.Global.keys(), ['bam'])
         self.assertEquals(config.Global.bam, 'wow')
 
     def test_add_arguments(self):
-
-        class MyLoader(ArgParseConfigLoader):
-            def _add_arguments(self):
-                subparsers = self.parser.add_subparsers(dest='subparser_name')
-                subparser1 = subparsers.add_parser('1')
-                subparser1.add_argument('-x',dest='Global.x')
-                subparser2 = subparsers.add_parser('2')
-                subparser2.add_argument('y')
-
-        cl = MyLoader()
+        cl = MyLoader2()
         config = cl.load_config('2 frobble'.split())
         self.assertEquals(config.subparser_name, '2')
         self.assertEquals(config.y, 'frobble')
         config = cl.load_config('1 -x frobble'.split())
         self.assertEquals(config.subparser_name, '1')
         self.assertEquals(config.Global.x, 'frobble')
 
     def test_argv(self):
-        cl = ArgParseConfigLoader(
-            argv='-f hi -b 10 -n wow'.split(), 
-            arguments=arguments
-        )
+        cl = MyLoader1(argv='-f hi -b 10 -n wow'.split())
         config = cl.load_config()
         self.assertEquals(config.Global.foo, 'hi')
         self.assertEquals(config.MyClass.bar, 10)
         self.assertEquals(config.n, True)
         self.assertEquals(config.Global.bam, 'wow')
 
 
 class TestConfig(TestCase):
 
     def test_setget(self):
         c = Config()
         c.a = 10
         self.assertEquals(c.a, 10)
         self.assertEquals(c.has_key('b'), False)
 
     def test_auto_section(self):
         c = Config()
         self.assertEquals(c.has_key('A'), True)
         self.assertEquals(c._has_section('A'), False)
         A = c.A
         A.foo = 'hi there'
         self.assertEquals(c._has_section('A'), True)
         self.assertEquals(c.A.foo, 'hi there')
         del c.A
         self.assertEquals(len(c.A.keys()),0)
 
     def test_merge_doesnt_exist(self):
         c1 = Config()
         c2 = Config()
         c2.bar = 10
         c2.Foo.bar = 10
         c1._merge(c2)
         self.assertEquals(c1.Foo.bar, 10)
         self.assertEquals(c1.bar, 10)
         c2.Bar.bar = 10
         c1._merge(c2)
         self.assertEquals(c1.Bar.bar, 10)
 
     def test_merge_exists(self):
         c1 = Config()
         c2 = Config()
         c1.Foo.bar = 10
         c1.Foo.bam = 30
         c2.Foo.bar = 20
         c2.Foo.wow = 40
         c1._merge(c2)
         self.assertEquals(c1.Foo.bam, 30)
         self.assertEquals(c1.Foo.bar, 20)
         self.assertEquals(c1.Foo.wow, 40)
         c2.Foo.Bam.bam = 10
         c1._merge(c2)
         self.assertEquals(c1.Foo.Bam.bam, 10)
 
     def test_deepcopy(self):
         c1 = Config()
         c1.Foo.bar = 10
         c1.Foo.bam = 30
         c1.a = 'asdf'
         c1.b = range(10)
         import copy
         c2 = copy.deepcopy(c1)
         self.assertEquals(c1, c2)
         self.assert_(c1 is not c2)
         self.assert_(c1.Foo is not c2.Foo)
 
     def test_builtin(self):
         c1 = Config()
         exec 'foo = True' in c1
         self.assertEquals(c1.foo, True)
         self.assertRaises(ConfigError, setattr, c1, 'ValueError', 10)

IPython/core/application.py

@@ -1,489 +1,463 b''
 # encoding: utf-8
 """
 An application for IPython.
 
 All top-level applications should use the classes in this module for
 handling configuration and creating componenets.
 
 The job of an :class:`Application` is to create the master configuration 
 object and then create the components, passing the config to them.
 
 Authors:
 
 * Brian Granger
 * Fernando Perez
 
 Notes
 -----
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008-2009  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import logging
 import os
 import sys
 
 from IPython.core import release, crashhandler
 from IPython.utils.path import get_ipython_dir, get_ipython_package_dir
 from IPython.config.loader import (
     PyFileConfigLoader,
     ArgParseConfigLoader,
     Config,
 )
 
 #-----------------------------------------------------------------------------
 # Classes and functions
 #-----------------------------------------------------------------------------
 
 class ApplicationError(Exception):
     pass
 
 
-app_cl_args = (
-    (('--ipython-dir', ), dict(
-        dest='Global.ipython_dir',type=unicode,
-        help=
-        """Set to override default location of the IPython directory
-        IPYTHON_DIR, stored as Global.ipython_dir.  This can also be specified
-        through the environment variable IPYTHON_DIR.""",
-        metavar='Global.ipython_dir') ),
-    (('-p', '--profile',), dict(
-        dest='Global.profile',type=unicode,
-        help=
-        """The string name of the ipython profile to be used.  Assume that your
-        config file is ipython_config-<name>.py (looks in current dir first,
-        then in IPYTHON_DIR). This is a quick way to keep and load multiple
-        config files for different tasks, especially if include your basic one
-        in your more specialized ones.  You can keep a basic
-        IPYTHON_DIR/ipython_config.py file and then have other 'profiles' which
-        include this one and load extra things for particular tasks.""",
-        metavar='Global.profile') ),
-    (('--log-level',), dict(
-        dest="Global.log_level",type=int,
-        help='Set the log level (0,10,20,30,40,50).  Default is 30.',
-        metavar='Global.log_level')),
-    (('--config-file',), dict(
-        dest='Global.config_file',type=unicode,
-        help=
-        """Set the config file name to override default.  Normally IPython
-        loads ipython_config.py (from current directory) or
-        IPYTHON_DIR/ipython_config.py.  If the loading of your config file
-        fails, IPython starts with a bare bones configuration (no modules
-        loaded at all).""",
-        metavar='Global.config_file')),
-    )
+class BaseAppConfigLoader(ArgParseConfigLoader):
+    """Default command line options for IPython based applications."""
+
+    def _add_ipython_dir(self, parser):
+        """Add the --ipython-dir option to the parser."""
+        paa = parser.add_argument
+        paa('--ipython-dir', 
+            dest='Global.ipython_dir',type=unicode,
+            help=
+            """Set to override default location of the IPython directory
+            IPYTHON_DIR, stored as Global.ipython_dir.  This can also be 
+            specified through the environment variable IPYTHON_DIR.""",
+            metavar='Global.ipython_dir')
+
+    def _add_log_level(self, parser):
+        """Add the --log-level option to the parser."""
+        paa = parser.add_argument
+        paa('--log-level',
+            dest="Global.log_level",type=int,
+            help='Set the log level (0,10,20,30,40,50).  Default is 30.',
+            metavar='Global.log_level')
+
+    def _add_arguments(self):
+        self._add_ipython_dir(self.parser)
+        self._add_log_level(self.parser)
+
 
 class Application(object):
     """Load a config, construct components and set them running.
 
-    The configuration of an application can be done via four different Config
-    objects, which are loaded and ultimately merged into a single one used from
-    that point on by the app.  These are:
+    The configuration of an application can be done via three different Config
+    objects, which are loaded and ultimately merged into a single one used
+    from that point on by the app. These are:
 
     1. default_config: internal defaults, implemented in code.
     2. file_config: read from the filesystem.
     3. command_line_config: read from the system's command line flags.
-    4. constructor_config: passed parametrically to the constructor.
 
     During initialization, 3 is actually read before 2, since at the
     command-line one may override the location of the file to be read.  But the
     above is the order in which the merge is made.
-
-    There is a final config object can be created and passed to the
-    constructor: override_config.  If it exists, this completely overrides the
-    configs 2-4 above (the default is still used to ensure that all needed
-    fields at least are created).  This makes it easier to create
-    parametrically (e.g. in testing or sphinx plugins) objects with a known
-    configuration, that are unaffected by whatever arguments may be present in
-    sys.argv or files in the user's various directories.
     """
 
     name = u'ipython'
     description = 'IPython: an enhanced interactive Python shell.'
-    #: usage message printed by argparse. If None, auto-generate
+    #: Usage message printed by argparse. If None, auto-generate
     usage = None
+    #: The command line config loader.  Subclass of ArgParseConfigLoader.
+    command_line_loader = BaseAppConfigLoader
+    #: The name of the config file to load.
     config_file_name = u'ipython_config.py'
-    #: Track the default and actual separately because some messages are
-    #: only printed if we aren't using the default.
+    #: The name of the default config file. Track separately from the actual
+    #: name because some logic happens only if we aren't using the default.
     default_config_file_name = config_file_name
     default_log_level = logging.WARN
     #: Set by --profile option
     profile_name = None
     #: User's ipython directory, typically ~/.ipython/
     ipython_dir = None
-    #: internal defaults, implemented in code.
+    #: Internal defaults, implemented in code.
     default_config = None
-    #: read from the filesystem
+    #: Read from the filesystem.
     file_config = None
-    #: read from the system's command line flags
+    #: Read from the system's command line flags.
     command_line_config = None
-    #: passed parametrically to the constructor.
-    constructor_config = None
-    #: final override, if given supercedes file/command/constructor configs
-    override_config = None
+    #: The final config that will be passed to the component.
+    master_config = None
     #: A reference to the argv to be used (typically ends up being sys.argv[1:])
     argv = None
-    #: Default command line arguments.  Subclasses should create a new tuple
-    #: that *includes* these.
-    cl_arguments = app_cl_args
-
     #: extra arguments computed by the command-line loader
     extra_args = None
 
     # Private attributes
     _exiting = False
     _initialized = False
 
     # Class choices for things that will be instantiated at runtime.
     _CrashHandler = crashhandler.CrashHandler
 
-    def __init__(self, argv=None, constructor_config=None, override_config=None):
+    def __init__(self, argv=None):
         self.argv = sys.argv[1:] if argv is None else argv
-        self.constructor_config = constructor_config
-        self.override_config = override_config
         self.init_logger()
 
     def init_logger(self):
         self.log = logging.getLogger(self.__class__.__name__)
         # This is used as the default until the command line arguments are read.
         self.log.setLevel(self.default_log_level)
         self._log_handler = logging.StreamHandler()
         self._log_formatter = logging.Formatter("[%(name)s] %(message)s")
         self._log_handler.setFormatter(self._log_formatter)
         self.log.addHandler(self._log_handler)
 
     def _set_log_level(self, level):
         self.log.setLevel(level)
 
     def _get_log_level(self):
         return self.log.level
 
     log_level = property(_get_log_level, _set_log_level)
 
     def initialize(self):
         """Initialize the application.
 
         Loads all configuration information and sets all application state, but
         does not start any relevant processing (typically some kind of event
         loop).
 
         Once this method has been called, the application is flagged as
         initialized and the method becomes a no-op."""
         
         if self._initialized:
             return
 
         # The first part is protected with an 'attempt' wrapper, that will log
         # failures with the basic system traceback machinery.  Once our crash
         # handler is in place, we can let any subsequent exception propagate,
         # as our handler will log it with much better detail than the default.
         self.attempt(self.create_crash_handler)
 
         # Configuration phase
         # Default config (internally hardwired in application code)
         self.create_default_config()
         self.log_default_config()
         self.set_default_config_log_level()
 
-        if self.override_config is None:
-            # Command-line config
-            self.pre_load_command_line_config()
-            self.load_command_line_config()
-            self.set_command_line_config_log_level()
-            self.post_load_command_line_config()
-            self.log_command_line_config()
+        # Command-line config
+        self.pre_load_command_line_config()
+        self.load_command_line_config()
+        self.set_command_line_config_log_level()
+        self.post_load_command_line_config()
+        self.log_command_line_config()
 
         # Find resources needed for filesystem access, using information from
         # the above two
         self.find_ipython_dir()
         self.find_resources()
         self.find_config_file_name()
         self.find_config_file_paths()
 
-        if self.override_config is None:
-            # File-based config
-            self.pre_load_file_config()
-            self.load_file_config()
-            self.set_file_config_log_level()
-            self.post_load_file_config()
-            self.log_file_config()
+        # File-based config
+        self.pre_load_file_config()
+        self.load_file_config()
+        self.set_file_config_log_level()
+        self.post_load_file_config()
+        self.log_file_config()
 
         # Merge all config objects into a single one the app can then use
         self.merge_configs()
         self.log_master_config()
 
         # Construction phase
         self.pre_construct()
         self.construct()
         self.post_construct()
 
         # Done, flag as such and
         self._initialized = True
 
     def start(self):
         """Start the application."""
         self.initialize()
         self.start_app()
 
     #-------------------------------------------------------------------------
     # Various stages of Application creation
     #-------------------------------------------------------------------------
 
     def create_crash_handler(self):
         """Create a crash handler, typically setting sys.excepthook to it."""
         self.crash_handler = self._CrashHandler(self, self.name)
         sys.excepthook = self.crash_handler
 
     def create_default_config(self):
         """Create defaults that can't be set elsewhere.
 
         For the most part, we try to set default in the class attributes
         of Components.  But, defaults the top-level Application (which is
         not a HasTraitlets or Component) are not set in this way.  Instead
         we set them here.  The Global section is for variables like this that
         don't belong to a particular component.
         """
         c = Config()
         c.Global.ipython_dir = get_ipython_dir()
         c.Global.log_level = self.log_level
         self.default_config = c
 
     def log_default_config(self):
         self.log.debug('Default config loaded:')
         self.log.debug(repr(self.default_config))
 
     def set_default_config_log_level(self):
         try:
             self.log_level = self.default_config.Global.log_level
         except AttributeError:
             # Fallback to the default_log_level class attribute
             pass
 
     def create_command_line_config(self):
         """Create and return a command line config loader."""
-        return ArgParseConfigLoader(self.argv, self.cl_arguments,
-                                    description=self.description, 
-                                    version=release.version,
-                                    usage=self.usage,
-                                    )
+        return self.command_line_loader(
+            self.argv,
+            description=self.description, 
+            version=release.version,
+            usage=self.usage
+        )
 
     def pre_load_command_line_config(self):
         """Do actions just before loading the command line config."""
         pass
 
     def load_command_line_config(self):
         """Load the command line config."""
         loader = self.create_command_line_config()
         self.command_line_config = loader.load_config()
         self.extra_args = loader.get_extra_args()
 
     def set_command_line_config_log_level(self):
         try:
             self.log_level = self.command_line_config.Global.log_level
         except AttributeError:
             pass
 
     def post_load_command_line_config(self):
         """Do actions just after loading the command line config."""
         pass
 
     def log_command_line_config(self):
         self.log.debug("Command line config loaded:")
         self.log.debug(repr(self.command_line_config))
 
     def find_ipython_dir(self):
         """Set the IPython directory.
 
         This sets ``self.ipython_dir``, but the actual value that is passed to
         the application is kept in either ``self.default_config`` or
         ``self.command_line_config``.  This also adds ``self.ipython_dir`` to
         ``sys.path`` so config files there can be referenced by other config
         files.
         """
 
         try:
             self.ipython_dir = self.command_line_config.Global.ipython_dir
         except AttributeError:
             self.ipython_dir = self.default_config.Global.ipython_dir
         sys.path.append(os.path.abspath(self.ipython_dir))
         if not os.path.isdir(self.ipython_dir):
             os.makedirs(self.ipython_dir, mode=0777)
         self.log.debug("IPYTHON_DIR set to: %s" % self.ipython_dir)
 
     def find_resources(self):
         """Find other resources that need to be in place.
 
         Things like cluster directories need to be in place to find the
         config file.  These happen right after the IPython directory has
         been set.
         """
         pass
 
     def find_config_file_name(self):
         """Find the config file name for this application.
 
         This must set ``self.config_file_name`` to the filename of the
         config file to use (just the filename). The search paths for the
         config file are set in :meth:`find_config_file_paths` and then passed
         to the config file loader where they are resolved to an absolute path.
 
         If a profile has been set at the command line, this will resolve it.
         """
 
         try:
             self.config_file_name = self.command_line_config.Global.config_file
         except AttributeError:
             pass
 
         try:
             self.profile_name = self.command_line_config.Global.profile
         except AttributeError:
             pass
         else:
             name_parts = self.config_file_name.split('.')
             name_parts.insert(1, u'_' + self.profile_name + u'.')
             self.config_file_name = ''.join(name_parts)
 
     def find_config_file_paths(self):
         """Set the search paths for resolving the config file.
 
         This must set ``self.config_file_paths`` to a sequence of search
         paths to pass to the config file loader.
         """
         # Include our own profiles directory last, so that users can still find
         # our shipped copies of builtin profiles even if they don't have them
         # in their local ipython directory.
         prof_dir = os.path.join(get_ipython_package_dir(), 'config', 'profile')
         self.config_file_paths = (os.getcwd(), self.ipython_dir, prof_dir)
 
     def pre_load_file_config(self):
         """Do actions before the config file is loaded."""
         pass
 
     def load_file_config(self):
         """Load the config file.
         
         This tries to load the config file from disk.  If successful, the
         ``CONFIG_FILE`` config variable is set to the resolved config file
         location.  If not successful, an empty config is used.
         """
         self.log.debug("Attempting to load config file: %s" %
                        self.config_file_name)
         loader = PyFileConfigLoader(self.config_file_name,
                                     path=self.config_file_paths)
         try:
             self.file_config = loader.load_config()
             self.file_config.Global.config_file = loader.full_filename
         except IOError:
             # Only warn if the default config file was NOT being used.
             if not self.config_file_name==self.default_config_file_name:
                 self.log.warn("Config file not found, skipping: %s" %
                                self.config_file_name, exc_info=True)
             self.file_config = Config()
         except:
             self.log.warn("Error loading config file: %s" %
                           self.config_file_name, exc_info=True)
             self.file_config = Config()
 
     def set_file_config_log_level(self):
         # We need to keeep self.log_level updated.  But we only use the value
         # of the file_config if a value was not specified at the command
         # line, because the command line overrides everything.
         if not hasattr(self.command_line_config.Global, 'log_level'):
             try:
                 self.log_level = self.file_config.Global.log_level
             except AttributeError:
                 pass # Use existing value
 
     def post_load_file_config(self):
         """Do actions after the config file is loaded."""
         pass
 
     def log_file_config(self):
         if hasattr(self.file_config.Global, 'config_file'):
             self.log.debug("Config file loaded: %s" %
                            self.file_config.Global.config_file)
             self.log.debug(repr(self.file_config))
 
     def merge_configs(self):
         """Merge the default, command line and file config objects."""
         config = Config()
         config._merge(self.default_config)
-        if self.override_config is None:
-            config._merge(self.file_config)
-            config._merge(self.command_line_config)
-            if self.constructor_config is not None:
-                config._merge(self.constructor_config)
-        else:
-            config._merge(self.override_config)
+        config._merge(self.file_config)
+        config._merge(self.command_line_config)
+
         # XXX fperez - propose to Brian we rename master_config to simply
         # config, I think this is going to be heavily used in examples and
         # application code and the name is shorter/easier to find/remember.
         # For now, just alias it...
         self.master_config = config
         self.config = config
 
     def log_master_config(self):
         self.log.debug("Master config created:")
         self.log.debug(repr(self.master_config))
 
     def pre_construct(self):
         """Do actions after the config has been built, but before construct."""
         pass
 
     def construct(self):
         """Construct the main components that make up this app."""
         self.log.debug("Constructing components for application")
 
     def post_construct(self):
         """Do actions after construct, but before starting the app."""
         pass
 
     def start_app(self):
         """Actually start the app."""
         self.log.debug("Starting application")
 
     #-------------------------------------------------------------------------
     # Utility methods
     #-------------------------------------------------------------------------
 
     def abort(self):
         """Abort the starting of the application."""
         if self._exiting:
             pass
         else:
             self.log.critical("Aborting application: %s" % self.name, exc_info=True)
             self._exiting = True
             sys.exit(1)
 
     def exit(self, exit_status=0):
         if self._exiting:
             pass
         else:
             self.log.debug("Exiting application: %s" % self.name)
             self._exiting = True
             sys.exit(exit_status)
 
     def attempt(self, func, action='abort'):
         try:
             func()
         except SystemExit:
             raise
         except:
             if action == 'abort':
                 self.log.critical("Aborting application: %s" % self.name,
                                   exc_info=True)
                 self.abort()
                 raise
             elif action == 'exit':
                 self.exit(0)
 

IPython/core/crashhandler.py

@@ -1,223 +1,224 b''
 # -*- coding: utf-8 -*-
 """sys.excepthook for IPython itself, leaves a detailed report on disk.
 
 
 Authors
 -------
 - Fernando Perez <[e-mail]>
 """
 
 #*****************************************************************************
 #       Copyright (C) 2008-2009 The IPython Development Team
 #       Copyright (C) 2001-2007 Fernando Perez. <[e-mail]>
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #*****************************************************************************
 
 #****************************************************************************
 # Required modules
 
 # From the standard library
 import os
 import sys
 from pprint import pformat
 
 # Our own
 from IPython.core import release
 from IPython.core import ultratb
 from IPython.utils.sysinfo import sys_info
 
 from IPython.external.Itpl import itpl
 
 #****************************************************************************
 
 class CrashHandler(object):
     """Customizable crash handlers for IPython-based systems.
 
     Instances of this class provide a __call__ method which can be used as a
     sys.excepthook, i.e., the __call__ signature is:
 
         def __call__(self,etype, evalue, etb)
 
     """
 
     def __init__(self,app, app_name, contact_name=None, contact_email=None, 
                  bug_tracker=None, crash_report_fname='CrashReport.txt', 
                  show_crash_traceback=True, call_pdb=False):
         """New crash handler.
 
         Inputs:
 
         - app: a running application instance, which will be queried at crash
         time for internal information.
 
         - app_name: a string containing the name of your application.
 
         - contact_name: a string with the name of the person to contact.
 
         - contact_email: a string with the email address of the contact.
 
         - bug_tracker: a string with the URL for your project's bug tracker.
 
         - crash_report_fname: a string with the filename for the crash report
         to be saved in.  These reports are left in the ipython user directory
         as determined by the running IPython instance.
 
         Optional inputs:
         
         - show_crash_traceback(True): if false, don't print the crash
         traceback on stderr, only generate the on-disk report
 
 
         Non-argument instance attributes:
 
         These instances contain some non-argument attributes which allow for 
         further customization of the crash handler's behavior.  Please see the
         source for further details.
         """
 
         # apply args into instance
         self.app = app
         self.app_name = app_name
         self.contact_name = contact_name
         self.contact_email = contact_email
         self.bug_tracker = bug_tracker
         self.crash_report_fname = crash_report_fname
         self.show_crash_traceback = show_crash_traceback
         self.section_sep = '\n\n'+'*'*75+'\n\n'
         self.call_pdb = call_pdb
         #self.call_pdb = True # dbg
         
         # Hardcoded defaults, which can be overridden either by subclasses or
         # at runtime for the instance.
 
         # Template for the user message.  Subclasses which completely override
         # this, or user apps, can modify it to suit their tastes.  It gets
         # expanded using itpl, so calls of the kind $self.foo are valid.
         self.user_message_template = """
 Oops, $self.app_name crashed. We do our best to make it stable, but...
 
 A crash report was automatically generated with the following information:
   - A verbatim copy of the crash traceback.
   - A copy of your input history during this session.
   - Data on your current $self.app_name configuration.
 
 It was left in the file named:
 \t'$self.crash_report_fname'
 If you can email this file to the developers, the information in it will help
 them in understanding and correcting the problem.
 
 You can mail it to: $self.contact_name at $self.contact_email
 with the subject '$self.app_name Crash Report'.
 
 If you want to do it now, the following command will work (under Unix):
 mail -s '$self.app_name Crash Report' $self.contact_email < $self.crash_report_fname
 
 To ensure accurate tracking of this issue, please file a report about it at:
 $self.bug_tracker
 """
 
     def __call__(self,etype, evalue, etb):
         """Handle an exception, call for compatible with sys.excepthook"""
 
         # Report tracebacks shouldn't use color in general (safer for users)
         color_scheme = 'NoColor'
 
         # Use this ONLY for developer debugging (keep commented out for release)
         #color_scheme = 'Linux'   # dbg
         
         try:
             rptdir = self.app.ipython_dir
         except:
             rptdir = os.getcwd()
         if not os.path.isdir(rptdir):
             rptdir = os.getcwd()
         report_name = os.path.join(rptdir,self.crash_report_fname)
         # write the report filename into the instance dict so it can get
         # properly expanded out in the user message template
         self.crash_report_fname = report_name
         TBhandler = ultratb.VerboseTB(color_scheme=color_scheme,
                                       long_header=1,
                                       call_pdb=self.call_pdb,
                                       )
         if self.call_pdb:
             TBhandler(etype,evalue,etb)
             return
         else:
             traceback = TBhandler.text(etype,evalue,etb,context=31)
 
         # print traceback to screen
         if self.show_crash_traceback:
             print >> sys.stderr, traceback
 
         # and generate a complete report on disk
         try:
             report = open(report_name,'w')
         except:
             print >> sys.stderr, 'Could not create crash report on disk.'
             return
 
         # Inform user on stderr of what happened
         msg = itpl('\n'+'*'*70+'\n'+self.user_message_template)
         print >> sys.stderr, msg
 
         # Construct report on disk
         report.write(self.make_report(traceback))
         report.close()
         raw_input("Hit <Enter> to quit this message (your terminal may close):")
 
     def make_report(self,traceback):
         """Return a string containing a crash report."""
         
         sec_sep = self.section_sep
         
         report = ['*'*75+'\n\n'+'IPython post-mortem report\n\n']
         rpt_add = report.append
         rpt_add(sys_info())
         
         try:
             config = pformat(self.app.config)
             rpt_add(sec_sep+'Current user configuration structure:\n\n')
             rpt_add(config)
         except:
             pass
         rpt_add(sec_sep+'Crash traceback:\n\n' + traceback)
 
         return ''.join(report)
 
 
 class IPythonCrashHandler(CrashHandler):
     """sys.excepthook for IPython itself, leaves a detailed report on disk."""
     
     def __init__(self, app, app_name='IPython'):
 
         # Set here which of the IPython authors should be listed as contact
         AUTHOR_CONTACT = 'Fernando'
         
         # Set argument defaults
         bug_tracker = 'https://bugs.launchpad.net/ipython/+filebug'
         contact_name,contact_email = release.authors[AUTHOR_CONTACT][:2]
         crash_report_fname = 'IPython_crash_report.txt'
         # Call parent constructor
         CrashHandler.__init__(self,app,app_name,contact_name,contact_email,
                               bug_tracker,crash_report_fname)
 
     def make_report(self,traceback):
         """Return a string containing a crash report."""
 
         sec_sep = self.section_sep
         # Start with parent report
         report = [super(IPythonCrashHandler, self).make_report(traceback)]
         # Add interactive-specific info we may have
         rpt_add = report.append
         try:
             rpt_add(sec_sep+"History of session input:")
             for line in self.app.shell.user_ns['_ih']:
                 rpt_add(line)
             rpt_add('\n*** Last line of input (may not be in above history):\n')
             rpt_add(self.app.shell._last_input_line+'\n')
         except:
             pass
 
         return ''.join(report)
+

IPython/core/ipapp.py

@@ -1,648 +1,590 b''
 #!/usr/bin/env python
 # encoding: utf-8
 """
 The :class:`~IPython.core.application.Application` object for the command
 line :command:`ipython` program.
 
 Authors
 -------
 
 * Brian Granger
 * Fernando Perez
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008-2010  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 from __future__ import absolute_import
 
 import logging
 import os
 import sys
 
 from IPython.core import crashhandler
-from IPython.core.application import Application
+from IPython.core.application import Application, BaseAppConfigLoader
 from IPython.core.iplib import InteractiveShell
 from IPython.config.loader import (
     Config,
     PyFileConfigLoader
 )
 from IPython.lib import inputhook
 from IPython.utils.path import filefind, get_ipython_dir
 from . import usage
 
 #-----------------------------------------------------------------------------
 # Globals, utilities and helpers
 #-----------------------------------------------------------------------------
 
+#: The default config file name for this application.
 default_config_file_name = u'ipython_config.py'
 
-cl_args = (
-    (('--autocall',), dict(
-        type=int, dest='InteractiveShell.autocall',
-        help=
-        """Make IPython automatically call any callable object even if you
-        didn't type explicit parentheses. For example, 'str 43' becomes
-        'str(43)' automatically.  The value can be '0' to disable the feature,
-        '1' for 'smart' autocall, where it is not applied if there are no more
-        arguments on the line, and '2' for 'full' autocall, where all callable
-        objects are automatically called (even if no arguments are present).
-        The default is '1'.""",
-        metavar='InteractiveShell.autocall')
-    ),
-    (('--autoindent',), dict(
-        action='store_true', dest='InteractiveShell.autoindent',
-        help='Turn on autoindenting.')
-    ),
-    (('--no-autoindent',), dict(
-        action='store_false', dest='InteractiveShell.autoindent',
-        help='Turn off autoindenting.')
-    ),
-    (('--automagic',), dict(
-        action='store_true', dest='InteractiveShell.automagic',
-        help='Turn on the auto calling of magic commands.'
-        'Type %%magic at  the  IPython  prompt  for  more information.')
-     ),
-    (('--no-automagic',), dict(
-        action='store_false', dest='InteractiveShell.automagic',
-        help='Turn off the auto calling of magic commands.')
-    ),
-    (('--autoedit-syntax',), dict(
-        action='store_true', dest='InteractiveShell.autoedit_syntax',
-        help='Turn on auto editing of files with syntax errors.')
-    ),
-    (('--no-autoedit-syntax',), dict(
-        action='store_false', dest='InteractiveShell.autoedit_syntax',
-        help='Turn off auto editing of files with syntax errors.')
-    ),
-    (('--banner',), dict(
-        action='store_true', dest='Global.display_banner',
-        help='Display a banner upon starting IPython.')
-    ),
-    (('--no-banner',), dict(
-        action='store_false', dest='Global.display_banner',
-        help="Don't display a banner upon starting IPython.")
-    ),
-    (('--cache-size',), dict(
-        type=int, dest='InteractiveShell.cache_size',
-        help=
-        """Set the size of the output cache.  The default is 1000, you can
-        change it permanently in your config file.  Setting it to 0 completely
-        disables the caching system, and the minimum value accepted is 20 (if
-        you provide a value less than 20, it is reset to 0 and a warning is
-        issued).  This limit is defined because otherwise you'll spend more
-        time re-flushing a too small cache than working.
-        """,
-        metavar='InteractiveShell.cache_size')
-    ),
-    (('--classic',), dict(
-        action='store_true', dest='Global.classic',
-        help="Gives IPython a similar feel to the classic Python prompt.")
-    ),
-    (('--colors',), dict(
-        type=str, dest='InteractiveShell.colors',
-        help="Set the color scheme (NoColor, Linux, and LightBG).",
-        metavar='InteractiveShell.colors')
-    ),
-    (('--color-info',), dict(
-        action='store_true', dest='InteractiveShell.color_info',
-        help=
-        """IPython can display information about objects via a set of func-
-        tions, and optionally can use colors for this, syntax highlighting
-        source code and various other elements.  However, because this
-        information is passed through a pager (like 'less') and many pagers get
-        confused with color codes, this option is off by default.  You can test
-        it and turn it on permanently in your ipython_config.py file if it
-        works for you.  Test it and turn it on permanently if it works with
-        your system.  The magic function %%color_info allows you to toggle this
-        inter- actively for testing."""
-        )
-    ),
-    (('--no-color-info',), dict(
-        action='store_false', dest='InteractiveShell.color_info',
-        help="Disable using colors for info related things.")
-    ),
-    (('--confirm-exit',), dict(
-        action='store_true', dest='InteractiveShell.confirm_exit',
-        help=
-        """Set to confirm when you try to exit IPython with an EOF (Control-D
-        in Unix, Control-Z/Enter in Windows). By typing 'exit', 'quit' or
-        '%%Exit', you can force a direct exit without any confirmation.
-        """
-        )
-    ),
-    (('--no-confirm-exit',), dict(
-        action='store_false', dest='InteractiveShell.confirm_exit',
-        help="Don't prompt the user when exiting.")
-    ),
-    (('--deep-reload',), dict(
-        action='store_true', dest='InteractiveShell.deep_reload',
-        help=
-        """Enable deep (recursive) reloading by default. IPython can use the
-        deep_reload module which reloads changes in modules recursively (it
-        replaces the reload() function, so you don't need to change anything to
-        use it). deep_reload() forces a full reload of modules whose code may
-        have changed, which the default reload() function does not.  When
-        deep_reload is off, IPython will use the normal reload(), but
-        deep_reload will still be available as dreload(). This fea- ture is off
-        by default [which means that you have both normal reload() and
-        dreload()].""")
-    ),
-    (('--no-deep-reload',), dict(
-        action='store_false', dest='InteractiveShell.deep_reload',
-        help="Disable deep (recursive) reloading by default.")
-    ),
-    (('--editor',), dict(
-        type=str, dest='InteractiveShell.editor',
-        help="Set the editor used by IPython (default to $EDITOR/vi/notepad).",
-        metavar='InteractiveShell.editor')
-    ),
-    (('--log','-l'), dict(
-        action='store_true', dest='InteractiveShell.logstart',
-        help="Start logging to the default log file (./ipython_log.py).")
-    ),
-    (('--logfile','-lf'), dict(
-        type=unicode, dest='InteractiveShell.logfile',
-        help="Start logging to logfile with this name.",
-        metavar='InteractiveShell.logfile')
-    ),
-    (('--log-append','-la'), dict(
-        type=unicode, dest='InteractiveShell.logappend',
-        help="Start logging to the given file in append mode.",
-        metavar='InteractiveShell.logfile')
-    ),
-    (('--pdb',), dict(
-        action='store_true', dest='InteractiveShell.pdb',
-        help="Enable auto calling the pdb debugger after every exception.")
-    ),
-    (('--no-pdb',), dict(
-        action='store_false', dest='InteractiveShell.pdb',
-        help="Disable auto calling the pdb debugger after every exception.")
-    ),
-    (('--pprint',), dict(
-        action='store_true', dest='InteractiveShell.pprint',
-        help="Enable auto pretty printing of results.")
-    ),
-    (('--no-pprint',), dict(
-        action='store_false', dest='InteractiveShell.pprint',
-        help="Disable auto auto pretty printing of results.")
-    ),
-    (('--prompt-in1','-pi1'), dict(
-        type=str, dest='InteractiveShell.prompt_in1',
-        help=
-        """Set the main input prompt ('In [\#]: ').  Note that if you are using
-        numbered prompts, the number is represented with a '\#' in the string.
-        Don't forget to quote strings with spaces embedded in them. Most
-        bash-like escapes can be used to customize IPython's prompts, as well
-        as a few additional ones which are IPython-spe- cific.  All valid
-        prompt escapes are described in detail in the Customization section of
-        the IPython manual.""",
-        metavar='InteractiveShell.prompt_in1')
-    ),
-    (('--prompt-in2','-pi2'), dict(
-        type=str, dest='InteractiveShell.prompt_in2',
-        help=
-        """Set the secondary input prompt (' .\D.: ').  Similar to the previous
-        option, but used for the continuation prompts. The special sequence
-        '\D' is similar to '\#', but with all digits replaced by dots (so you
-        can have your continuation prompt aligned with your input prompt).
-        Default: ' .\D.: ' (note three spaces at the start for alignment with
-        'In [\#]')""",
-        metavar='InteractiveShell.prompt_in2')
-    ),
-    (('--prompt-out','-po'), dict(
-        type=str, dest='InteractiveShell.prompt_out',
-        help="Set the output prompt ('Out[\#]:')",
-        metavar='InteractiveShell.prompt_out')
-    ),
-    (('--quick',), dict(
-        action='store_true', dest='Global.quick',
-        help="Enable quick startup with no config files.")
-    ),
-    (('--readline',), dict(
-        action='store_true', dest='InteractiveShell.readline_use',
-        help="Enable readline for command line usage.")
-    ),
-    (('--no-readline',), dict(
-        action='store_false', dest='InteractiveShell.readline_use',
-        help="Disable readline for command line usage.")
-    ),
-    (('--screen-length','-sl'), dict(
-        type=int, dest='InteractiveShell.screen_length',
-        help=
-        """Number of lines of your screen, used to control printing of very
-        long strings.  Strings longer than this number of lines will be sent
-        through a pager instead of directly printed.  The default value for
-        this is 0, which means IPython will auto-detect your screen size every
-        time it needs to print certain potentially long strings (this doesn't
-        change the behavior of the 'print' keyword, it's only triggered
-        internally). If for some reason this isn't working well (it needs
-        curses support), specify it yourself. Otherwise don't change the
-        default.""",
-        metavar='InteractiveShell.screen_length')
-    ),
-    (('--separate-in','-si'), dict(
-        type=str, dest='InteractiveShell.separate_in',
-        help="Separator before input prompts.  Default '\\n'.",
-        metavar='InteractiveShell.separate_in')
-    ),
-    (('--separate-out','-so'), dict(
-        type=str, dest='InteractiveShell.separate_out',
-        help="Separator before output prompts.  Default 0 (nothing).",
-        metavar='InteractiveShell.separate_out')
-    ),
-    (('--separate-out2','-so2'), dict(
-        type=str, dest='InteractiveShell.separate_out2',
-        help="Separator after output prompts.  Default 0 (nonight).",
-        metavar='InteractiveShell.separate_out2')
-    ),
-    (('-no-sep',), dict(
-        action='store_true', dest='Global.nosep',
-        help="Eliminate all spacing between prompts.")
-    ),
-    (('--term-title',), dict(
-        action='store_true', dest='InteractiveShell.term_title',
-        help="Enable auto setting the terminal title.")
-    ),
-    (('--no-term-title',), dict(
-        action='store_false', dest='InteractiveShell.term_title',
-        help="Disable auto setting the terminal title.")
-    ),
-    (('--xmode',), dict(
-        type=str, dest='InteractiveShell.xmode',
-        help=
-        """Exception reporting mode ('Plain','Context','Verbose').  Plain:
-        similar to python's normal traceback printing.  Context: prints 5 lines
-        of context source code around each line in the traceback.  Verbose:
-        similar to Context, but additionally prints the variables currently
-        visible where the exception happened (shortening their strings if too
-        long).  This can potentially be very slow, if you happen to have a huge
-        data structure whose string representation is complex to compute.
-        Your computer may appear to freeze for a while with cpu usage at 100%%.
-        If this occurs, you can cancel the traceback with Ctrl-C (maybe hitting
-        it more than once).
-        """,
-        metavar='InteractiveShell.xmode')
-    ),
-    (('--ext',), dict(
-        type=str, dest='Global.extra_extension',
-        help="The dotted module name of an IPython extension to load.",
-        metavar='Global.extra_extension')
-    ),
-    (('-c',), dict(
-        type=str, dest='Global.code_to_run',
-        help="Execute the given command string.",
-        metavar='Global.code_to_run')
-    ),
-    (('-i',), dict(
-        action='store_true', dest='Global.force_interact',
-        help=
-        "If running code from the command line, become interactive afterwards."
-        )
-    ),
-
-    # Options to start with GUI control enabled from the beginning
-    (('--gui',), dict(
-        type=str, dest='Global.gui',
-        help="Enable GUI event loop integration ('qt', 'wx', 'gtk').",
-        metavar='gui-mode')
-    ),
-
-    (('--pylab','-pylab'), dict(
-        type=str, dest='Global.pylab',
-        nargs='?', const='auto', metavar='gui-mode',
-        help="Pre-load matplotlib and numpy for interactive use. "+
-        "If no value is given, the gui backend is matplotlib's, else use "+
-        "one of:  ['tk', 'qt', 'wx', 'gtk'].")
-    ),
-    
-    # Legacy GUI options.  Leave them in for backwards compatibility, but the
-    # 'thread' names are really a misnomer now.
-    (('--wthread','-wthread'), dict(
-        action='store_true', dest='Global.wthread',
-        help="Enable wxPython event loop integration "+
-             "(DEPRECATED, use --gui wx)")
-    ),
-    (('--q4thread','--qthread','-q4thread','-qthread'), dict(
-        action='store_true', dest='Global.q4thread',
-        help="Enable Qt4 event loop integration. Qt3 is no longer supported. "+
-             "(DEPRECATED, use --gui qt)")
-    ),
-    (('--gthread','-gthread'), dict(
-        action='store_true', dest='Global.gthread',
-        help="Enable GTK event loop integration. "+
-             "(DEPRECATED, use --gui gtk)")
-    ),
-)
+
+class IPAppConfigLoader(BaseAppConfigLoader):
+
+    def _add_arguments(self):
+        super(IPAppConfigLoader, self)._add_arguments()
+        paa = self.parser.add_argument
+        paa('-p', 
+            '--profile', dest='Global.profile', type=unicode,
+            help=
+            """The string name of the ipython profile to be used.  Assume that your
+            config file is ipython_config-<name>.py (looks in current dir first,
+            then in IPYTHON_DIR). This is a quick way to keep and load multiple
+            config files for different tasks, especially if include your basic one
+            in your more specialized ones.  You can keep a basic
+            IPYTHON_DIR/ipython_config.py file and then have other 'profiles' which
+            include this one and load extra things for particular tasks.""",
+            metavar='Global.profile')
+        paa('--config-file', 
+            dest='Global.config_file', type=unicode,
+            help=
+            """Set the config file name to override default.  Normally IPython
+            loads ipython_config.py (from current directory) or
+            IPYTHON_DIR/ipython_config.py.  If the loading of your config file
+            fails, IPython starts with a bare bones configuration (no modules
+            loaded at all).""",
+            metavar='Global.config_file')
+        paa('--autocall', 
+            dest='InteractiveShell.autocall', type=int, 
+            help=
+            """Make IPython automatically call any callable object even if you
+            didn't type explicit parentheses. For example, 'str 43' becomes
+            'str(43)' automatically.  The value can be '0' to disable the feature,
+            '1' for 'smart' autocall, where it is not applied if there are no more
+            arguments on the line, and '2' for 'full' autocall, where all callable
+            objects are automatically called (even if no arguments are present).
+            The default is '1'.""",
+            metavar='InteractiveShell.autocall')
+        paa('--autoindent',
+            action='store_true', dest='InteractiveShell.autoindent',
+            help='Turn on autoindenting.')
+        paa('--no-autoindent',
+            action='store_false', dest='InteractiveShell.autoindent',
+            help='Turn off autoindenting.')
+        paa('--automagic',
+            action='store_true', dest='InteractiveShell.automagic',
+            help=
+            """Turn on the auto calling of magic commands. Type %%magic at the
+            IPython  prompt  for  more information.""")
+        paa('--no-automagic',
+            action='store_false', dest='InteractiveShell.automagic',
+            help='Turn off the auto calling of magic commands.')
+        paa('--autoedit-syntax',
+            action='store_true', dest='InteractiveShell.autoedit_syntax',
+            help='Turn on auto editing of files with syntax errors.')
+        paa('--no-autoedit-syntax',
+            action='store_false', dest='InteractiveShell.autoedit_syntax',
+            help='Turn off auto editing of files with syntax errors.')
+        paa('--banner',
+            action='store_true', dest='Global.display_banner',
+            help='Display a banner upon starting IPython.')
+        paa('--no-banner',
+            action='store_false', dest='Global.display_banner',
+            help="Don't display a banner upon starting IPython.")
+        paa('--cache-size',
+            type=int, dest='InteractiveShell.cache_size',
+            help=
+            """Set the size of the output cache.  The default is 1000, you can
+            change it permanently in your config file.  Setting it to 0 completely
+            disables the caching system, and the minimum value accepted is 20 (if
+            you provide a value less than 20, it is reset to 0 and a warning is
+            issued).  This limit is defined because otherwise you'll spend more
+            time re-flushing a too small cache than working""",
+            metavar='InteractiveShell.cache_size')
+        paa('--classic',
+            action='store_true', dest='Global.classic',
+            help="Gives IPython a similar feel to the classic Python prompt.")
+        paa('--colors',
+            type=str, dest='InteractiveShell.colors',
+            help="Set the color scheme (NoColor, Linux, and LightBG).",
+            metavar='InteractiveShell.colors')
+        paa('--color-info',
+            action='store_true', dest='InteractiveShell.color_info',
+            help=
+            """IPython can display information about objects via a set of func-
+            tions, and optionally can use colors for this, syntax highlighting
+            source code and various other elements.  However, because this
+            information is passed through a pager (like 'less') and many pagers get
+            confused with color codes, this option is off by default.  You can test
+            it and turn it on permanently in your ipython_config.py file if it
+            works for you.  Test it and turn it on permanently if it works with
+            your system.  The magic function %%color_info allows you to toggle this
+            inter- actively for testing.""")
+        paa('--no-color-info',
+            action='store_false', dest='InteractiveShell.color_info',
+            help="Disable using colors for info related things.")
+        paa('--confirm-exit',
+            action='store_true', dest='InteractiveShell.confirm_exit',
+            help=
+            """Set to confirm when you try to exit IPython with an EOF (Control-D
+            in Unix, Control-Z/Enter in Windows). By typing 'exit', 'quit' or
+            '%%Exit', you can force a direct exit without any confirmation.""")
+        paa('--no-confirm-exit',
+            action='store_false', dest='InteractiveShell.confirm_exit',
+            help="Don't prompt the user when exiting.")
+        paa('--deep-reload',
+            action='store_true', dest='InteractiveShell.deep_reload',
+            help=
+            """Enable deep (recursive) reloading by default. IPython can use the
+            deep_reload module which reloads changes in modules recursively (it
+            replaces the reload() function, so you don't need to change anything to
+            use it). deep_reload() forces a full reload of modules whose code may
+            have changed, which the default reload() function does not.  When
+            deep_reload is off, IPython will use the normal reload(), but
+            deep_reload will still be available as dreload(). This fea- ture is off
+            by default [which means that you have both normal reload() and
+            dreload()].""")
+        paa('--no-deep-reload',
+            action='store_false', dest='InteractiveShell.deep_reload',
+            help="Disable deep (recursive) reloading by default.")
+        paa('--editor',
+            type=str, dest='InteractiveShell.editor',
+            help="Set the editor used by IPython (default to $EDITOR/vi/notepad).",
+            metavar='InteractiveShell.editor')
+        paa('--log','-l', 
+            action='store_true', dest='InteractiveShell.logstart',
+            help="Start logging to the default log file (./ipython_log.py).")
+        paa('--logfile','-lf',
+            type=unicode, dest='InteractiveShell.logfile',
+            help="Start logging to logfile with this name.",
+            metavar='InteractiveShell.logfile')
+        paa('--log-append','-la',
+            type=unicode, dest='InteractiveShell.logappend',
+            help="Start logging to the given file in append mode.",
+            metavar='InteractiveShell.logfile')
+        paa('--pdb',
+            action='store_true', dest='InteractiveShell.pdb',
+            help="Enable auto calling the pdb debugger after every exception.")
+        paa('--no-pdb',
+            action='store_false', dest='InteractiveShell.pdb',
+            help="Disable auto calling the pdb debugger after every exception.")
+        paa('--pprint',
+            action='store_true', dest='InteractiveShell.pprint',
+            help="Enable auto pretty printing of results.")
+        paa('--no-pprint',
+            action='store_false', dest='InteractiveShell.pprint',
+            help="Disable auto auto pretty printing of results.")
+        paa('--prompt-in1','-pi1',
+            type=str, dest='InteractiveShell.prompt_in1',
+            help=
+            """Set the main input prompt ('In [\#]: ').  Note that if you are using
+            numbered prompts, the number is represented with a '\#' in the string.
+            Don't forget to quote strings with spaces embedded in them. Most
+            bash-like escapes can be used to customize IPython's prompts, as well
+            as a few additional ones which are IPython-spe- cific.  All valid
+            prompt escapes are described in detail in the Customization section of
+            the IPython manual.""",
+            metavar='InteractiveShell.prompt_in1')
+        paa('--prompt-in2','-pi2',
+            type=str, dest='InteractiveShell.prompt_in2',
+            help=
+            """Set the secondary input prompt (' .\D.: ').  Similar to the previous
+            option, but used for the continuation prompts. The special sequence
+            '\D' is similar to '\#', but with all digits replaced by dots (so you
+            can have your continuation prompt aligned with your input prompt).
+            Default: ' .\D.: ' (note three spaces at the start for alignment with
+            'In [\#]')""",
+            metavar='InteractiveShell.prompt_in2')
+        paa('--prompt-out','-po',
+            type=str, dest='InteractiveShell.prompt_out',
+            help="Set the output prompt ('Out[\#]:')",
+            metavar='InteractiveShell.prompt_out')
+        paa('--quick',
+            action='store_true', dest='Global.quick',
+            help="Enable quick startup with no config files.")
+        paa('--readline',
+            action='store_true', dest='InteractiveShell.readline_use',
+            help="Enable readline for command line usage.")
+        paa('--no-readline',
+            action='store_false', dest='InteractiveShell.readline_use',
+            help="Disable readline for command line usage.")
+        paa('--screen-length','-sl',
+            type=int, dest='InteractiveShell.screen_length',
+            help=
+            """Number of lines of your screen, used to control printing of very
+            long strings.  Strings longer than this number of lines will be sent
+            through a pager instead of directly printed.  The default value for
+            this is 0, which means IPython will auto-detect your screen size every
+            time it needs to print certain potentially long strings (this doesn't
+            change the behavior of the 'print' keyword, it's only triggered
+            internally). If for some reason this isn't working well (it needs
+            curses support), specify it yourself. Otherwise don't change the
+            default.""",
+            metavar='InteractiveShell.screen_length')
+        paa('--separate-in','-si',
+            type=str, dest='InteractiveShell.separate_in',
+            help="Separator before input prompts.  Default '\\n'.",
+            metavar='InteractiveShell.separate_in')
+        paa('--separate-out','-so',
+            type=str, dest='InteractiveShell.separate_out',
+            help="Separator before output prompts.  Default 0 (nothing).",
+            metavar='InteractiveShell.separate_out')
+        paa('--separate-out2','-so2', 
+            type=str, dest='InteractiveShell.separate_out2',
+            help="Separator after output prompts.  Default 0 (nonight).",
+            metavar='InteractiveShell.separate_out2')
+        paa('-no-sep',
+            action='store_true', dest='Global.nosep',
+            help="Eliminate all spacing between prompts.")
+        paa('--term-title',
+            action='store_true', dest='InteractiveShell.term_title',
+            help="Enable auto setting the terminal title.")
+        paa('--no-term-title',
+            action='store_false', dest='InteractiveShell.term_title',
+            help="Disable auto setting the terminal title.")
+        paa('--xmode',
+            type=str, dest='InteractiveShell.xmode',
+            help=
+            """Exception reporting mode ('Plain','Context','Verbose').  Plain:
+            similar to python's normal traceback printing.  Context: prints 5 lines
+            of context source code around each line in the traceback.  Verbose:
+            similar to Context, but additionally prints the variables currently
+            visible where the exception happened (shortening their strings if too
+            long).  This can potentially be very slow, if you happen to have a huge
+            data structure whose string representation is complex to compute.
+            Your computer may appear to freeze for a while with cpu usage at 100%%.
+            If this occurs, you can cancel the traceback with Ctrl-C (maybe hitting
+            it more than once).
+            """,
+            metavar='InteractiveShell.xmode')
+        paa('--ext',
+            type=str, dest='Global.extra_extension',
+            help="The dotted module name of an IPython extension to load.",
+            metavar='Global.extra_extension')
+        paa('-c',
+            type=str, dest='Global.code_to_run',
+            help="Execute the given command string.",
+            metavar='Global.code_to_run')
+        paa('-i',
+            action='store_true', dest='Global.force_interact',
+            help=
+            "If running code from the command line, become interactive afterwards.")
+
+        # Options to start with GUI control enabled from the beginning
+        paa('--gui',
+            type=str, dest='Global.gui',
+            help="Enable GUI event loop integration ('qt', 'wx', 'gtk').",
+            metavar='gui-mode')
+        paa('--pylab','-pylab',
+            type=str, dest='Global.pylab',
+            nargs='?', const='auto', metavar='gui-mode',
+            help="Pre-load matplotlib and numpy for interactive use. "+
+            "If no value is given, the gui backend is matplotlib's, else use "+
+            "one of:  ['tk', 'qt', 'wx', 'gtk'].")
+
+        # Legacy GUI options.  Leave them in for backwards compatibility, but the
+        # 'thread' names are really a misnomer now.
+        paa('--wthread', '-wthread',
+            action='store_true', dest='Global.wthread',
+            help=
+            """Enable wxPython event loop integration. (DEPRECATED, use --gui wx)""")
+        paa('--q4thread', '--qthread', '-q4thread', '-qthread',
+            action='store_true', dest='Global.q4thread',
+            help=
+            """Enable Qt4 event loop integration. Qt3 is no longer supported.
+            (DEPRECATED, use --gui qt)""")
+        paa('--gthread', '-gthread',
+            action='store_true', dest='Global.gthread',
+            help=
+            """Enable GTK event loop integration. (DEPRECATED, use --gui gtk)""")
+
 
 #-----------------------------------------------------------------------------
 # Main classes and functions
 #-----------------------------------------------------------------------------
 
 class IPythonApp(Application):
     name = u'ipython'
     #: argparse formats better the 'usage' than the 'description' field
     description = None
-    #: usage message printed by argparse. If None, auto-generate
     usage = usage.cl_usage
-
+    command_line_loader = IPAppConfigLoader
     config_file_name = default_config_file_name
 
-    cl_arguments = Application.cl_arguments + cl_args
-
     # Private and configuration attributes
     _CrashHandler = crashhandler.IPythonCrashHandler
-    
-    def __init__(self, argv=None,
-                 constructor_config=None, override_config=None,
-                 **shell_params):
-        """Create a new IPythonApp.
-
-        See the parent class for details on how configuration is handled.
-
-        Parameters
-        ----------
-        argv : optional, list
-          If given, used as the command-line argv environment to read arguments
-          from.
-
-        constructor_config : optional, Config
-          If given, additional config that is merged last, after internal
-          defaults, command-line and file-based configs.
-
-        override_config : optional, Config
-          If given, config that overrides all others unconditionally (except
-          for internal defaults, which ensure that all parameters exist).
-
-        shell_params : optional, dict
-          All other keywords are passed to the :class:`iplib.InteractiveShell`
-          constructor. 
-        """
-        super(IPythonApp, self).__init__(argv, constructor_config,
-                                         override_config)
-        self.shell_params = shell_params
 
     def create_default_config(self):
         super(IPythonApp, self).create_default_config()
         # Eliminate multiple lookups
         Global = self.default_config.Global
 
         # Set all default values
         Global.display_banner = True
         
         # If the -c flag is given or a file is given to run at the cmd line
         # like "ipython foo.py", normally we exit without starting the main
         # loop.  The force_interact config variable allows a user to override
         # this and interact.  It is also set by the -i cmd line flag, just
         # like Python.
         Global.force_interact = False
 
         # By default always interact by starting the IPython mainloop.
         Global.interact = True
 
         # No GUI integration by default
         Global.gui = False
         # Pylab off by default
         Global.pylab = False
 
         # Deprecated versions of gui support that used threading, we support
         # them just for bacwards compatibility as an alternate spelling for
         # '--gui X'
         Global.qthread = False
         Global.q4thread = False
         Global.wthread = False
         Global.gthread = False
 
     def load_file_config(self):
         if hasattr(self.command_line_config.Global, 'quick'):
             if self.command_line_config.Global.quick:
                 self.file_config = Config()
                 return
         super(IPythonApp, self).load_file_config()
 
     def post_load_file_config(self):
         if hasattr(self.command_line_config.Global, 'extra_extension'):
             if not hasattr(self.file_config.Global, 'extensions'):
                 self.file_config.Global.extensions = []
             self.file_config.Global.extensions.append(
                 self.command_line_config.Global.extra_extension)
             del self.command_line_config.Global.extra_extension
 
     def pre_construct(self):
         config = self.master_config
 
         if hasattr(config.Global, 'classic'):
             if config.Global.classic:
                 config.InteractiveShell.cache_size = 0
                 config.InteractiveShell.pprint = 0
                 config.InteractiveShell.prompt_in1 = '>>> '
                 config.InteractiveShell.prompt_in2 = '... '
                 config.InteractiveShell.prompt_out = ''
                 config.InteractiveShell.separate_in = \
                     config.InteractiveShell.separate_out = \
                     config.InteractiveShell.separate_out2 = ''
                 config.InteractiveShell.colors = 'NoColor'
                 config.InteractiveShell.xmode = 'Plain'
 
         if hasattr(config.Global, 'nosep'):
             if config.Global.nosep:
                 config.InteractiveShell.separate_in = \
                 config.InteractiveShell.separate_out = \
                 config.InteractiveShell.separate_out2 = ''
 
         # if there is code of files to run from the cmd line, don't interact
         # unless the -i flag (Global.force_interact) is true.
         code_to_run = config.Global.get('code_to_run','')
         file_to_run = False
         if self.extra_args and self.extra_args[0]:
                 file_to_run = True
         if file_to_run or code_to_run:
             if not config.Global.force_interact:
                 config.Global.interact = False
 
     def construct(self):
         # I am a little hesitant to put these into InteractiveShell itself.
         # But that might be the place for them
         sys.path.insert(0, '')
 
         # Create an InteractiveShell instance
-        self.shell = InteractiveShell(None, self.master_config,
-                                      **self.shell_params )
+        self.shell = InteractiveShell(None, self.master_config)
 
     def post_construct(self):
         """Do actions after construct, but before starting the app."""
         config = self.master_config
         
         # shell.display_banner should always be False for the terminal 
         # based app, because we call shell.show_banner() by hand below
         # so the banner shows *before* all extension loading stuff.
         self.shell.display_banner = False
 
         if config.Global.display_banner and \
             config.Global.interact:
             self.shell.show_banner()
 
         # Make sure there is a space below the banner.
         if self.log_level <= logging.INFO: print
 
         # Now a variety of things that happen after the banner is printed.
         self._enable_gui_pylab()
         self._load_extensions()
         self._run_exec_lines()
         self._run_exec_files()
         self._run_cmd_line_code()
 
     def _enable_gui_pylab(self):
         """Enable GUI event loop integration, taking pylab into account."""
         Global = self.master_config.Global
 
         # Select which gui to use
         if Global.gui:
             gui = Global.gui
         # The following are deprecated, but there's likely to be a lot of use
         # of this form out there, so we might as well support it for now.  But
         # the --gui option above takes precedence.
         elif Global.wthread:
             gui = inputhook.GUI_WX
         elif Global.qthread:
             gui = inputhook.GUI_QT
         elif Global.gthread:
             gui = inputhook.GUI_GTK
         else:
             gui = None
 
         # Using --pylab will also require gui activation, though which toolkit
         # to use may be chosen automatically based on mpl configuration.
         if Global.pylab:
             activate = self.shell.enable_pylab
             if Global.pylab == 'auto':
                 gui = None
             else:
                 gui = Global.pylab
         else:
             # Enable only GUI integration, no pylab
             activate = inputhook.enable_gui
 
         if gui or Global.pylab:
             try:
                 self.log.info("Enabling GUI event loop integration, "
                               "toolkit=%s, pylab=%s" % (gui, Global.pylab) )
                 activate(gui)
             except:
                 self.log.warn("Error in enabling GUI event loop integration:")
                 self.shell.showtraceback()
 
     def _load_extensions(self):
         """Load all IPython extensions in Global.extensions.
 
         This uses the :meth:`InteractiveShell.load_extensions` to load all
         the extensions listed in ``self.master_config.Global.extensions``.
         """
         try:
             if hasattr(self.master_config.Global, 'extensions'):
                 self.log.debug("Loading IPython extensions...")
                 extensions = self.master_config.Global.extensions
                 for ext in extensions:
                     try:
                         self.log.info("Loading IPython extension: %s" % ext)                        
                         self.shell.load_extension(ext)
                     except:
                         self.log.warn("Error in loading extension: %s" % ext)
                         self.shell.showtraceback()
         except:
             self.log.warn("Unknown error in loading extensions:")
             self.shell.showtraceback()
 
     def _run_exec_lines(self):
         """Run lines of code in Global.exec_lines in the user's namespace."""
         try:
             if hasattr(self.master_config.Global, 'exec_lines'):
                 self.log.debug("Running code from Global.exec_lines...")
                 exec_lines = self.master_config.Global.exec_lines
                 for line in exec_lines:
                     try:
                         self.log.info("Running code in user namespace: %s" % line)
                         self.shell.runlines(line)
                     except:
                         self.log.warn("Error in executing line in user namespace: %s" % line)
                         self.shell.showtraceback()
         except:
             self.log.warn("Unknown error in handling Global.exec_lines:")
             self.shell.showtraceback()
 
     def _exec_file(self, fname):
         full_filename = filefind(fname, [u'.', self.ipython_dir])
         if os.path.isfile(full_filename):
             if full_filename.endswith(u'.py'):
                 self.log.info("Running file in user namespace: %s" % full_filename)
                 self.shell.safe_execfile(full_filename, self.shell.user_ns)
             elif full_filename.endswith('.ipy'):
                 self.log.info("Running file in user namespace: %s" % full_filename)
                 self.shell.safe_execfile_ipy(full_filename)
             else:
                 self.log.warn("File does not have a .py or .ipy extension: <%s>" % full_filename)
 
     def _run_exec_files(self):
         try:
             if hasattr(self.master_config.Global, 'exec_files'):
                 self.log.debug("Running files in Global.exec_files...")
                 exec_files = self.master_config.Global.exec_files
                 for fname in exec_files:
                     self._exec_file(fname)
         except:
             self.log.warn("Unknown error in handling Global.exec_files:")
             self.shell.showtraceback()
 
     def _run_cmd_line_code(self):
         if hasattr(self.master_config.Global, 'code_to_run'):
             line = self.master_config.Global.code_to_run
             try:
                 self.log.info("Running code given at command line (-c): %s" % line)
                 self.shell.runlines(line)
             except:
                 self.log.warn("Error in executing line in user namespace: %s" % line)
                 self.shell.showtraceback()
             return
         # Like Python itself, ignore the second if the first of these is present
         try:
             fname = self.extra_args[0]
         except:
             pass
         else:
             try:
                 self._exec_file(fname)
             except:
                 self.log.warn("Error in executing file in user namespace: %s" % fname)
                 self.shell.showtraceback()
 
     def start_app(self):
         if self.master_config.Global.interact:
             self.log.debug("Starting IPython's mainloop...")
             self.shell.mainloop()
         else:
             self.log.debug("IPython not interactive, start_app is no-op...")
 
 
 def load_default_config(ipython_dir=None):
     """Load the default config file from the default ipython_dir.
 
     This is useful for embedded shells.
     """
     if ipython_dir is None:
         ipython_dir = get_ipython_dir()
     cl = PyFileConfigLoader(default_config_file_name, ipython_dir)
     config = cl.load_config()
     return config
 
 
 def launch_new_instance():
     """Create and run a full blown IPython instance"""
     app = IPythonApp()
     app.start()
+

IPython/kernel/clusterdir.py

@@ -1,464 +1,493 b''
 #!/usr/bin/env python
 # encoding: utf-8
 """
 The IPython cluster directory
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008-2009  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 from __future__ import with_statement
 
 import os
 import shutil
 import sys
 import warnings
 
 from twisted.python import log
 
 from IPython.config.loader import PyFileConfigLoader
-from IPython.core.application import Application
+from IPython.core.application import Application, BaseAppConfigLoader
 from IPython.core.component import Component
 from IPython.utils.path import (
     get_ipython_package_dir,
     expand_path
 )
 from IPython.utils.traitlets import Unicode
 
 #-----------------------------------------------------------------------------
 # Warnings control
 #-----------------------------------------------------------------------------
 # Twisted generates annoying warnings with Python 2.6, as will do other code
 # that imports 'sets' as of today
 warnings.filterwarnings('ignore', 'the sets module is deprecated',
                         DeprecationWarning )
 
 # This one also comes from Twisted
 warnings.filterwarnings('ignore', 'the sha module is deprecated',
                         DeprecationWarning)
 
 #-----------------------------------------------------------------------------
-# Classes and functions
+# Module errors
 #-----------------------------------------------------------------------------
 
 class ClusterDirError(Exception):
     pass
 
 
 class PIDFileError(Exception):
     pass
 
 
+#-----------------------------------------------------------------------------
+# Class for managing cluster directories
+#-----------------------------------------------------------------------------
+
 class ClusterDir(Component):
     """An object to manage the cluster directory and its resources.
 
     The cluster directory is used by :command:`ipcontroller`, 
     :command:`ipcontroller` and :command:`ipcontroller` to manage the
     configuration, logging and security of these applications.
 
     This object knows how to find, create and manage these directories. This
     should be used by any code that want's to handle cluster directories.
     """
 
     security_dir_name = Unicode('security')
     log_dir_name = Unicode('log')
     pid_dir_name = Unicode('pid')
     security_dir = Unicode(u'')
     log_dir = Unicode(u'')
     pid_dir = Unicode(u'')
     location = Unicode(u'')
 
     def __init__(self, location):
         super(ClusterDir, self).__init__(None)
         self.location = location
 
     def _location_changed(self, name, old, new):
         if not os.path.isdir(new):
             os.makedirs(new)
         self.security_dir = os.path.join(new, self.security_dir_name)
         self.log_dir = os.path.join(new, self.log_dir_name)
         self.pid_dir = os.path.join(new, self.pid_dir_name)
         self.check_dirs()
 
     def _log_dir_changed(self, name, old, new):
         self.check_log_dir()
 
     def check_log_dir(self):
         if not os.path.isdir(self.log_dir):
             os.mkdir(self.log_dir)
 
     def _security_dir_changed(self, name, old, new):
         self.check_security_dir()
 
     def check_security_dir(self):
         if not os.path.isdir(self.security_dir):
             os.mkdir(self.security_dir, 0700)
         os.chmod(self.security_dir, 0700)
 
     def _pid_dir_changed(self, name, old, new):
         self.check_pid_dir()
 
     def check_pid_dir(self):
         if not os.path.isdir(self.pid_dir):
             os.mkdir(self.pid_dir, 0700)
         os.chmod(self.pid_dir, 0700)
 
     def check_dirs(self):
         self.check_security_dir()
         self.check_log_dir()
         self.check_pid_dir()
 
     def load_config_file(self, filename):
         """Load a config file from the top level of the cluster dir.
 
         Parameters
         ----------
         filename : unicode or str
             The filename only of the config file that must be located in
             the top-level of the cluster directory.
         """
         loader = PyFileConfigLoader(filename, self.location)
         return loader.load_config()
 
     def copy_config_file(self, config_file, path=None, overwrite=False):
         """Copy a default config file into the active cluster directory.
 
         Default configuration files are kept in :mod:`IPython.config.default`.
         This function moves these from that location to the working cluster
         directory.
         """
         if path is None:
             import IPython.config.default
             path = IPython.config.default.__file__.split(os.path.sep)[:-1]
             path = os.path.sep.join(path)
         src = os.path.join(path, config_file)
         dst = os.path.join(self.location, config_file)
         if not os.path.isfile(dst) or overwrite:
             shutil.copy(src, dst)
 
     def copy_all_config_files(self, path=None, overwrite=False):
         """Copy all config files into the active cluster directory."""
         for f in [u'ipcontroller_config.py', u'ipengine_config.py',
                   u'ipcluster_config.py']:
             self.copy_config_file(f, path=path, overwrite=overwrite)
 
     @classmethod
     def create_cluster_dir(csl, cluster_dir):
         """Create a new cluster directory given a full path.
 
         Parameters
         ----------
         cluster_dir : str
             The full path to the cluster directory.  If it does exist, it will
             be used.  If not, it will be created.
         """
         return ClusterDir(cluster_dir)
 
     @classmethod
     def create_cluster_dir_by_profile(cls, path, profile=u'default'):
         """Create a cluster dir by profile name and path.
 
         Parameters
         ----------
         path : str
             The path (directory) to put the cluster directory in.
         profile : str
             The name of the profile.  The name of the cluster directory will
             be "cluster_<profile>".
         """
         if not os.path.isdir(path):
             raise ClusterDirError('Directory not found: %s' % path)
         cluster_dir = os.path.join(path, u'cluster_' + profile)
         return ClusterDir(cluster_dir)
 
     @classmethod
     def find_cluster_dir_by_profile(cls, ipython_dir, profile=u'default'):
         """Find an existing cluster dir by profile name, return its ClusterDir.
 
         This searches through a sequence of paths for a cluster dir.  If it
         is not found, a :class:`ClusterDirError` exception will be raised.
 
         The search path algorithm is:
         1. ``os.getcwd()``
         2. ``ipython_dir``
         3. The directories found in the ":" separated 
            :env:`IPCLUSTER_DIR_PATH` environment variable.
 
         Parameters
         ----------
         ipython_dir : unicode or str
             The IPython directory to use.
         profile : unicode or str
             The name of the profile.  The name of the cluster directory
             will be "cluster_<profile>".
         """
         dirname = u'cluster_' + profile
         cluster_dir_paths = os.environ.get('IPCLUSTER_DIR_PATH','')
         if cluster_dir_paths:
             cluster_dir_paths = cluster_dir_paths.split(':')
         else:
             cluster_dir_paths = []
         paths = [os.getcwd(), ipython_dir] + cluster_dir_paths
         for p in paths:
             cluster_dir = os.path.join(p, dirname)
             if os.path.isdir(cluster_dir):
                 return ClusterDir(cluster_dir)
         else:
             raise ClusterDirError('Cluster directory not found in paths: %s' % dirname)
 
     @classmethod
     def find_cluster_dir(cls, cluster_dir):
         """Find/create a cluster dir and return its ClusterDir.
 
         This will create the cluster directory if it doesn't exist.
 
         Parameters
         ----------
         cluster_dir : unicode or str
             The path of the cluster directory.  This is expanded using
             :func:`IPython.utils.genutils.expand_path`.
         """
         cluster_dir = expand_path(cluster_dir)
         if not os.path.isdir(cluster_dir):
             raise ClusterDirError('Cluster directory not found: %s' % cluster_dir)
         return ClusterDir(cluster_dir)
 
 
-# Default command line options for IPython cluster applications.
-cl_args = (
-    (('--ipython-dir',), dict(
-        dest='Global.ipython_dir',type=unicode,
-        help='Set to override default location of Global.ipython_dir.',
-        metavar='Global.ipython_dir') ),
-    (('-p', '--profile',), dict(
-        dest='Global.profile',type=unicode,
-        help=
-        """The string name of the profile to be used. This determines the name
-        of the cluster dir as: cluster_<profile>. The default profile is named
-        'default'.  The cluster directory is resolve this way if the
-        --cluster-dir option is not used.""",
-        metavar='Global.profile') ),
-    (('--cluster-dir',), dict(
-        dest='Global.cluster_dir',type=unicode,
-        help="""Set the cluster dir. This overrides the logic used by the
-        --profile option.""",
-        metavar='Global.cluster_dir') ),
-    (('--work-dir',), dict(
-      dest='Global.work_dir',type=unicode,
-      help='Set the working dir for the process.',
-      metavar='Global.work_dir') ),
-    (('--clean-logs',), dict(
-      dest='Global.clean_logs', action='store_true',
-      help='Delete old log flies before starting.') ),
-    (('--no-clean-logs',), dict(
-      dest='Global.clean_logs', action='store_false',
-      help="Don't Delete old log flies before starting.") ),
-    )
+#-----------------------------------------------------------------------------
+# Command line options
+#-----------------------------------------------------------------------------
+
+class ClusterDirConfigLoader(BaseAppConfigLoader):
+
+    def _add_cluster_profile(self, parser):
+        paa = parser.add_argument
+        paa('-p', '--profile',
+            dest='Global.profile',type=unicode,
+            help=
+            """The string name of the profile to be used. This determines the name
+            of the cluster dir as: cluster_<profile>. The default profile is named
+            'default'.  The cluster directory is resolve this way if the
+            --cluster-dir option is not used.""",
+            metavar='Global.profile')
+
+    def _add_cluster_dir(self, parser):
+        paa = parser.add_argument
+        paa('--cluster-dir',
+            dest='Global.cluster_dir',type=unicode,
+            help="""Set the cluster dir. This overrides the logic used by the
+            --profile option.""",
+            metavar='Global.cluster_dir')
+
+    def _add_work_dir(self, parser):
+        paa = parser.add_argument
+        paa('--work-dir',
+            dest='Global.work_dir',type=unicode,
+            help='Set the working dir for the process.',
+            metavar='Global.work_dir')
+
+    def _add_clean_logs(self, parser):
+        paa = parser.add_argument
+        paa('--clean-logs',
+            dest='Global.clean_logs', action='store_true',
+            help='Delete old log flies before starting.')
+
+    def _add_no_clean_logs(self, parser):
+        paa = parser.add_argument
+        paa('--no-clean-logs',
+            dest='Global.clean_logs', action='store_false',
+            help="Don't Delete old log flies before starting.")
+
+    def _add_arguments(self):
+        super(ClusterDirConfigLoader, self)._add_arguments()
+        self._add_cluster_profile(self.parser)
+        self._add_cluster_dir(self.parser)
+        self._add_work_dir(self.parser)
+        self._add_clean_logs(self.parser)
+        self._add_no_clean_logs(self.parser)
 
 
+#-----------------------------------------------------------------------------
+# Main application
+#-----------------------------------------------------------------------------
+
 class ApplicationWithClusterDir(Application):
     """An application that puts everything into a cluster directory.
 
     Instead of looking for things in the ipython_dir, this type of application
     will use its own private directory called the "cluster directory"
     for things like config files, log files, etc.
 
     The cluster directory is resolved as follows:
 
     * If the ``--cluster-dir`` option is given, it is used.
     * If ``--cluster-dir`` is not given, the application directory is 
       resolve using the profile name as ``cluster_<profile>``. The search 
       path for this directory is then i) cwd if it is found there
       and ii) in ipython_dir otherwise.
 
     The config file for the application is to be put in the cluster
     dir and named the value of the ``config_file_name`` class attribute.
     """
 
+    command_line_loader = ClusterDirConfigLoader
     auto_create_cluster_dir = True
 
-    cl_arguments = Application.cl_arguments + cl_args
-
     def create_default_config(self):
         super(ApplicationWithClusterDir, self).create_default_config()
         self.default_config.Global.profile = u'default'
         self.default_config.Global.cluster_dir = u''
         self.default_config.Global.work_dir = os.getcwd()
         self.default_config.Global.log_to_file = False
         self.default_config.Global.clean_logs = False
 
     def find_resources(self):
         """This resolves the cluster directory.
 
         This tries to find the cluster directory and if successful, it will
         have done:
         * Sets ``self.cluster_dir_obj`` to the :class:`ClusterDir` object for 
           the application.
         * Sets ``self.cluster_dir`` attribute of the application and config
           objects.
 
         The algorithm used for this is as follows:
         1. Try ``Global.cluster_dir``.
         2. Try using ``Global.profile``.
         3. If both of these fail and ``self.auto_create_cluster_dir`` is
            ``True``, then create the new cluster dir in the IPython directory.
         4. If all fails, then raise :class:`ClusterDirError`.
         """
 
         try:
             cluster_dir = self.command_line_config.Global.cluster_dir
         except AttributeError:
             cluster_dir = self.default_config.Global.cluster_dir
         cluster_dir = expand_path(cluster_dir)
         try:
             self.cluster_dir_obj = ClusterDir.find_cluster_dir(cluster_dir)
         except ClusterDirError:
             pass
         else:
             self.log.info('Using existing cluster dir: %s' % \
                 self.cluster_dir_obj.location
             )
             self.finish_cluster_dir()
             return
 
         try:
             self.profile = self.command_line_config.Global.profile
         except AttributeError:
             self.profile = self.default_config.Global.profile
         try:
             self.cluster_dir_obj = ClusterDir.find_cluster_dir_by_profile(
                 self.ipython_dir, self.profile)
         except ClusterDirError:
             pass
         else:
             self.log.info('Using existing cluster dir: %s' % \
                 self.cluster_dir_obj.location
             )
             self.finish_cluster_dir()
             return
 
         if self.auto_create_cluster_dir:
             self.cluster_dir_obj = ClusterDir.create_cluster_dir_by_profile(
                 self.ipython_dir, self.profile
             )
             self.log.info('Creating new cluster dir: %s' % \
                 self.cluster_dir_obj.location
             )
             self.finish_cluster_dir()
         else:
             raise ClusterDirError('Could not find a valid cluster directory.')
 
     def finish_cluster_dir(self):
         # Set the cluster directory
         self.cluster_dir = self.cluster_dir_obj.location
         
         # These have to be set because they could be different from the one
         # that we just computed.  Because command line has the highest
         # priority, this will always end up in the master_config.
         self.default_config.Global.cluster_dir = self.cluster_dir
         self.command_line_config.Global.cluster_dir = self.cluster_dir
 
     def find_config_file_name(self):
         """Find the config file name for this application."""
         # For this type of Application it should be set as a class attribute.
         if not hasattr(self, 'config_file_name'):
             self.log.critical("No config filename found")
 
     def find_config_file_paths(self):
         # Include our own config directory last, so that users can still find
         # our shipped copies of builtin config files even if they don't have
         # them in their ipython cluster directory.
         conf_dir = os.path.join(get_ipython_package_dir(), 'config', 'default')
         self.config_file_paths = (self.cluster_dir, conf_dir)
 
     def pre_construct(self):
         # The log and security dirs were set earlier, but here we put them
         # into the config and log them.
         config = self.master_config
         sdir = self.cluster_dir_obj.security_dir
         self.security_dir = config.Global.security_dir = sdir
         ldir = self.cluster_dir_obj.log_dir
         self.log_dir = config.Global.log_dir = ldir
         pdir = self.cluster_dir_obj.pid_dir
         self.pid_dir = config.Global.pid_dir = pdir
         self.log.info("Cluster directory set to: %s" % self.cluster_dir)
         config.Global.work_dir = unicode(expand_path(config.Global.work_dir))
         # Change to the working directory. We do this just before construct
         # is called so all the components there have the right working dir.
         self.to_work_dir()
 
     def to_work_dir(self):
         wd = self.master_config.Global.work_dir
         if unicode(wd) != unicode(os.getcwd()):
             os.chdir(wd)
             self.log.info("Changing to working dir: %s" % wd)
 
     def start_logging(self):
         # Remove old log files
         if self.master_config.Global.clean_logs:
             log_dir = self.master_config.Global.log_dir
             for f in os.listdir(log_dir):
                 if f.startswith(self.name + u'-') and f.endswith('.log'):
                     os.remove(os.path.join(log_dir, f))
         # Start logging to the new log file
         if self.master_config.Global.log_to_file:
             log_filename = self.name + u'-' + str(os.getpid()) + u'.log'
             logfile = os.path.join(self.log_dir, log_filename)
             open_log_file = open(logfile, 'w')
         else:
             open_log_file = sys.stdout
         log.startLogging(open_log_file)
 
     def write_pid_file(self, overwrite=False):
         """Create a .pid file in the pid_dir with my pid.
 
         This must be called after pre_construct, which sets `self.pid_dir`.
         This raises :exc:`PIDFileError` if the pid file exists already.
         """
         pid_file = os.path.join(self.pid_dir, self.name + u'.pid')
         if os.path.isfile(pid_file):
             pid = self.get_pid_from_file()
             if not overwrite:
                 raise PIDFileError(
                     'The pid file [%s] already exists. \nThis could mean that this '
                     'server is already running with [pid=%s].' % (pid_file, pid)
                 )
         with open(pid_file, 'w') as f:
             self.log.info("Creating pid file: %s" % pid_file)
             f.write(repr(os.getpid())+'\n')
 
     def remove_pid_file(self):
         """Remove the pid file.
 
         This should be called at shutdown by registering a callback with
         :func:`reactor.addSystemEventTrigger`. This needs to return
         ``None``.
         """
         pid_file = os.path.join(self.pid_dir, self.name + u'.pid')
         if os.path.isfile(pid_file):
             try:
                 self.log.info("Removing pid file: %s" % pid_file)
                 os.remove(pid_file)
             except:
                 self.log.warn("Error removing the pid file: %s" % pid_file)
 
     def get_pid_from_file(self):
         """Get the pid from the pid file.
 
         If the  pid file doesn't exist a :exc:`PIDFileError` is raised.
         """
         pid_file = os.path.join(self.pid_dir, self.name + u'.pid')
         if os.path.isfile(pid_file):
             with open(pid_file, 'r') as f:
                 pid = int(f.read().strip())
                 return pid
         else:
             raise PIDFileError('pid file not found: %s' % pid_file)
+

IPython/kernel/ipclusterapp.py

@@ -1,462 +1,459 b''
 #!/usr/bin/env python
 # encoding: utf-8
 """
 The ipcluster application.
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008-2009  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import logging
 import os
 import signal
 
 if os.name=='posix':
     from twisted.scripts._twistd_unix import daemonize
 
-from IPython.core import release
+from twisted.internet import reactor, defer
+from twisted.python import log, failure
+
+
 from IPython.external.argparse import ArgumentParser, SUPPRESS
-from IPython.config.loader import ArgParseConfigLoader
 from IPython.utils.importstring import import_item
-
 from IPython.kernel.clusterdir import (
-    ApplicationWithClusterDir, ClusterDirError, PIDFileError
+    ApplicationWithClusterDir, ClusterDirConfigLoader,
+    ClusterDirError, PIDFileError
 )
 
-from twisted.internet import reactor, defer
-from twisted.python import log, failure
-
 
 #-----------------------------------------------------------------------------
-# The ipcluster application
+# Module level variables
 #-----------------------------------------------------------------------------
 
 
+default_config_file_name = u'ipcluster_config.py'
+
+
+_description = """\
+Start an IPython cluster for parallel computing.\n\n
+
+An IPython cluster consists of 1 controller and 1 or more engines.
+This command automates the startup of these processes using a wide
+range of startup methods (SSH, local processes, PBS, mpiexec,
+Windows HPC Server 2008). To start a cluster with 4 engines on your
+local host simply do "ipcluster start -n 4". For more complex usage 
+you will typically do "ipcluster create -p mycluster", then edit
+configuration files, followed by "ipcluster start -p mycluster -n 4".
+"""
+
+
 # Exit codes for ipcluster
 
 # This will be the exit code if the ipcluster appears to be running because
 # a .pid file exists
 ALREADY_STARTED = 10
 
+
 # This will be the exit code if ipcluster stop is run, but there is not .pid
 # file to be found.
 ALREADY_STOPPED = 11
 
 
-class IPClusterCLLoader(ArgParseConfigLoader):
+#-----------------------------------------------------------------------------
+# Command line options
+#-----------------------------------------------------------------------------
+
 
-    def _add_other_arguments(self):
+class IPClusterAppConfigLoader(ClusterDirConfigLoader):
+
+    def _add_arguments(self):
+        # Don't call ClusterDirConfigLoader._add_arguments as we don't want
+        # its defaults on self.parser. Instead, we will put those on
+        # default options on our subparsers.
+        
         # This has all the common options that all subcommands use
-        parent_parser1 = ArgumentParser(add_help=False,
-                                        argument_default=SUPPRESS)
-        parent_parser1.add_argument('--ipython-dir', 
-            dest='Global.ipython_dir',type=unicode,
-            help='Set to override default location of Global.ipython_dir.',
-            metavar='Global.ipython_dir')
-        parent_parser1.add_argument('--log-level',
-            dest="Global.log_level",type=int,
-            help='Set the log level (0,10,20,30,40,50).  Default is 30.',
-            metavar='Global.log_level')
+        parent_parser1 = ArgumentParser(
+            add_help=False,
+            argument_default=SUPPRESS
+        )
+        self._add_ipython_dir(parent_parser1)
+        self._add_log_level(parent_parser1)
 
         # This has all the common options that other subcommands use
-        parent_parser2 = ArgumentParser(add_help=False,
-                                        argument_default=SUPPRESS)
-        parent_parser2.add_argument('-p','--profile',
-            dest='Global.profile',type=unicode,
-            help='The string name of the profile to be used. This determines '
-            'the name of the cluster dir as: cluster_<profile>. The default profile '
-            'is named "default".  The cluster directory is resolve this way '
-            'if the --cluster-dir option is not used.',
-            metavar='Global.profile')
-        parent_parser2.add_argument('--cluster-dir',
-            dest='Global.cluster_dir',type=unicode,
-            help='Set the cluster dir. This overrides the logic used by the '
-            '--profile option.',
-            metavar='Global.cluster_dir'),
-        parent_parser2.add_argument('--work-dir',
-            dest='Global.work_dir',type=unicode,
-            help='Set the working dir for the process.',
-            metavar='Global.work_dir')
-        parent_parser2.add_argument('--log-to-file',
-            action='store_true', dest='Global.log_to_file', 
-            help='Log to a file in the log directory (default is stdout)'
+        parent_parser2 = ArgumentParser(
+            add_help=False,
+            argument_default=SUPPRESS
         )
+        self._add_cluster_profile(parent_parser2)
+        self._add_cluster_dir(parent_parser2)
+        self._add_work_dir(parent_parser2)
+        paa = parent_parser2.add_argument
+        paa('--log-to-file',
+            action='store_true', dest='Global.log_to_file', 
+            help='Log to a file in the log directory (default is stdout)')
 
+        # Create the object used to create the subparsers.
         subparsers = self.parser.add_subparsers(
             dest='Global.subcommand',
             title='ipcluster subcommands',
-            description='ipcluster has a variety of subcommands. '
-            'The general way of running ipcluster is "ipcluster <cmd> '
-            ' [options]""',
-            help='For more help, type "ipcluster <cmd> -h"')
+            description=
+            """ipcluster has a variety of subcommands. The general way of 
+            running ipcluster is 'ipcluster <cmd> [options]'""",
+            help="For more help, type 'ipcluster <cmd> -h'"
+        )
 
+        # The "list" subcommand parser
         parser_list = subparsers.add_parser(
             'list',
             help='List all clusters in cwd and ipython_dir.',
             parents=[parent_parser1]
         )
 
+        # The "create" subcommand parser
         parser_create = subparsers.add_parser(
             'create',
             help='Create a new cluster directory.',
             parents=[parent_parser1, parent_parser2] 
         )
-        parser_create.add_argument(
-            '--reset-config',
+        paa = parser_create.add_argument
+        paa('--reset-config',
             dest='Global.reset_config', action='store_true',
-            help='Recopy the default config files to the cluster directory. '
-            'You will loose any modifications you have made to these files.'
-        )
+            help=
+            """Recopy the default config files to the cluster directory.
+            You will loose any modifications you have made to these files.""")
 
+        # The "start" subcommand parser
         parser_start = subparsers.add_parser(
             'start',
             help='Start a cluster.',
             parents=[parent_parser1, parent_parser2]
         )
-        parser_start.add_argument(
-            '-n', '--number',
+        paa = parser_start.add_argument
+        paa('-n', '--number',
             type=int, dest='Global.n',
             help='The number of engines to start.',
-            metavar='Global.n'
-        )
-        parser_start.add_argument('--clean-logs',
+            metavar='Global.n')
+        paa('--clean-logs',
             dest='Global.clean_logs', action='store_true',
-            help='Delete old log flies before starting.',
-        )
-        parser_start.add_argument('--no-clean-logs',
+            help='Delete old log flies before starting.')
+        paa('--no-clean-logs',
             dest='Global.clean_logs', action='store_false',
-            help="Don't delete old log flies before starting.",
-        )
-        parser_start.add_argument('--daemon',
+            help="Don't delete old log flies before starting.")
+        paa('--daemon',
             dest='Global.daemonize', action='store_true',
-            help='Daemonize the ipcluster program. This implies --log-to-file',
-        )
-        parser_start.add_argument('--no-daemon',
+            help='Daemonize the ipcluster program. This implies --log-to-file')
+        paa('--no-daemon',
             dest='Global.daemonize', action='store_false',
-            help="Dont't daemonize the ipcluster program.",
-        )
+            help="Dont't daemonize the ipcluster program.")
 
-        parser_start = subparsers.add_parser(
+        # The "stop" subcommand parser
+        parser_stop = subparsers.add_parser(
             'stop',
             help='Stop a cluster.',
             parents=[parent_parser1, parent_parser2]
         )
-        parser_start.add_argument('--signal',
+        paa = parser_stop.add_argument
+        paa('--signal',
             dest='Global.signal', type=int,
             help="The signal number to use in stopping the cluster (default=2).",
-            metavar="Global.signal",
-        )
-
-
-default_config_file_name = u'ipcluster_config.py'
+            metavar="Global.signal")
 
 
-_description = """Start an IPython cluster for parallel computing.\n\n
-
-An IPython cluster consists of 1 controller and 1 or more engines.
-This command automates the startup of these processes using a wide
-range of startup methods (SSH, local processes, PBS, mpiexec,
-Windows HPC Server 2008). To start a cluster with 4 engines on your
-local host simply do "ipcluster start -n 4". For more complex usage 
-you will typically do "ipcluster create -p mycluster", then edit
-configuration files, followed by "ipcluster start -p mycluster -n 4".
-"""
+#-----------------------------------------------------------------------------
+# Main application
+#-----------------------------------------------------------------------------
 
 
 class IPClusterApp(ApplicationWithClusterDir):
 
     name = u'ipcluster'
     description = _description
+    usage = None
+    command_line_loader = IPClusterAppConfigLoader
     config_file_name = default_config_file_name
     default_log_level = logging.INFO
     auto_create_cluster_dir = False
 
     def create_default_config(self):
         super(IPClusterApp, self).create_default_config()
         self.default_config.Global.controller_launcher = \
             'IPython.kernel.launcher.LocalControllerLauncher'
         self.default_config.Global.engine_launcher = \
             'IPython.kernel.launcher.LocalEngineSetLauncher'
         self.default_config.Global.n = 2
         self.default_config.Global.reset_config = False
         self.default_config.Global.clean_logs = True
         self.default_config.Global.signal = 2
         self.default_config.Global.daemonize = False
 
-    def create_command_line_config(self):
-        """Create and return a command line config loader."""
-        return IPClusterCLLoader(
-            description=self.description, 
-            version=release.version
-        )
-
     def find_resources(self):
         subcommand = self.command_line_config.Global.subcommand
         if subcommand=='list':
             self.list_cluster_dirs()
             # Exit immediately because there is nothing left to do.
             self.exit()
         elif subcommand=='create':
             self.auto_create_cluster_dir = True
             super(IPClusterApp, self).find_resources()
         elif subcommand=='start' or subcommand=='stop':
             self.auto_create_cluster_dir = True
             try:
                 super(IPClusterApp, self).find_resources()
             except ClusterDirError:
                 raise ClusterDirError(
                     "Could not find a cluster directory. A cluster dir must "
                     "be created before running 'ipcluster start'.  Do "
                     "'ipcluster create -h' or 'ipcluster list -h' for more "
                     "information about creating and listing cluster dirs."
                 )
 
     def list_cluster_dirs(self):
         # Find the search paths
         cluster_dir_paths = os.environ.get('IPCLUSTER_DIR_PATH','')
         if cluster_dir_paths:
             cluster_dir_paths = cluster_dir_paths.split(':')
         else:
             cluster_dir_paths = []
         try:
             ipython_dir = self.command_line_config.Global.ipython_dir
         except AttributeError:
             ipython_dir = self.default_config.Global.ipython_dir
         paths = [os.getcwd(), ipython_dir] + \
             cluster_dir_paths
         paths = list(set(paths))
 
         self.log.info('Searching for cluster dirs in paths: %r' % paths)
         for path in paths:
             files = os.listdir(path)
             for f in files:
                 full_path = os.path.join(path, f)
                 if os.path.isdir(full_path) and f.startswith('cluster_'):
                     profile = full_path.split('_')[-1]
                     start_cmd = 'ipcluster start -p %s -n 4' % profile
                     print start_cmd + " ==> " + full_path
 
     def pre_construct(self):
         # IPClusterApp.pre_construct() is where we cd to the working directory.
         super(IPClusterApp, self).pre_construct()
         config = self.master_config
         try:
             daemon = config.Global.daemonize
             if daemon:
                 config.Global.log_to_file = True
         except AttributeError:
             pass
 
     def construct(self):
         config = self.master_config
         subcmd = config.Global.subcommand
         reset = config.Global.reset_config
         if subcmd == 'list':
             return
         if subcmd == 'create':
             self.log.info('Copying default config files to cluster directory '
             '[overwrite=%r]' % (reset,))
             self.cluster_dir_obj.copy_all_config_files(overwrite=reset)
         if subcmd =='start':
             self.cluster_dir_obj.copy_all_config_files(overwrite=False)
             self.start_logging()
             reactor.callWhenRunning(self.start_launchers)
 
     def start_launchers(self):
         config = self.master_config
 
         # Create the launchers. In both bases, we set the work_dir of
         # the launcher to the cluster_dir. This is where the launcher's
         # subprocesses will be launched. It is not where the controller
         # and engine will be launched.
         el_class = import_item(config.Global.engine_launcher)
         self.engine_launcher = el_class(
             work_dir=self.cluster_dir, config=config
         )
         cl_class = import_item(config.Global.controller_launcher)
         self.controller_launcher = cl_class(
             work_dir=self.cluster_dir, config=config
         )
 
         # Setup signals
         signal.signal(signal.SIGINT, self.sigint_handler)
 
         # Setup the observing of stopping. If the controller dies, shut
         # everything down as that will be completely fatal for the engines.
         d1 = self.controller_launcher.observe_stop()
         d1.addCallback(self.stop_launchers)
         # But, we don't monitor the stopping of engines. An engine dying
         # is just fine and in principle a user could start a new engine.
         # Also, if we did monitor engine stopping, it is difficult to
         # know what to do when only some engines die. Currently, the
         # observing of engine stopping is inconsistent. Some launchers
         # might trigger on a single engine stopping, other wait until
         # all stop.  TODO: think more about how to handle this.
         
         # Start the controller and engines
         self._stopping = False  # Make sure stop_launchers is not called 2x.
         d = self.start_controller()
         d.addCallback(self.start_engines)
         d.addCallback(self.startup_message)
         # If the controller or engines fail to start, stop everything
         d.addErrback(self.stop_launchers)
         return d
 
     def startup_message(self, r=None):
         log.msg("IPython cluster: started")
         return r
         
     def start_controller(self, r=None):
         # log.msg("In start_controller")
         config = self.master_config
         d = self.controller_launcher.start(
             cluster_dir=config.Global.cluster_dir
         )
         return d
             
     def start_engines(self, r=None):
         # log.msg("In start_engines")
         config = self.master_config
         d = self.engine_launcher.start(
             config.Global.n,
             cluster_dir=config.Global.cluster_dir
         )
         return d
 
     def stop_controller(self, r=None):
         # log.msg("In stop_controller")
         if self.controller_launcher.running:
             d = self.controller_launcher.stop()
             d.addErrback(self.log_err)
             return d
         else:
             return defer.succeed(None)
 
     def stop_engines(self, r=None):
         # log.msg("In stop_engines")
         if self.engine_launcher.running:
             d = self.engine_launcher.stop()
             d.addErrback(self.log_err)
             return d
         else:
             return defer.succeed(None)
 
     def log_err(self, f):
         log.msg(f.getTraceback())
         return None
         
     def stop_launchers(self, r=None):
         if not self._stopping:
             self._stopping = True
             if isinstance(r, failure.Failure):
                 log.msg('Unexpected error in ipcluster:')
                 log.msg(r.getTraceback())
             log.msg("IPython cluster: stopping")
             # These return deferreds. We are not doing anything with them
             # but we are holding refs to them as a reminder that they 
             # do return deferreds.
             d1 = self.stop_engines()
             d2 = self.stop_controller()
             # Wait a few seconds to let things shut down.
             reactor.callLater(4.0, reactor.stop)
 
     def sigint_handler(self, signum, frame):
         self.stop_launchers()
         
     def start_logging(self):
         # Remove old log files of the controller and engine
         if self.master_config.Global.clean_logs:
             log_dir = self.master_config.Global.log_dir
             for f in os.listdir(log_dir):
                 if f.startswith('ipengine' + '-'):
                     if f.endswith('.log') or f.endswith('.out') or f.endswith('.err'):
                         os.remove(os.path.join(log_dir, f))
                 if f.startswith('ipcontroller' + '-'):
                     if f.endswith('.log') or f.endswith('.out') or f.endswith('.err'):
                         os.remove(os.path.join(log_dir, f))
         # This will remote old log files for ipcluster itself
         super(IPClusterApp, self).start_logging()
 
     def start_app(self):
         """Start the application, depending on what subcommand is used."""
         subcmd = self.master_config.Global.subcommand
         if subcmd=='create' or subcmd=='list':
             return
         elif subcmd=='start':
             self.start_app_start()
         elif subcmd=='stop':
             self.start_app_stop()
 
     def start_app_start(self):
         """Start the app for the start subcommand."""
         config = self.master_config
         # First see if the cluster is already running
         try:
             pid = self.get_pid_from_file()
         except PIDFileError:
             pass
         else:
             self.log.critical(
                 'Cluster is already running with [pid=%s]. '
                 'use "ipcluster stop" to stop the cluster.' % pid
             )
             # Here I exit with a unusual exit status that other processes
             # can watch for to learn how I existed.
             self.exit(ALREADY_STARTED)
 
         # Now log and daemonize
         self.log.info(
             'Starting ipcluster with [daemon=%r]' % config.Global.daemonize
         )
         # TODO: Get daemonize working on Windows or as a Windows Server.
         if config.Global.daemonize:
             if os.name=='posix':
                 daemonize()
 
         # Now write the new pid file AFTER our new forked pid is active.
         self.write_pid_file()
         reactor.addSystemEventTrigger('during','shutdown', self.remove_pid_file)
         reactor.run()
 
     def start_app_stop(self):
         """Start the app for the stop subcommand."""
         config = self.master_config
         try:
             pid = self.get_pid_from_file()
         except PIDFileError:
             self.log.critical(
                 'Problem reading pid file, cluster is probably not running.'
             )
             # Here I exit with a unusual exit status that other processes
             # can watch for to learn how I existed.
             self.exit(ALREADY_STOPPED)
         else:
             if os.name=='posix':
                 sig = config.Global.signal
                 self.log.info(
                     "Stopping cluster [pid=%r] with [signal=%r]" % (pid, sig)
                 )
                 os.kill(pid, sig)
             elif os.name=='nt':
                 # As of right now, we don't support daemonize on Windows, so
                 # stop will not do anything. Minimally, it should clean up the
                 # old .pid files.
                 self.remove_pid_file()
 
+
 def launch_new_instance():
     """Create and run the IPython cluster."""
     app = IPClusterApp()
     app.start()
 
 
 if __name__ == '__main__':
     launch_new_instance()
 

IPython/kernel/ipcontrollerapp.py

@@ -1,253 +1,260 b''
 #!/usr/bin/env python
 # encoding: utf-8
 """
 The IPython controller application.
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008-2009  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 from __future__ import with_statement
 
 import copy
 import sys
 
 from twisted.application import service
 from twisted.internet import reactor
 from twisted.python import log
 
 from IPython.config.loader import Config
-from IPython.core.application import Application
 from IPython.kernel import controllerservice
-from IPython.kernel.clusterdir import ApplicationWithClusterDir
+from IPython.kernel.clusterdir import (
+    ApplicationWithClusterDir,
+    ClusterDirConfigLoader
+)
 from IPython.kernel.fcutil import FCServiceFactory
 from IPython.utils.traitlets import Instance, Unicode
 
+
+#-----------------------------------------------------------------------------
+# Module level variables
+#-----------------------------------------------------------------------------
+
+
+#: The default config file name for this application
+default_config_file_name = u'ipcontroller_config.py'
+
+
+_description = """Start the IPython controller for parallel computing.
+
+The IPython controller provides a gateway between the IPython engines and
+clients. The controller needs to be started before the engines and can be
+configured using command line options or using a cluster directory. Cluster
+directories contain config, log and security files and are usually located in
+your .ipython directory and named as "cluster_<profile>". See the --profile
+and --cluster-dir options for details.
+"""
+
 #-----------------------------------------------------------------------------
 # Default interfaces
 #-----------------------------------------------------------------------------
 
 # The default client interfaces for FCClientServiceFactory.interfaces
 default_client_interfaces = Config()
 default_client_interfaces.Task.interface_chain = [
     'IPython.kernel.task.ITaskController',
     'IPython.kernel.taskfc.IFCTaskController'
 ]
 
 default_client_interfaces.Task.furl_file = 'ipcontroller-tc.furl'
 
 default_client_interfaces.MultiEngine.interface_chain = [
     'IPython.kernel.multiengine.IMultiEngine',
     'IPython.kernel.multienginefc.IFCSynchronousMultiEngine'
 ]
 
 default_client_interfaces.MultiEngine.furl_file = u'ipcontroller-mec.furl'
 
 # Make this a dict we can pass to Config.__init__ for the default
 default_client_interfaces = dict(copy.deepcopy(default_client_interfaces.items()))
 
 
 
 # The default engine interfaces for FCEngineServiceFactory.interfaces
 default_engine_interfaces = Config()
 default_engine_interfaces.Default.interface_chain = [
     'IPython.kernel.enginefc.IFCControllerBase'
 ]
 
 default_engine_interfaces.Default.furl_file = u'ipcontroller-engine.furl'
 
 # Make this a dict we can pass to Config.__init__ for the default
 default_engine_interfaces = dict(copy.deepcopy(default_engine_interfaces.items()))
 
 
 #-----------------------------------------------------------------------------
 # Service factories
 #-----------------------------------------------------------------------------
 
 
 class FCClientServiceFactory(FCServiceFactory):
     """A Foolscap implementation of the client services."""
 
     cert_file = Unicode(u'ipcontroller-client.pem', config=True)
     interfaces = Instance(klass=Config, kw=default_client_interfaces,
                           allow_none=False, config=True)
 
 
 class FCEngineServiceFactory(FCServiceFactory):
     """A Foolscap implementation of the engine services."""
 
     cert_file = Unicode(u'ipcontroller-engine.pem', config=True)
     interfaces = Instance(klass=dict, kw=default_engine_interfaces,
                           allow_none=False, config=True)
 
 
 #-----------------------------------------------------------------------------
-# The main application
+# Command line options
 #-----------------------------------------------------------------------------
 
 
-cl_args = (
-    # Client config
-    (('--client-ip',), dict(
-        type=str, dest='FCClientServiceFactory.ip', 
-        help='The IP address or hostname the controller will listen on for '
-        'client connections.',
-        metavar='FCClientServiceFactory.ip')
-    ),
-    (('--client-port',), dict(
-        type=int, dest='FCClientServiceFactory.port', 
-        help='The port the controller will listen on for client connections. '
-        'The default is to use 0, which will autoselect an open port.',
-        metavar='FCClientServiceFactory.port')
-    ),
-    (('--client-location',), dict(
-        type=str, dest='FCClientServiceFactory.location',
-        help='The hostname or IP that clients should connect to. This does '
-        'not control which interface the controller listens on. Instead, this '
-        'determines the hostname/IP that is listed in the FURL, which is how '
-        'clients know where to connect. Useful if the controller is listening '
-        'on multiple interfaces.',
-        metavar='FCClientServiceFactory.location')
-    ),
-    # Engine config
-    (('--engine-ip',), dict(
-        type=str, dest='FCEngineServiceFactory.ip', 
-        help='The IP address or hostname the controller will listen on for '
-        'engine connections.',
-        metavar='FCEngineServiceFactory.ip')
-    ),
-    (('--engine-port',), dict(
-        type=int, dest='FCEngineServiceFactory.port',
-        help='The port the controller will listen on for engine connections. '
-        'The default is to use 0, which will autoselect an open port.',
-        metavar='FCEngineServiceFactory.port')
-    ),
-    (('--engine-location',), dict(
-        type=str, dest='FCEngineServiceFactory.location',
-        help='The hostname or IP that engines should connect to. This does '
-        'not control which interface the controller listens on. Instead, this '
-        'determines the hostname/IP that is listed in the FURL, which is how '
-        'engines know where to connect. Useful if the controller is listening '
-        'on multiple interfaces.',
-        metavar='FCEngineServiceFactory.location')
-    ),
-    # Global config
-    (('--log-to-file',), dict(
-        action='store_true', dest='Global.log_to_file',
-        help='Log to a file in the log directory (default is stdout)')
-    ),
-    (('-r','--reuse-furls'), dict(
-        action='store_true', dest='Global.reuse_furls',
-        help='Try to reuse all FURL files. If this is not set all FURL files '
-        'are deleted before the controller starts. This must be set if '
-        'specific ports are specified by --engine-port or --client-port.')
-    ),
-    (('--no-secure',), dict(
-        action='store_false', dest='Global.secure',
-        help='Turn off SSL encryption for all connections.')
-    ),
-    (('--secure',), dict(
-        action='store_true', dest='Global.secure',
-        help='Turn off SSL encryption for all connections.')
-    )
-)
+class IPControllerAppConfigLoader(ClusterDirConfigLoader):
+
+    def _add_arguments(self):
+        super(IPControllerAppConfigLoader, self)._add_arguments()
+        paa = self.parser.add_argument
+        # Client config
+        paa('--client-ip',
+            type=str, dest='FCClientServiceFactory.ip', 
+            help='The IP address or hostname the controller will listen on for '
+            'client connections.',
+            metavar='FCClientServiceFactory.ip')
+        paa('--client-port',
+            type=int, dest='FCClientServiceFactory.port', 
+            help='The port the controller will listen on for client connections. '
+            'The default is to use 0, which will autoselect an open port.',
+            metavar='FCClientServiceFactory.port')
+        paa('--client-location',), dict(
+            type=str, dest='FCClientServiceFactory.location',
+            help='The hostname or IP that clients should connect to. This does '
+            'not control which interface the controller listens on. Instead, this '
+            'determines the hostname/IP that is listed in the FURL, which is how '
+            'clients know where to connect. Useful if the controller is listening '
+            'on multiple interfaces.',
+            metavar='FCClientServiceFactory.location')
+        # Engine config
+        paa('--engine-ip',
+            type=str, dest='FCEngineServiceFactory.ip', 
+            help='The IP address or hostname the controller will listen on for '
+            'engine connections.',
+            metavar='FCEngineServiceFactory.ip')
+        paa('--engine-port',
+            type=int, dest='FCEngineServiceFactory.port',
+            help='The port the controller will listen on for engine connections. '
+            'The default is to use 0, which will autoselect an open port.',
+            metavar='FCEngineServiceFactory.port')
+        paa('--engine-location',
+            type=str, dest='FCEngineServiceFactory.location',
+            help='The hostname or IP that engines should connect to. This does '
+            'not control which interface the controller listens on. Instead, this '
+            'determines the hostname/IP that is listed in the FURL, which is how '
+            'engines know where to connect. Useful if the controller is listening '
+            'on multiple interfaces.',
+            metavar='FCEngineServiceFactory.location')
+        # Global config
+        paa('--log-to-file',
+            action='store_true', dest='Global.log_to_file',
+            help='Log to a file in the log directory (default is stdout)')
+        paa('-r','--reuse-furls', 
+            action='store_true', dest='Global.reuse_furls',
+            help='Try to reuse all FURL files. If this is not set all FURL files '
+            'are deleted before the controller starts. This must be set if '
+            'specific ports are specified by --engine-port or --client-port.')
+        paa('--no-secure',
+            action='store_false', dest='Global.secure',
+            help='Turn off SSL encryption for all connections.')
+        paa('--secure',
+            action='store_true', dest='Global.secure',
+            help='Turn off SSL encryption for all connections.')
 
 
-_description = """Start the IPython controller for parallel computing.
-
-The IPython controller provides a gateway between the IPython engines and
-clients. The controller needs to be started before the engines and can be
-configured using command line options or using a cluster directory. Cluster
-directories contain config, log and security files and are usually located in
-your .ipython directory and named as "cluster_<profile>". See the --profile
-and --cluster-dir options for details.
-"""
-
-default_config_file_name = u'ipcontroller_config.py'
+#-----------------------------------------------------------------------------
+# The main application
+#-----------------------------------------------------------------------------
 
 
 class IPControllerApp(ApplicationWithClusterDir):
 
     name = u'ipcontroller'
     description = _description
+    command_line_loader = IPControllerAppConfigLoader
     config_file_name = default_config_file_name
     auto_create_cluster_dir = True
-    cl_arguments = Application.cl_arguments + cl_args
 
     def create_default_config(self):
         super(IPControllerApp, self).create_default_config()
         self.default_config.Global.reuse_furls = False
         self.default_config.Global.secure = True
         self.default_config.Global.import_statements = []
         self.default_config.Global.clean_logs = True
 
     def post_load_command_line_config(self):
         # Now setup reuse_furls
         c = self.command_line_config
         if hasattr(c.Global, 'reuse_furls'):
             c.FCClientServiceFactory.reuse_furls = c.Global.reuse_furls
             c.FCEngineServiceFactory.reuse_furls = c.Global.reuse_furls
             del c.Global.reuse_furls
         if hasattr(c.Global, 'secure'):
             c.FCClientServiceFactory.secure = c.Global.secure
             c.FCEngineServiceFactory.secure = c.Global.secure
             del c.Global.secure
 
     def construct(self):
         # This is the working dir by now.
         sys.path.insert(0, '')
 
         self.start_logging()
         self.import_statements()
 
         # Create the service hierarchy
         self.main_service = service.MultiService()
         # The controller service
         controller_service = controllerservice.ControllerService()
         controller_service.setServiceParent(self.main_service)
         # The client tub and all its refereceables
         csfactory = FCClientServiceFactory(self.master_config, controller_service)
         client_service = csfactory.create()
         client_service.setServiceParent(self.main_service)
         # The engine tub
         esfactory = FCEngineServiceFactory(self.master_config, controller_service)
         engine_service = esfactory.create()
         engine_service.setServiceParent(self.main_service)
 
     def import_statements(self):
         statements = self.master_config.Global.import_statements
         for s in statements:
             try:
                 log.msg("Executing statement: '%s'" % s)
                 exec s in globals(), locals()
             except:
                 log.msg("Error running statement: %s" % s)
 
     def start_app(self):
         # Start the controller service.
         self.main_service.startService()
         # Write the .pid file overwriting old ones. This allow multiple
         # controllers to clober each other. But Windows is not cleaning
         # these up properly.
         self.write_pid_file(overwrite=True)
         # Add a trigger to delete the .pid file upon shutting down.
         reactor.addSystemEventTrigger('during','shutdown', self.remove_pid_file)
         reactor.run()
 
 
 def launch_new_instance():
     """Create and run the IPython controller"""
     app = IPControllerApp()
     app.start()
 
 
 if __name__ == '__main__':
     launch_new_instance()

IPython/kernel/ipengineapp.py

@@ -1,229 +1,242 b''
 #!/usr/bin/env python
 # encoding: utf-8
 """
 The IPython controller application
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008-2009  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import os
 import sys
 
 from twisted.application import service
 from twisted.internet import reactor
 from twisted.python import log
 
-from IPython.core.application import Application
-from IPython.kernel.clusterdir import ApplicationWithClusterDir
+from IPython.kernel.clusterdir import (
+    ApplicationWithClusterDir,
+    ClusterDirConfigLoader
+)
 from IPython.kernel.engineconnector import EngineConnector
 from IPython.kernel.engineservice import EngineService
 from IPython.kernel.fcutil import Tub
 from IPython.utils.importstring import import_item
 
 #-----------------------------------------------------------------------------
-# The main application
+# Module level variables
 #-----------------------------------------------------------------------------
 
-cl_args = (
-    # Controller config
-    (('--furl-file',), dict(
-        type=unicode, dest='Global.furl_file',
-        help='The full location of the file containing the FURL of the '
-        'controller. If this is not given, the FURL file must be in the '
-        'security directory of the cluster directory.  This location is '
-        'resolved using the --profile and --app-dir options.',
-        metavar='Global.furl_file')
-    ),
-    # MPI
-    (('--mpi',), dict(
-        type=str, dest='MPI.use',
-        help='How to enable MPI (mpi4py, pytrilinos, or empty string to disable).',
-        metavar='MPI.use')
-    ),
-    # Global config
-    (('--log-to-file',), dict(
-        action='store_true', dest='Global.log_to_file',
-        help='Log to a file in the log directory (default is stdout)')
-    )
-)
+#: The default config file name for this application
+default_config_file_name = u'ipengine_config.py'
 
 
 mpi4py_init = """from mpi4py import MPI as mpi
 mpi.size = mpi.COMM_WORLD.Get_size()
 mpi.rank = mpi.COMM_WORLD.Get_rank()
 """
 
+
 pytrilinos_init = """from PyTrilinos import Epetra
 class SimpleStruct:
 pass
 mpi = SimpleStruct()
 mpi.rank = 0
 mpi.size = 0
 """
 
 
-default_config_file_name = u'ipengine_config.py'
-
-
 _description = """Start an IPython engine for parallel computing.\n\n
 
 IPython engines run in parallel and perform computations on behalf of a client
 and controller. A controller needs to be started before the engines. The
 engine can be configured using command line options or using a cluster
 directory. Cluster directories contain config, log and security files and are
 usually located in your .ipython directory and named as "cluster_<profile>".
 See the --profile and --cluster-dir options for details.
 """
 
+#-----------------------------------------------------------------------------
+# Command line options
+#-----------------------------------------------------------------------------
+
+
+class IPEngineAppConfigLoader(ClusterDirConfigLoader):
+
+    def _add_arguments(self):
+        super(IPEngineAppConfigLoader, self)._add_arguments()
+        paa = self.parser.add_argument
+        # Controller config
+        paa('--furl-file',
+            type=unicode, dest='Global.furl_file',
+            help='The full location of the file containing the FURL of the '
+            'controller. If this is not given, the FURL file must be in the '
+            'security directory of the cluster directory.  This location is '
+            'resolved using the --profile and --app-dir options.',
+            metavar='Global.furl_file')
+        # MPI
+        paa('--mpi',
+            type=str, dest='MPI.use',
+            help='How to enable MPI (mpi4py, pytrilinos, or empty string to disable).',
+            metavar='MPI.use')
+        # Global config
+        paa('--log-to-file',
+            action='store_true', dest='Global.log_to_file',
+            help='Log to a file in the log directory (default is stdout)')
+
+
+#-----------------------------------------------------------------------------
+# Main application
+#-----------------------------------------------------------------------------
+
 
 class IPEngineApp(ApplicationWithClusterDir):
 
     name = u'ipengine'
     description = _description
+    command_line_loader = IPEngineAppConfigLoader
     config_file_name = default_config_file_name
     auto_create_cluster_dir = True
-    cl_arguments = Application.cl_arguments + cl_args
 
     def create_default_config(self):
         super(IPEngineApp, self).create_default_config()
 
         # The engine should not clean logs as we don't want to remove the
         # active log files of other running engines.
         self.default_config.Global.clean_logs = False
 
         # Global config attributes
         self.default_config.Global.exec_lines = []
         self.default_config.Global.shell_class = 'IPython.kernel.core.interpreter.Interpreter'
 
         # Configuration related to the controller
         # This must match the filename (path not included) that the controller
         # used for the FURL file.
         self.default_config.Global.furl_file_name = u'ipcontroller-engine.furl'
         # If given, this is the actual location of the controller's FURL file.
         # If not, this is computed using the profile, app_dir and furl_file_name
         self.default_config.Global.furl_file = u''
 
         # The max number of connection attemps and the initial delay between
         # those attemps.
         self.default_config.Global.connect_delay = 0.1
         self.default_config.Global.connect_max_tries = 15
 
         # MPI related config attributes
         self.default_config.MPI.use = ''
         self.default_config.MPI.mpi4py = mpi4py_init
         self.default_config.MPI.pytrilinos = pytrilinos_init
 
     def post_load_command_line_config(self):
         pass
 
     def pre_construct(self):
         super(IPEngineApp, self).pre_construct()
         self.find_cont_furl_file()
 
     def find_cont_furl_file(self):
         """Set the furl file.
 
         Here we don't try to actually see if it exists for is valid as that
         is hadled by the connection logic.
         """
         config = self.master_config
         # Find the actual controller FURL file
         if not config.Global.furl_file:
             try_this = os.path.join(
                 config.Global.cluster_dir, 
                 config.Global.security_dir,
                 config.Global.furl_file_name
             )
             config.Global.furl_file = try_this
 
     def construct(self):
         # This is the working dir by now.
         sys.path.insert(0, '')
 
         self.start_mpi()
         self.start_logging()
 
         # Create the underlying shell class and EngineService
         shell_class = import_item(self.master_config.Global.shell_class)
         self.engine_service = EngineService(shell_class, mpi=mpi)
 
         self.exec_lines()
 
         # Create the service hierarchy
         self.main_service = service.MultiService()
         self.engine_service.setServiceParent(self.main_service)
         self.tub_service = Tub()
         self.tub_service.setServiceParent(self.main_service)
         # This needs to be called before the connection is initiated
         self.main_service.startService()
 
         # This initiates the connection to the controller and calls
         # register_engine to tell the controller we are ready to do work
         self.engine_connector = EngineConnector(self.tub_service)
 
         log.msg("Using furl file: %s" % self.master_config.Global.furl_file)
 
         reactor.callWhenRunning(self.call_connect)
 
     def call_connect(self):
         d = self.engine_connector.connect_to_controller(
             self.engine_service, 
             self.master_config.Global.furl_file,
             self.master_config.Global.connect_delay,
             self.master_config.Global.connect_max_tries
         )
 
         def handle_error(f):
             log.msg('Error connecting to controller. This usually means that '
             'i) the controller was not started, ii) a firewall was blocking '
             'the engine from connecting to the controller or iii) the engine '
             ' was not pointed at the right FURL file:')
             log.msg(f.getErrorMessage())
             reactor.callLater(0.1, reactor.stop)
 
         d.addErrback(handle_error)
 
     def start_mpi(self):
         global mpi
         mpikey = self.master_config.MPI.use
         mpi_import_statement = self.master_config.MPI.get(mpikey, None)
         if mpi_import_statement is not None:
             try:
                 self.log.info("Initializing MPI:")
                 self.log.info(mpi_import_statement)
                 exec mpi_import_statement in globals()
             except:
                 mpi = None
         else:
             mpi = None
 
     def exec_lines(self):
         for line in self.master_config.Global.exec_lines:
             try:
                 log.msg("Executing statement: '%s'" % line)
                 self.engine_service.execute(line)
             except:
                 log.msg("Error executing statement: %s" % line)
 
     def start_app(self):
         reactor.run()
 
 
 def launch_new_instance():
     """Create and run the IPython controller"""
     app = IPEngineApp()
     app.start()
 
 
 if __name__ == '__main__':
     launch_new_instance()