promote aliases and flags, to ensure they have priority over config files

add Application.flatten_flags() method, which adjusts the alias
and flag dicts, such that they point to the subclass in the Application.classes list when passed to the argv parser.

This prevents TerminalInteractiveShell.colors in a config file overriding
`--colors` on the command-line, which points to InteractiveShell.colors.

Flattening is only done when the answer is unambiguous, so multiply inherited classes (e.g. Launchers in ipcluster) are not touched.

also remove now-obsolete manual workaround for this in IPClusterStart

closes gh-849

IPython/config/application.py

@@ -1,436 +1,485 b''
 # encoding: utf-8
 """
 A base class for a configurable application.
 
 Authors:
 
 * Brian Granger
 * Min RK
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008-2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import logging
 import os
 import re
 import sys
 from copy import deepcopy
+from collections import defaultdict
 
 from IPython.config.configurable import SingletonConfigurable
 from IPython.config.loader import (
     KVArgParseConfigLoader, PyFileConfigLoader, Config, ArgumentError, ConfigFileNotFound,
 )
 
 from IPython.utils.traitlets import (
     Unicode, List, Int, Enum, Dict, Instance, TraitError
 )
 from IPython.utils.importstring import import_item
 from IPython.utils.text import indent, wrap_paragraphs, dedent
 
 #-----------------------------------------------------------------------------
 # function for re-wrapping a helpstring
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Descriptions for the various sections
 #-----------------------------------------------------------------------------
 
 # merge flags&aliases into options
 option_description = """
 Arguments that take values are actually convenience aliases to full
 Configurables, whose aliases are listed on the help line. For more information
 on full configurables, see '--help-all'.
 """.strip() # trim newlines of front and back
 
 keyvalue_description = """
 Parameters are set from command-line arguments of the form:
 `--Class.trait=value`.
 This line is evaluated in Python, so simple expressions are allowed, e.g.::
 `--C.a='range(3)'` For setting C.a=[0,1,2].
 """.strip() # trim newlines of front and back
 
 subcommand_description = """
 Subcommands are launched as `{app} cmd [args]`. For information on using
 subcommand 'cmd', do: `{app} cmd -h`.
 """.strip().format(app=os.path.basename(sys.argv[0]))
 # get running program name
 
 #-----------------------------------------------------------------------------
 # Application class
 #-----------------------------------------------------------------------------
 
 
 class ApplicationError(Exception):
     pass
 
 
 class Application(SingletonConfigurable):
     """A singleton application with full configuration support."""
 
     # The name of the application, will usually match the name of the command
     # line application
     name = Unicode(u'application')
 
     # The description of the application that is printed at the beginning
     # of the help.
     description = Unicode(u'This is an application.')
     # default section descriptions
     option_description = Unicode(option_description)
     keyvalue_description = Unicode(keyvalue_description)
     subcommand_description = Unicode(subcommand_description)
 
     # The usage and example string that goes at the end of the help string.
     examples = Unicode()
 
     # A sequence of Configurable subclasses whose config=True attributes will
     # be exposed at the command line.
     classes = List([])
 
     # The version string of this application.
     version = Unicode(u'0.0')
 
     # The log level for the application
     log_level = Enum((0,10,20,30,40,50,'DEBUG','INFO','WARN','ERROR','CRITICAL'),
                     default_value=logging.WARN,
                     config=True,
                     help="Set the log level by value or name.")
     def _log_level_changed(self, name, old, new):
         """Adjust the log level when log_level is set."""
         if isinstance(new, basestring):
             new = getattr(logging, new)
             self.log_level = new
         self.log.setLevel(new)
 
     # the alias map for configurables
     aliases = Dict({'log-level' : 'Application.log_level'})
 
     # flags for loading Configurables or store_const style flags
     # flags are loaded from this dict by '--key' flags
     # this must be a dict of two-tuples, the first element being the Config/dict
     # and the second being the help string for the flag
     flags = Dict()
     def _flags_changed(self, name, old, new):
         """ensure flags dict is valid"""
         for key,value in new.iteritems():
             assert len(value) == 2, "Bad flag: %r:%s"%(key,value)
             assert isinstance(value[0], (dict, Config)), "Bad flag: %r:%s"%(key,value)
             assert isinstance(value[1], basestring), "Bad flag: %r:%s"%(key,value)
 
 
     # subcommands for launching other applications
     # if this is not empty, this will be a parent Application
     # this must be a dict of two-tuples,
     # the first element being the application class/import string
     # and the second being the help string for the subcommand
     subcommands = Dict()
     # parse_command_line will initialize a subapp, if requested
     subapp = Instance('IPython.config.application.Application', allow_none=True)
 
     # extra command-line arguments that don't set config values
     extra_args = List(Unicode)
 
 
     def __init__(self, **kwargs):
         SingletonConfigurable.__init__(self, **kwargs)
         # Ensure my class is in self.classes, so my attributes appear in command line
         # options and config files.
         if self.__class__ not in self.classes:
             self.classes.insert(0, self.__class__)
 
         self.init_logging()
 
     def _config_changed(self, name, old, new):
         SingletonConfigurable._config_changed(self, name, old, new)
         self.log.debug('Config changed:')
         self.log.debug(repr(new))
 
     def init_logging(self):
         """Start logging for this application.
 
         The default is to log to stdout using a StreaHandler. The log level
         starts at loggin.WARN, but this can be adjusted by setting the
         ``log_level`` attribute.
         """
         self.log = logging.getLogger(self.__class__.__name__)
         self.log.setLevel(self.log_level)
         if sys.executable.endswith('pythonw.exe'):
             # this should really go to a file, but file-logging is only
             # hooked up in parallel applications
             self._log_handler = logging.StreamHandler(open(os.devnull, 'w'))
         else:
             self._log_handler = logging.StreamHandler()
         self._log_formatter = logging.Formatter("[%(name)s] %(message)s")
         self._log_handler.setFormatter(self._log_formatter)
         self.log.addHandler(self._log_handler)
 
     def initialize(self, argv=None):
         """Do the basic steps to configure me.
 
         Override in subclasses.
         """
         self.parse_command_line(argv)
 
 
     def start(self):
         """Start the app mainloop.
 
         Override in subclasses.
         """
         if self.subapp is not None:
             return self.subapp.start()
 
     def print_alias_help(self):
         """Print the alias part of the help."""
         if not self.aliases:
             return
 
         lines = []
         classdict = {}
         for cls in self.classes:
             # include all parents (up to, but excluding Configurable) in available names
             for c in cls.mro()[:-3]:
                 classdict[c.__name__] = c
 
         for alias, longname in self.aliases.iteritems():
             classname, traitname = longname.split('.',1)
             cls = classdict[classname]
 
             trait = cls.class_traits(config=True)[traitname]
             help = cls.class_get_trait_help(trait).splitlines()
             # reformat first line
             help[0] = help[0].replace(longname, alias) + ' (%s)'%longname
             if len(alias) == 1:
                 help[0] = help[0].replace('--%s='%alias, '-%s '%alias)
             lines.extend(help)
         # lines.append('')
         print os.linesep.join(lines)
 
     def print_flag_help(self):
         """Print the flag part of the help."""
         if not self.flags:
             return
 
         lines = []
         for m, (cfg,help) in self.flags.iteritems():
             prefix = '--' if len(m) > 1 else '-'
             lines.append(prefix+m)
             lines.append(indent(dedent(help.strip())))
         # lines.append('')
         print os.linesep.join(lines)
 
     def print_options(self):
         if not self.flags and not self.aliases:
             return
         lines = ['Options']
         lines.append('-'*len(lines[0]))
         lines.append('')
         for p in wrap_paragraphs(self.option_description):
             lines.append(p)
             lines.append('')
         print os.linesep.join(lines)
         self.print_flag_help()
         self.print_alias_help()
         print
 
     def print_subcommands(self):
         """Print the subcommand part of the help."""
         if not self.subcommands:
             return
 
         lines = ["Subcommands"]
         lines.append('-'*len(lines[0]))
         lines.append('')
         for p in wrap_paragraphs(self.subcommand_description):
             lines.append(p)
             lines.append('')
         for subc, (cls, help) in self.subcommands.iteritems():
             lines.append(subc)
             if help:
                 lines.append(indent(dedent(help.strip())))
         lines.append('')
         print os.linesep.join(lines)
 
     def print_help(self, classes=False):
         """Print the help for each Configurable class in self.classes.
 
         If classes=False (the default), only flags and aliases are printed.
         """
         self.print_subcommands()
         self.print_options()
 
         if classes:
             if self.classes:
                 print "Class parameters"
                 print "----------------"
                 print
                 for p in wrap_paragraphs(self.keyvalue_description):
                     print p
                     print
 
             for cls in self.classes:
                 cls.class_print_help()
                 print
         else:
             print "To see all available configurables, use `--help-all`"
             print
 
     def print_description(self):
         """Print the application description."""
         for p in wrap_paragraphs(self.description):
             print p
             print
 
     def print_examples(self):
         """Print usage and examples.
 
         This usage string goes at the end of the command line help string
         and should contain examples of the application's usage.
         """
         if self.examples:
             print "Examples"
             print "--------"
             print
             print indent(dedent(self.examples.strip()))
             print
 
     def print_version(self):
         """Print the version string."""
         print self.version
 
     def update_config(self, config):
         """Fire the traits events when the config is updated."""
         # Save a copy of the current config.
         newconfig = deepcopy(self.config)
         # Merge the new config into the current one.
         newconfig._merge(config)
         # Save the combined config as self.config, which triggers the traits
         # events.
         self.config = newconfig
 
     def initialize_subcommand(self, subc, argv=None):
         """Initialize a subcommand with argv."""
         subapp,help = self.subcommands.get(subc)
 
         if isinstance(subapp, basestring):
             subapp = import_item(subapp)
 
         # clear existing instances
         self.__class__.clear_instance()
         # instantiate
         self.subapp = subapp.instance()
         # and initialize subapp
         self.subapp.initialize(argv)
 
+    def flatten_flags(self):
+        """flatten flags and aliases, so cl-args override as expected.
+        
+        This prevents issues such as an alias pointing to InteractiveShell,
+        but a config file setting the same trait in TerminalInteraciveShell
+        getting inappropriate priority over the command-line arg.
+
+        Only aliases with exactly one descendent in the class list
+        will be promoted.
+        
+        """
+        # build a tree of classes in our list that inherit from a particular
+        # it will be a dict by parent classname of classes in our list
+        # that are descendents
+        mro_tree = defaultdict(list)
+        for cls in self.classes:
+            clsname = cls.__name__
+            for parent in cls.mro()[1:-3]:
+                # exclude cls itself and Configurable,HasTraits,object
+                mro_tree[parent.__name__].append(clsname)
+        # flatten aliases, which have the form:
+        # { 'alias' : 'Class.trait' }
+        aliases = {}
+        for alias, cls_trait in self.aliases.iteritems():
+            cls,trait = cls_trait.split('.',1)
+            children = mro_tree[cls]
+            if len(children) == 1:
+                # exactly one descendent, promote alias
+                cls = children[0]
+            aliases[alias] = '.'.join([cls,trait])
+        
+        # flatten flags, which are of the form:
+        # { 'key' : ({'Cls' : {'trait' : value}}, 'help')}
+        flags = {}
+        for key, (flagdict, help) in self.flags.iteritems():
+            newflag = {}
+            for cls, subdict in flagdict.iteritems():
+                children = mro_tree[cls]
+                # exactly one descendent, promote flag section
+                if len(children) == 1:
+                    cls = children[0]
+                newflag[cls] = subdict
+            flags[key] = (newflag, help)
+        return flags, aliases
+
     def parse_command_line(self, argv=None):
         """Parse the command line arguments."""
         argv = sys.argv[1:] if argv is None else argv
 
         if self.subcommands and len(argv) > 0:
             # we have subcommands, and one may have been specified
             subc, subargv = argv[0], argv[1:]
             if re.match(r'^\w(\-?\w)*$', subc) and subc in self.subcommands:
                 # it's a subcommand, and *not* a flag or class parameter
                 return self.initialize_subcommand(subc, subargv)
 
         if '-h' in argv or '--help' in argv or '--help-all' in argv:
             self.print_description()
             self.print_help('--help-all' in argv)
             self.print_examples()
             self.exit(0)
 
         if '--version' in argv:
             self.print_version()
             self.exit(0)
 
-        loader = KVArgParseConfigLoader(argv=argv, aliases=self.aliases,
-                                        flags=self.flags)
+        # flatten flags&aliases, so cl-args get appropriate priority:
+        flags,aliases = self.flatten_flags()
+
+        loader = KVArgParseConfigLoader(argv=argv, aliases=aliases,
+                                        flags=flags)
         try:
             config = loader.load_config()
             self.update_config(config)
         except (TraitError, ArgumentError) as e:
             self.print_description()
             self.print_help()
             self.print_examples()
             self.log.fatal(str(e))
             self.exit(1)
         # store unparsed args in extra_args
         self.extra_args = loader.extra_args
 
     def load_config_file(self, filename, path=None):
         """Load a .py based config file by filename and path."""
         loader = PyFileConfigLoader(filename, path=path)
         try:
             config = loader.load_config()
         except ConfigFileNotFound:
             # problem finding the file, raise
             raise
         except Exception:
             # try to get the full filename, but it will be empty in the
             # unlikely event that the error raised before filefind finished
             filename = loader.full_filename or filename
             # problem while running the file
             self.log.error("Exception while loading config file %s",
                             filename, exc_info=True)
         else:
             self.log.debug("Loaded config file: %s", loader.full_filename)
             self.update_config(config)
 
     def generate_config_file(self):
         """generate default config file from Configurables"""
         lines = ["# Configuration file for %s."%self.name]
         lines.append('')
         lines.append('c = get_config()')
         lines.append('')
         for cls in self.classes:
             lines.append(cls.class_config_section())
         return '\n'.join(lines)
 
     def exit(self, exit_status=0):
         self.log.debug("Exiting application: %s" % self.name)
         sys.exit(exit_status)
 
 #-----------------------------------------------------------------------------
 # utility functions, for convenience
 #-----------------------------------------------------------------------------
 
 def boolean_flag(name, configurable, set_help='', unset_help=''):
     """Helper for building basic --trait, --no-trait flags.
 
     Parameters
     ----------
 
     name : str
         The name of the flag.
     configurable : str
         The 'Class.trait' string of the trait to be set/unset with the flag
     set_help : unicode
         help string for --name flag
     unset_help : unicode
         help string for --no-name flag
 
     Returns
     -------
 
     cfg : dict
         A dict with two keys: 'name', and 'no-name', for setting and unsetting
         the trait, respectively.
     """
     # default helpstrings
     set_help = set_help or "set %s=True"%configurable
     unset_help = unset_help or "set %s=False"%configurable
 
     cls,trait = configurable.split('.')
 
     setter = {cls : {trait : True}}
     unsetter = {cls : {trait : False}}
     return {name : (setter, set_help), 'no-'+name : (unsetter, unset_help)}
 

IPython/config/tests/test_application.py

@@ -1,146 +1,175 b''
 """
 Tests for IPython.config.application.Application
 
 Authors:
 
 * Brian Granger
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008-2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
+import logging
 from unittest import TestCase
 
 from IPython.config.configurable import Configurable
+from IPython.config.loader import Config
 
 from IPython.config.application import (
     Application
 )
 
 from IPython.utils.traitlets import (
     Bool, Unicode, Int, Float, List, Dict
 )
 
 #-----------------------------------------------------------------------------
 # Code
 #-----------------------------------------------------------------------------
 
 class Foo(Configurable):
 
     i = Int(0, config=True, help="The integer i.")
     j = Int(1, config=True, help="The integer j.")
     name = Unicode(u'Brian', config=True, help="First name.")
 
 
 class Bar(Configurable):
 
     b = Int(0, config=True, help="The integer b.")
     enabled = Bool(True, config=True, help="Enable bar.")
 
 
 class MyApp(Application):
 
     name = Unicode(u'myapp')
     running = Bool(False, config=True,
                    help="Is the app running?")
     classes = List([Bar, Foo])
     config_file = Unicode(u'', config=True,
                    help="Load this config file")
 
     aliases = Dict({
                     'i' : 'Foo.i',
                     'j' : 'Foo.j',
                     'name' : 'Foo.name',
                     'enabled' : 'Bar.enabled',
-                    'log-level' : 'MyApp.log_level',
+                    'log-level' : 'Application.log_level',
                 })
     
     flags = Dict(dict(enable=({'Bar': {'enabled' : True}}, "Set Bar.enabled to True"),
-                  disable=({'Bar': {'enabled' : False}}, "Set Bar.enabled to False")))
+                  disable=({'Bar': {'enabled' : False}}, "Set Bar.enabled to False"),
+                  crit=({'Application' : {'log_level' : logging.CRITICAL}},
+                        "set level=CRITICAL"),
+            ))
     
     def init_foo(self):
         self.foo = Foo(config=self.config)
 
     def init_bar(self):
         self.bar = Bar(config=self.config)
 
 
 class TestApplication(TestCase):
 
     def test_basic(self):
         app = MyApp()
         self.assertEquals(app.name, u'myapp')
         self.assertEquals(app.running, False)
         self.assertEquals(app.classes, [MyApp,Bar,Foo])
         self.assertEquals(app.config_file, u'')
 
     def test_config(self):
         app = MyApp()
         app.parse_command_line(["--i=10","--Foo.j=10","--enabled=False","--log-level=50"])
         config = app.config
         self.assertEquals(config.Foo.i, 10)
         self.assertEquals(config.Foo.j, 10)
         self.assertEquals(config.Bar.enabled, False)
         self.assertEquals(config.MyApp.log_level,50)
 
     def test_config_propagation(self):
         app = MyApp()
         app.parse_command_line(["--i=10","--Foo.j=10","--enabled=False","--log-level=50"])
         app.init_foo()
         app.init_bar()
         self.assertEquals(app.foo.i, 10)
         self.assertEquals(app.foo.j, 10)
         self.assertEquals(app.bar.enabled, False)
 
     def test_flags(self):
         app = MyApp()
         app.parse_command_line(["--disable"])
         app.init_bar()
         self.assertEquals(app.bar.enabled, False)
         app.parse_command_line(["--enable"])
         app.init_bar()
         self.assertEquals(app.bar.enabled, True)
     
     def test_aliases(self):
         app = MyApp()
         app.parse_command_line(["--i=5", "--j=10"])
         app.init_foo()
         self.assertEquals(app.foo.i, 5)
         app.init_foo()
         self.assertEquals(app.foo.j, 10)
     
     def test_flag_clobber(self):
         """test that setting flags doesn't clobber existing settings"""
         app = MyApp()
         app.parse_command_line(["--Bar.b=5", "--disable"])
         app.init_bar()
         self.assertEquals(app.bar.enabled, False)
         self.assertEquals(app.bar.b, 5)
         app.parse_command_line(["--enable", "--Bar.b=10"])
         app.init_bar()
         self.assertEquals(app.bar.enabled, True)
         self.assertEquals(app.bar.b, 10)
     
+    def test_flatten_flags(self):
+        cfg = Config()
+        cfg.MyApp.log_level = logging.WARN
+        app = MyApp()
+        app.update_config(cfg)
+        self.assertEquals(app.log_level, logging.WARN)
+        self.assertEquals(app.config.MyApp.log_level, logging.WARN)
+        app.initialize(["--crit"])
+        self.assertEquals(app.log_level, logging.CRITICAL)
+        # this would be app.config.Application.log_level if it failed:
+        self.assertEquals(app.config.MyApp.log_level, logging.CRITICAL)
+    
+    def test_flatten_aliases(self):
+        cfg = Config()
+        cfg.MyApp.log_level = logging.WARN
+        app = MyApp()
+        app.update_config(cfg)
+        self.assertEquals(app.log_level, logging.WARN)
+        self.assertEquals(app.config.MyApp.log_level, logging.WARN)
+        app.initialize(["--log-level", "CRITICAL"])
+        self.assertEquals(app.log_level, logging.CRITICAL)
+        # this would be app.config.Application.log_level if it failed:
+        self.assertEquals(app.config.MyApp.log_level, "CRITICAL")
+    
     def test_extra_args(self):
         app = MyApp()
         app.parse_command_line(["--Bar.b=5", 'extra', "--disable", 'args'])
         app.init_bar()
         self.assertEquals(app.bar.enabled, False)
         self.assertEquals(app.bar.b, 5)
         self.assertEquals(app.extra_args, ['extra', 'args'])
         app = MyApp()
         app.parse_command_line(["--Bar.b=5", '--', 'extra', "--disable", 'args'])
         app.init_bar()
         self.assertEquals(app.bar.enabled, True)
         self.assertEquals(app.bar.b, 5)
         self.assertEquals(app.extra_args, ['extra', '--disable', 'args'])
     
 

IPython/parallel/apps/ipclusterapp.py

@@ -1,528 +1,522 b''
 #!/usr/bin/env python
 # encoding: utf-8
 """
 The ipcluster application.
 
 Authors:
 
 * Brian Granger
 * MinRK
 
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008-2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import errno
 import logging
 import os
 import re
 import signal
 
 from subprocess import check_call, CalledProcessError, PIPE
 import zmq
 from zmq.eventloop import ioloop
 
 from IPython.config.application import Application, boolean_flag
 from IPython.config.loader import Config
 from IPython.core.application import BaseIPythonApplication
 from IPython.core.profiledir import ProfileDir
 from IPython.utils.daemonize import daemonize
 from IPython.utils.importstring import import_item
 from IPython.utils.sysinfo import num_cpus
 from IPython.utils.traitlets import (Int, Unicode, Bool, CFloat, Dict, List,
                                         DottedObjectName)
 
 from IPython.parallel.apps.baseapp import (
     BaseParallelApplication,
     PIDFileError,
     base_flags, base_aliases
 )
 
 
 #-----------------------------------------------------------------------------
 # Module level variables
 #-----------------------------------------------------------------------------
 
 
 default_config_file_name = u'ipcluster_config.py'
 
 
 _description = """Start an IPython cluster for parallel computing.
 
 An IPython cluster consists of 1 controller and 1 or more engines.
 This command automates the startup of these processes using a wide
 range of startup methods (SSH, local processes, PBS, mpiexec,
 Windows HPC Server 2008). To start a cluster with 4 engines on your
 local host simply do 'ipcluster start --n=4'. For more complex usage
 you will typically do 'ipython profile create mycluster --parallel', then edit
 configuration files, followed by 'ipcluster start --profile=mycluster --n=4'.
 """
 
 _main_examples = """
 ipcluster start --n=4 # start a 4 node cluster on localhost
 ipcluster start -h    # show the help string for the start subcmd
 
 ipcluster stop -h     # show the help string for the stop subcmd
 ipcluster engines -h  # show the help string for the engines subcmd
 """
 
 _start_examples = """
 ipython profile create mycluster --parallel # create mycluster profile
 ipcluster start --profile=mycluster --n=4   # start mycluster with 4 nodes
 """
 
 _stop_examples = """
 ipcluster stop --profile=mycluster  # stop a running cluster by profile name
 """
 
 _engines_examples = """
 ipcluster engines --profile=mycluster --n=4  # start 4 engines only
 """
 
 
 # Exit codes for ipcluster
 
 # This will be the exit code if the ipcluster appears to be running because
 # a .pid file exists
 ALREADY_STARTED = 10
 
 
 # This will be the exit code if ipcluster stop is run, but there is not .pid
 # file to be found.
 ALREADY_STOPPED = 11
 
 # This will be the exit code if ipcluster engines is run, but there is not .pid
 # file to be found.
 NO_CLUSTER = 12
 
 
 #-----------------------------------------------------------------------------
 # Main application
 #-----------------------------------------------------------------------------
 start_help = """Start an IPython cluster for parallel computing
 
 Start an ipython cluster by its profile name or cluster
 directory. Cluster directories contain configuration, log and
 security related files and are named using the convention
 'profile_<name>' and should be creating using the 'start'
 subcommand of 'ipcluster'. If your cluster directory is in
 the cwd or the ipython directory, you can simply refer to it
 using its profile name, 'ipcluster start --n=4 --profile=<profile>`,
 otherwise use the 'profile-dir' option.
 """
 stop_help = """Stop a running IPython cluster
 
 Stop a running ipython cluster by its profile name or cluster
 directory. Cluster directories are named using the convention
 'profile_<name>'. If your cluster directory is in
 the cwd or the ipython directory, you can simply refer to it
 using its profile name, 'ipcluster stop --profile=<profile>`, otherwise
 use the '--profile-dir' option.
 """
 engines_help = """Start engines connected to an existing IPython cluster
 
 Start one or more engines to connect to an existing Cluster
 by profile name or cluster directory.
 Cluster directories contain configuration, log and
 security related files and are named using the convention
 'profile_<name>' and should be creating using the 'start'
 subcommand of 'ipcluster'. If your cluster directory is in
 the cwd or the ipython directory, you can simply refer to it
 using its profile name, 'ipcluster engines --n=4 --profile=<profile>`,
 otherwise use the 'profile-dir' option.
 """
 stop_aliases = dict(
     signal='IPClusterStop.signal',
 )
 stop_aliases.update(base_aliases)
 
 class IPClusterStop(BaseParallelApplication):
     name = u'ipcluster'
     description = stop_help
     examples = _stop_examples
     config_file_name = Unicode(default_config_file_name)
 
     signal = Int(signal.SIGINT, config=True,
         help="signal to use for stopping processes.")
 
     aliases = Dict(stop_aliases)
 
     def start(self):
         """Start the app for the stop subcommand."""
         try:
             pid = self.get_pid_from_file()
         except PIDFileError:
             self.log.critical(
                 'Could not read pid file, cluster is probably not running.'
             )
             # Here I exit with a unusual exit status that other processes
             # can watch for to learn how I existed.
             self.remove_pid_file()
             self.exit(ALREADY_STOPPED)
 
         if not self.check_pid(pid):
             self.log.critical(
                 'Cluster [pid=%r] is not running.' % pid
             )
             self.remove_pid_file()
             # Here I exit with a unusual exit status that other processes
             # can watch for to learn how I existed.
             self.exit(ALREADY_STOPPED)
 
         elif os.name=='posix':
             sig = self.signal
             self.log.info(
                 "Stopping cluster [pid=%r] with [signal=%r]" % (pid, sig)
             )
             try:
                 os.kill(pid, sig)
             except OSError:
                 self.log.error("Stopping cluster failed, assuming already dead.",
                     exc_info=True)
                 self.remove_pid_file()
         elif os.name=='nt':
             try:
                 # kill the whole tree
                 p = check_call(['taskkill', '-pid', str(pid), '-t', '-f'], stdout=PIPE,stderr=PIPE)
             except (CalledProcessError, OSError):
                 self.log.error("Stopping cluster failed, assuming already dead.",
                     exc_info=True)
             self.remove_pid_file()
 
 engine_aliases = {}
 engine_aliases.update(base_aliases)
 engine_aliases.update(dict(
     n='IPClusterEngines.n',
     engines = 'IPClusterEngines.engine_launcher_class',
     daemonize = 'IPClusterEngines.daemonize',
 ))
 engine_flags = {}
 engine_flags.update(base_flags)
 
 engine_flags.update(dict(
     daemonize=(
         {'IPClusterEngines' : {'daemonize' : True}},
         """run the cluster into the background (not available on Windows)""",
     )
 ))
 class IPClusterEngines(BaseParallelApplication):
 
     name = u'ipcluster'
     description = engines_help
     examples = _engines_examples
     usage = None
     config_file_name = Unicode(default_config_file_name)
     default_log_level = logging.INFO
     classes = List()
     def _classes_default(self):
         from IPython.parallel.apps import launcher
         launchers = launcher.all_launchers
         eslaunchers = [ l for l in launchers if 'EngineSet' in l.__name__]
         return [ProfileDir]+eslaunchers
 
     n = Int(num_cpus(), config=True,
         help="""The number of engines to start. The default is to use one for each
         CPU on your machine""")
 
     engine_launcher_class = DottedObjectName('LocalEngineSetLauncher',
         config=True,
         help="""The class for launching a set of Engines. Change this value
         to use various batch systems to launch your engines, such as PBS,SGE,MPIExec,etc.
         Each launcher class has its own set of configuration options, for making sure
         it will work in your environment.
 
         You can also write your own launcher, and specify it's absolute import path,
         as in 'mymodule.launcher.FTLEnginesLauncher`.
 
         Examples include:
 
             LocalEngineSetLauncher : start engines locally as subprocesses [default]
             MPIExecEngineSetLauncher : use mpiexec to launch in an MPI environment
             PBSEngineSetLauncher : use PBS (qsub) to submit engines to a batch queue
             SGEEngineSetLauncher : use SGE (qsub) to submit engines to a batch queue
             SSHEngineSetLauncher : use SSH to start the controller
                                 Note that SSH does *not* move the connection files
                                 around, so you will likely have to do this manually
                                 unless the machines are on a shared file system.
             WindowsHPCEngineSetLauncher : use Windows HPC
         """
         )
     daemonize = Bool(False, config=True,
         help="""Daemonize the ipcluster program. This implies --log-to-file.
         Not available on Windows.
         """)
 
     def _daemonize_changed(self, name, old, new):
         if new:
             self.log_to_file = True
 
     aliases = Dict(engine_aliases)
     flags = Dict(engine_flags)
     _stopping = False
 
     def initialize(self, argv=None):
         super(IPClusterEngines, self).initialize(argv)
         self.init_signal()
         self.init_launchers()
 
     def init_launchers(self):
         self.engine_launcher = self.build_launcher(self.engine_launcher_class, 'EngineSet')
         self.engine_launcher.on_stop(lambda r: self.loop.stop())
 
     def init_signal(self):
         # Setup signals
         signal.signal(signal.SIGINT, self.sigint_handler)
 
     def build_launcher(self, clsname, kind=None):
         """import and instantiate a Launcher based on importstring"""
         if '.' not in clsname:
             # not a module, presume it's the raw name in apps.launcher
             if kind and kind not in clsname:
                 # doesn't match necessary full class name, assume it's
                 # just 'PBS' or 'MPIExec' prefix:
                 clsname = clsname + kind + 'Launcher'
             clsname = 'IPython.parallel.apps.launcher.'+clsname
         try:
             klass = import_item(clsname)
         except (ImportError, KeyError):
             self.log.fatal("Could not import launcher class: %r"%clsname)
             self.exit(1)
 
         launcher = klass(
             work_dir=u'.', config=self.config, log=self.log,
             profile_dir=self.profile_dir.location, cluster_id=self.cluster_id,
         )
         return launcher
 
     def start_engines(self):
         self.log.info("Starting %i engines"%self.n)
         self.engine_launcher.start(self.n)
 
     def stop_engines(self):
         self.log.info("Stopping Engines...")
         if self.engine_launcher.running:
             d = self.engine_launcher.stop()
             return d
         else:
             return None
 
     def stop_launchers(self, r=None):
         if not self._stopping:
             self._stopping = True
             self.log.error("IPython cluster: stopping")
             self.stop_engines()
             # Wait a few seconds to let things shut down.
             dc = ioloop.DelayedCallback(self.loop.stop, 4000, self.loop)
             dc.start()
 
     def sigint_handler(self, signum, frame):
         self.log.debug("SIGINT received, stopping launchers...")
         self.stop_launchers()
 
     def start_logging(self):
         # Remove old log files of the controller and engine
         if self.clean_logs:
             log_dir = self.profile_dir.log_dir
             for f in os.listdir(log_dir):
                 if re.match(r'ip(engine|controller)z-\d+\.(log|err|out)',f):
                     os.remove(os.path.join(log_dir, f))
         # This will remove old log files for ipcluster itself
         # super(IPBaseParallelApplication, self).start_logging()
 
     def start(self):
         """Start the app for the engines subcommand."""
         self.log.info("IPython cluster: started")
         # First see if the cluster is already running
 
         # Now log and daemonize
         self.log.info(
             'Starting engines with [daemon=%r]' % self.daemonize
         )
         # TODO: Get daemonize working on Windows or as a Windows Server.
         if self.daemonize:
             if os.name=='posix':
                 daemonize()
 
         dc = ioloop.DelayedCallback(self.start_engines, 0, self.loop)
         dc.start()
         # Now write the new pid file AFTER our new forked pid is active.
         # self.write_pid_file()
         try:
             self.loop.start()
         except KeyboardInterrupt:
             pass
         except zmq.ZMQError as e:
             if e.errno == errno.EINTR:
                 pass
             else:
                 raise
 
 start_aliases = {}
 start_aliases.update(engine_aliases)
 start_aliases.update(dict(
     delay='IPClusterStart.delay',
     controller = 'IPClusterStart.controller_launcher_class',
 ))
 start_aliases['clean-logs'] = 'IPClusterStart.clean_logs'
 
-# set inherited Start keys directly, to ensure command-line args get higher priority
-# than config file options.
-for key,value in start_aliases.items():
-    if value.startswith('IPClusterEngines'):
-        start_aliases[key] = value.replace('IPClusterEngines', 'IPClusterStart')
-
 class IPClusterStart(IPClusterEngines):
 
     name = u'ipcluster'
     description = start_help
     examples = _start_examples
     default_log_level = logging.INFO
     auto_create = Bool(True, config=True,
         help="whether to create the profile_dir if it doesn't exist")
     classes = List()
     def _classes_default(self,):
         from IPython.parallel.apps import launcher
         return [ProfileDir] + [IPClusterEngines] + launcher.all_launchers
 
     clean_logs = Bool(True, config=True,
         help="whether to cleanup old logs before starting")
 
     delay = CFloat(1., config=True,
         help="delay (in s) between starting the controller and the engines")
 
     controller_launcher_class = DottedObjectName('LocalControllerLauncher',
         config=True,
         helep="""The class for launching a Controller. Change this value if you want
         your controller to also be launched by a batch system, such as PBS,SGE,MPIExec,etc.
 
         Each launcher class has its own set of configuration options, for making sure
         it will work in your environment.
 
         Examples include:
 
             LocalControllerLauncher : start engines locally as subprocesses
             MPIExecControllerLauncher : use mpiexec to launch engines in an MPI universe
             PBSControllerLauncher : use PBS (qsub) to submit engines to a batch queue
             SGEControllerLauncher : use SGE (qsub) to submit engines to a batch queue
             SSHControllerLauncher : use SSH to start the controller
             WindowsHPCControllerLauncher : use Windows HPC
         """
         )
     reset = Bool(False, config=True,
         help="Whether to reset config files as part of '--create'."
         )
 
     # flags = Dict(flags)
     aliases = Dict(start_aliases)
 
     def init_launchers(self):
         self.controller_launcher = self.build_launcher(self.controller_launcher_class, 'Controller')
         self.engine_launcher = self.build_launcher(self.engine_launcher_class, 'EngineSet')
         self.controller_launcher.on_stop(self.stop_launchers)
 
     def start_controller(self):
         self.controller_launcher.start()
 
     def stop_controller(self):
         # self.log.info("In stop_controller")
         if self.controller_launcher and self.controller_launcher.running:
             return self.controller_launcher.stop()
 
     def stop_launchers(self, r=None):
         if not self._stopping:
             self.stop_controller()
             super(IPClusterStart, self).stop_launchers()
 
     def start(self):
         """Start the app for the start subcommand."""
         # First see if the cluster is already running
         try:
             pid = self.get_pid_from_file()
         except PIDFileError:
             pass
         else:
             if self.check_pid(pid):
                 self.log.critical(
                     'Cluster is already running with [pid=%s]. '
                     'use "ipcluster stop" to stop the cluster.' % pid
                 )
                 # Here I exit with a unusual exit status that other processes
                 # can watch for to learn how I existed.
                 self.exit(ALREADY_STARTED)
             else:
                 self.remove_pid_file()
 
 
         # Now log and daemonize
         self.log.info(
             'Starting ipcluster with [daemon=%r]' % self.daemonize
         )
         # TODO: Get daemonize working on Windows or as a Windows Server.
         if self.daemonize:
             if os.name=='posix':
                 daemonize()
 
         dc = ioloop.DelayedCallback(self.start_controller, 0, self.loop)
         dc.start()
         dc = ioloop.DelayedCallback(self.start_engines, 1000*self.delay, self.loop)
         dc.start()
         # Now write the new pid file AFTER our new forked pid is active.
         self.write_pid_file()
         try:
             self.loop.start()
         except KeyboardInterrupt:
             pass
         except zmq.ZMQError as e:
             if e.errno == errno.EINTR:
                 pass
             else:
                 raise
         finally:
             self.remove_pid_file()
 
 base='IPython.parallel.apps.ipclusterapp.IPCluster'
 
 class IPClusterApp(Application):
     name = u'ipcluster'
     description = _description
     examples = _main_examples
 
     subcommands = {
                 'start' : (base+'Start', start_help),
                 'stop' : (base+'Stop', stop_help),
                 'engines' : (base+'Engines', engines_help),
     }
 
     # no aliases or flags for parent App
     aliases = Dict()
     flags = Dict()
 
     def start(self):
         if self.subapp is None:
             print "No subcommand specified. Must specify one of: %s"%(self.subcommands.keys())
             print
             self.print_description()
             self.print_subcommands()
             self.exit(1)
         else:
             return self.subapp.start()
 
 def launch_new_instance():
     """Create and run the IPython cluster."""
     app = IPClusterApp.instance()
     app.initialize()
     app.start()
 
 
 if __name__ == '__main__':
     launch_new_instance()