Lots more work on the kernel scripts.

IPython/config/default/ipengine_config.py

@@ -1,32 +1,24 b''
 c = get_config()
 
-c.MPI.use = ''
+# c.Global.log_to_file = False
+# c.Global.exec_lines = ['import numpy']
+# c.Global.log_dir_name = 'log'
+# c.Global.security_dir_name = 'security'
+# c.Global.log_level = 10
+# c.Global.shell_class = 'IPython.kernel.core.interpreter.Interpreter'
+# c.Global.furl_file_name = 'ipcontroller-engine.furl'
+# c.Global.furl_file = ''
+
+# c.MPI.use = ''
+# c.MPI.mpi4py = """from mpi4py import MPI as mpi
+# mpi.size = mpi.COMM_WORLD.Get_size()
+# mpi.rank = mpi.COMM_WORLD.Get_rank()
+# """
+# c.MPI.pytrilinos = """from PyTrilinos import Epetra
+# class SimpleStruct:
+# pass
+# mpi = SimpleStruct()
+# mpi.rank = 0
+# mpi.size = 0
+# """
 
-c.MPI.mpi4py = """from mpi4py import MPI as mpi
-mpi.size = mpi.COMM_WORLD.Get_size()
-mpi.rank = mpi.COMM_WORLD.Get_rank()
-"""
-
-c.MPI.pytrilinos = """from PyTrilinos import Epetra
-class SimpleStruct:
-pass
-mpi = SimpleStruct()
-mpi.rank = 0
-mpi.size = 0
-"""
-
-c.Global.log_to_file = False
-
-c.Global.exec_lines = ['import numpy']
-
-c.Global.log_dir_name = 'log'
-
-c.Global.security_dir_name = 'security'
-
-c.Global.log_level = 10
-
-c.Global.shell_class = 'IPython.kernel.core.interpreter.Interpreter'
-
-c.Global.furl_file_name = 'ipcontroller-engine.furl'
-
-c.Global.furl_file = ''

IPython/core/application.py

@@ -86,6 +86,7 b' class Application(object):'
     default_log_level = logging.WARN
 
     def __init__(self):
+        self._exiting = False
         self.init_logger()
         # Track the default and actual separately because some messages are
         # only printed if we aren't using the default.
@@ -119,6 +120,7 b' class Application(object):'
         self.attempt(self.post_load_command_line_config)
         self.log_command_line_config()
         self.attempt(self.find_ipythondir)
+        self.attempt(self.find_resources)
         self.attempt(self.find_config_file_name)
         self.attempt(self.find_config_file_paths)
         self.attempt(self.pre_load_file_config)
@@ -211,6 +213,15 b' class Application(object):'
             os.makedirs(self.ipythondir, mode=0777)
         self.log.debug("IPYTHONDIR set to: %s" % self.ipythondir)
 
+    def find_resources(self):
+        """Find other resources that need to be in place.
+
+        Things like cluster directories need to be in place to find the
+        config file.  These happen right after the IPython directory has
+        been set.
+        """
+        pass
+
     def find_config_file_name(self):
         """Find the config file name for this application.
 
@@ -325,18 +336,26 b' class Application(object):'
 
     def abort(self):
         """Abort the starting of the application."""
+        if self._exiting:
+            pass
+        else:
         self.log.critical("Aborting application: %s" % self.name, exc_info=True)
+            self._exiting = True
         sys.exit(1)
 
     def exit(self):
-        self.log.critical("Aborting application: %s" % self.name)
+        if self._exiting:
+            pass
+        else:
+            self.log.debug("Exiting application: %s" % self.name)
+            self._exiting = True
         sys.exit(1)
 
     def attempt(self, func, action='abort'):
         try:
             func()
         except SystemExit:
-            self.exit()
+            raise
         except:
             if action == 'abort':
                 self.abort()

IPython/kernel/clusterdir.py

@@ -23,13 +23,17 b' from IPython.config.loader import PyFileConfigLoader'
 from IPython.core.application import Application
 from IPython.core.component import Component
 from IPython.config.loader import ArgParseConfigLoader, NoConfigDefault
-from IPython.utils.traitlets import Unicode
+from IPython.utils.traitlets import Unicode, Bool
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 
+class ClusterDirError(Exception):
+    pass
+
+
 class ClusterDir(Component):
     """An object to manage the cluster directory and its resources.
 
@@ -117,28 +121,68 b' class ClusterDir(Component):'
             self.copy_config_file(f, path=path, overwrite=overwrite)
 
     @classmethod
-    def find_cluster_dir_by_profile(cls, path, profile='default'):
-        """Find/create a cluster dir by profile name and return its ClusterDir.
+    def create_cluster_dir(csl, cluster_dir):
+        """Create a new cluster directory given a full path.
 
-        This will create the cluster directory if it doesn't exist.
+        Parameters
+        ----------
+        cluster_dir : str
+            The full path to the cluster directory.  If it does exist, it will
+            be used.  If not, it will be created.
+        """
+        return ClusterDir(cluster_dir)
+
+    @classmethod
+    def create_cluster_dir_by_profile(cls, path, profile='default'):
+        """Create a cluster dir by profile name and path.
 
         Parameters
         ----------
-        path : unicode or str
-            The directory path to look for the cluster directory in.
+        path : str
+            The path (directory) to put the cluster directory in.
+        profile : str
+            The name of the profile.  The name of the cluster directory will
+            be "cluster_<profile>".
+        """
+        if not os.path.isdir(path):
+            raise ClusterDirError('Directory not found: %s' % path)
+        cluster_dir = os.path.join(path, 'cluster_' + profile)
+        return ClusterDir(cluster_dir)
+
+    @classmethod
+    def find_cluster_dir_by_profile(cls, ipythondir, profile='default'):
+        """Find an existing cluster dir by profile name, return its ClusterDir.
+
+        This searches through a sequence of paths for a cluster dir.  If it
+        is not found, a :class:`ClusterDirError` exception will be raised.
+
+        The search path algorithm is:
+        1. ``os.getcwd()``
+        2. ``ipythondir``
+        3. The directories found in the ":" separated 
+           :env:`IPCLUSTERDIR_PATH` environment variable.
+
+        Parameters
+        ----------
+        ipythondir : unicode or str
+            The IPython directory to use.
         profile : unicode or str
             The name of the profile.  The name of the cluster directory
             will be "cluster_<profile>".
         """
         dirname = 'cluster_' + profile
-        cluster_dir = os.path.join(os.getcwd(), dirname)
+        cluster_dir_paths = os.environ.get('IPCLUSTERDIR_PATH','')
+        if cluster_dir_paths:
+            cluster_dir_paths = cluster_dir_paths.split(':')
+        else:
+            cluster_dir_paths = []
+        paths = [os.getcwd(), ipythondir] + cluster_dir_paths
+        for p in paths:
+            cluster_dir = os.path.join(p, dirname)
         if os.path.isdir(cluster_dir):
             return ClusterDir(cluster_dir)
         else:
-            if not os.path.isdir(path):
-                raise IOError("Directory doesn't exist: %s" % path)
-            cluster_dir = os.path.join(path, dirname)
-            return ClusterDir(cluster_dir)
+            raise ClusterDirError('Cluster directory not found in paths: %s' % dirname)
 
     @classmethod
     def find_cluster_dir(cls, cluster_dir):
@@ -153,6 +197,8 b' class ClusterDir(Component):'
             :func:`os.path.expandvars` and :func:`os.path.expanduser`.
         """
         cluster_dir = os.path.expandvars(os.path.expanduser(cluster_dir))
+        if not os.path.isdir(cluster_dir):
+            raise ClusterDirError('Cluster directory not found: %s' % cluster_dir)
         return ClusterDir(cluster_dir)
 
 
@@ -176,7 +222,8 b' class AppWithClusterDirArgParseConfigLoader(ArgParseConfigLoader):'
         self.parser.add_argument('-log_level', '--log-level',
             dest="Global.log_level",type=int,
             help='Set the log level (0,10,20,30,40,50).  Default is 30.',
-            default=NoConfigDefault)
+            default=NoConfigDefault,
+            metavar="Global.log_level")
         self.parser.add_argument('-cluster_dir', '--cluster-dir',
             dest='Global.cluster_dir',type=str,
             help='Set the cluster dir. This overrides the logic used by the '
@@ -204,6 +251,8 b' class ApplicationWithClusterDir(Application):'
     dir and named the value of the ``config_file_name`` class attribute.
     """
 
+    auto_create_cluster_dir = True
+
     def create_default_config(self):
         super(ApplicationWithClusterDir, self).create_default_config()
         self.default_config.Global.profile = 'default'
@@ -216,40 +265,68 b' class ApplicationWithClusterDir(Application):'
             version=release.version
         )
 
-    def find_config_file_name(self):
-        """Find the config file name for this application."""
-        # For this type of Application it should be set as a class attribute.
-        if not hasattr(self, 'config_file_name'):
-            self.log.critical("No config filename found")
-
-    def find_config_file_paths(self):
-        """This resolves the cluster directory and sets ``config_file_paths``.
+    def find_resources(self):
+        """This resolves the cluster directory.
 
-        This does the following:
-        * Create the :class:`ClusterDir` object for the application.
-        * Set the ``cluster_dir`` attribute of the application and config
+        This tries to find the cluster directory and if successful, it will
+        have done:
+        * Sets ``self.cluster_dir_obj`` to the :class:`ClusterDir` object for 
+          the application.
+        * Sets ``self.cluster_dir`` attribute of the application and config
           objects.
-        * Set ``config_file_paths`` to point to the cluster directory.
+
+        The algorithm used for this is as follows:
+        1. Try ``Global.cluster_dir``.
+        2. Try using ``Global.profile``.
+        3. If both of these fail and ``self.auto_create_cluster_dir`` is
+           ``True``, then create the new cluster dir in the IPython directory.
+        4. If all fails, then raise :class:`ClusterDirError`.
         """
 
-        # Create the ClusterDir object for managing everything
         try:
             cluster_dir = self.command_line_config.Global.cluster_dir
         except AttributeError:
             cluster_dir = self.default_config.Global.cluster_dir
         cluster_dir = os.path.expandvars(os.path.expanduser(cluster_dir))
-        if cluster_dir:
-            # Just use cluster_dir
+        try:
             self.cluster_dir_obj = ClusterDir.find_cluster_dir(cluster_dir)
+        except ClusterDirError:
+            pass
         else:
-            # Then look for a profile
+            self.log.info('Using existing cluster dir: %s' % \
+                self.cluster_dir_obj.location
+            )
+            self.finish_cluster_dir()
+            return
+
             try:
                 self.profile = self.command_line_config.Global.profile
             except AttributeError:
                 self.profile = self.default_config.Global.profile
+        try:
             self.cluster_dir_obj = ClusterDir.find_cluster_dir_by_profile(
                 self.ipythondir, self.profile)
+        except ClusterDirError:
+            pass
+        else:
+            self.log.info('Using existing cluster dir: %s' % \
+                self.cluster_dir_obj.location
+            )
+            self.finish_cluster_dir()
+            return
 
+        if self.auto_create_cluster_dir:
+            self.cluster_dir_obj = ClusterDir.create_cluster_dir_by_profile(
+                self.ipythondir, self.profile
+            )
+            self.log.info('Creating new cluster dir: %s' % \
+                self.cluster_dir_obj.location
+            )
+            self.finish_cluster_dir()
+        else:
+            raise ClusterDirError('Could not find a valid cluster directory.')
+
+    def finish_cluster_dir(self):
         # Set the cluster directory
         self.cluster_dir = self.cluster_dir_obj.location
         
@@ -261,3 +338,15 b' class ApplicationWithClusterDir(Application):'
 
         # Set the search path to the cluster directory
         self.config_file_paths = (self.cluster_dir,)
+
+    def find_config_file_name(self):
+        """Find the config file name for this application."""
+        # For this type of Application it should be set as a class attribute.
+        if not hasattr(self, 'config_file_name'):
+            self.log.critical("No config filename found")
+
+    def find_config_file_paths(self):
+        # Set the search path to the cluster directory
+        self.config_file_paths = (self.cluster_dir,)
+
+

IPython/kernel/ipcontrollerapp.py

@@ -178,6 +178,7 b' class IPControllerApp(ApplicationWithClusterDir):'
     name = 'ipcontroller'
     description = 'Start the IPython controller for parallel computing.'
     config_file_name = default_config_file_name
+    auto_create_cluster_dir = True
 
     def create_default_config(self):
         super(IPControllerApp, self).create_default_config()

IPython/kernel/ipengineapp.py

@@ -92,6 +92,7 b' class IPEngineApp(ApplicationWithClusterDir):'
     name = 'ipengine'
     description = 'Start the IPython engine for parallel computing.'
     config_file_name = default_config_file_name
+    auto_create_cluster_dir = True
 
     def create_default_config(self):
         super(IPEngineApp, self).create_default_config()

IPython/kernel/twistedutil.py

@@ -254,3 +254,12 b' def sleep_deferred(seconds):'
     d = defer.Deferred()
     reactor.callLater(seconds, d.callback, seconds)
     return d
+
+
+def make_deferred(func):
+    """A decorator that calls a function with :func`maybeDeferred`."""
+
+    def _wrapper(*args, **kwargs):
+        return defer.maybeDeferred(func, *args, **kwargs)
+
+    return _wrapper
  No newline at end of file