The ipengine script has been refactored to use the new config system.

It is also now a formal Application: IPython.kernel.ipengineapp

IPython/kernel/ipengineapp.py

@@ -0,0 +1,257 b''
+#!/usr/bin/env python
+# encoding: utf-8
+"""
+The IPython controller application
+"""
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2009  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+import os
+import sys
+
+from twisted.application import service
+from twisted.internet import reactor
+from twisted.python import log
+
+from IPython.config.loader import NoConfigDefault
+
+from IPython.core.application import (
+    ApplicationWithDir, 
+    AppWithDirArgParseConfigLoader
+)
+from IPython.core import release
+
+from IPython.utils.importstring import import_item
+
+from IPython.kernel.engineservice import EngineService
+from IPython.kernel.fcutil import Tub
+from IPython.kernel.engineconnector import EngineConnector
+
+#-----------------------------------------------------------------------------
+# The main application
+#-----------------------------------------------------------------------------
+
+
+cl_args = (
+    # Controller config
+    (('--furl-file',), dict(
+        type=str, dest='Global.furl_file', default=NoConfigDefault,
+        help='The full location of the file containing the FURL of the '
+        'controller. If this is not given, the FURL file must be in the '
+        'security directory of the cluster directory.  This location is '
+        'resolved using the --profile and --app-dir options.',
+        metavar='Global.furl_file')
+    ),
+    # MPI
+    (('--mpi',), dict(
+        type=str, dest='MPI.use', default=NoConfigDefault,
+        help='How to enable MPI (mpi4py, pytrilinos, or empty string to disable).',
+        metavar='MPI.use')
+    ),
+    # Global config
+    (('--log-to-file',), dict(
+        action='store_true', dest='Global.log_to_file', default=NoConfigDefault,
+        help='Log to a file in the log directory (default is stdout)')
+    )
+)
+
+
+class IPEngineAppCLConfigLoader(AppWithDirArgParseConfigLoader):
+
+    arguments = cl_args
+
+
+mpi4py_init = """from mpi4py import MPI as mpi
+mpi.size = mpi.COMM_WORLD.Get_size()
+mpi.rank = mpi.COMM_WORLD.Get_rank()
+"""
+
+pytrilinos_init = """from PyTrilinos import Epetra
+class SimpleStruct:
+pass
+mpi = SimpleStruct()
+mpi.rank = 0
+mpi.size = 0
+"""
+
+
+default_config_file_name = 'ipengine_config.py'
+
+
+class IPEngineApp(ApplicationWithDir):
+
+    name = 'ipengine'
+    app_dir_basename = 'cluster'
+    description = 'Start the IPython engine for parallel computing.'
+    config_file_name = default_config_file_name
+
+    def create_default_config(self):
+        super(IPEngineApp, self).create_default_config()
+
+        # Global config attributes
+        self.default_config.Global.log_to_file = False
+        self.default_config.Global.exec_lines = []
+        # The log and security dir names must match that of the controller
+        self.default_config.Global.log_dir_name = 'log'
+        self.default_config.Global.security_dir_name = 'security'
+        self.default_config.Global.shell_class = 'IPython.kernel.core.interpreter.Interpreter'
+
+        # Configuration related to the controller
+        # This must match the filename (path not included) that the controller
+        # used for the FURL file.
+        self.default_config.Global.furl_file_name = 'ipcontroller-engine.furl'
+        # If given, this is the actual location of the controller's FURL file.
+        # If not, this is computed using the profile, app_dir and furl_file_name
+        self.default_config.Global.furl_file = ''
+
+        # MPI related config attributes
+        self.default_config.MPI.use = ''
+        self.default_config.MPI.mpi4py = mpi4py_init
+        self.default_config.MPI.pytrilinos = pytrilinos_init
+
+    def create_command_line_config(self):
+        """Create and return a command line config loader."""
+        return IPEngineAppCLConfigLoader(
+            description=self.description, 
+            version=release.version
+        )
+
+    def post_load_command_line_config(self):
+        pass
+
+    def pre_construct(self):
+        config = self.master_config
+        # Now set the security_dir and log_dir and create them.  We use
+        # the names an construct the absolute paths.
+        security_dir = os.path.join(config.Global.app_dir,
+                                    config.Global.security_dir_name)
+        log_dir = os.path.join(config.Global.app_dir,
+                               config.Global.log_dir_name)
+        if not os.path.isdir(security_dir):
+            os.mkdir(security_dir, 0700)
+        else:
+            os.chmod(security_dir, 0700)
+        if not os.path.isdir(log_dir):
+            os.mkdir(log_dir, 0777)
+
+        self.security_dir = config.Global.security_dir = security_dir
+        self.log_dir = config.Global.log_dir = log_dir
+        self.log.info("Log directory set to: %s" % self.log_dir)
+        self.log.info("Security directory set to: %s" % self.security_dir)
+
+        self.find_cont_furl_file()
+
+    def find_cont_furl_file(self):
+        config = self.master_config
+        # Find the actual controller FURL file
+        if os.path.isfile(config.Global.furl_file):
+            return
+        else:
+            # We should know what the app dir is
+            try_this = os.path.join(
+                config.Global.app_dir, 
+                config.Global.security_dir,
+                config.Global.furl_file_name
+            )
+            if os.path.isfile(try_this):
+                config.Global.furl_file = try_this
+                return
+            else:
+                self.log.critical("Could not find a valid controller FURL file.")
+                self.abort()
+
+    def construct(self):
+        # I am a little hesitant to put these into InteractiveShell itself.
+        # But that might be the place for them
+        sys.path.insert(0, '')
+
+        self.start_mpi()
+        self.start_logging()
+
+        # Create the underlying shell class and EngineService
+        shell_class = import_item(self.master_config.Global.shell_class)
+        self.engine_service = EngineService(shell_class, mpi=mpi)
+
+        self.exec_lines()
+
+        # Create the service hierarchy
+        self.main_service = service.MultiService()
+        self.engine_service.setServiceParent(self.main_service)
+        self.tub_service = Tub()
+        self.tub_service.setServiceParent(self.main_service)
+        # This needs to be called before the connection is initiated
+        self.main_service.startService()
+
+        # This initiates the connection to the controller and calls
+        # register_engine to tell the controller we are ready to do work
+        self.engine_connector = EngineConnector(self.tub_service)
+
+        log.msg("Using furl file: %s" % self.master_config.Global.furl_file)
+
+        reactor.callWhenRunning(self.call_connect)
+
+    def call_connect(self):
+        d = self.engine_connector.connect_to_controller(
+            self.engine_service, 
+            self.master_config.Global.furl_file
+        )
+
+        def handle_error(f):
+            # If this print statement is replaced by a log.err(f) I get
+            # an unhandled error, which makes no sense.  I shouldn't have
+            # to use a print statement here.  My only thought is that
+            # at the beginning of the process the logging is still starting up
+            print "Error connecting to controller:", f.getErrorMessage()
+            reactor.callLater(0.1, reactor.stop)
+
+        d.addErrback(handle_error)
+
+    def start_mpi(self):
+        global mpi
+        mpikey = self.master_config.MPI.use
+        mpi_import_statement = self.master_config.MPI.get(mpikey, None)
+        if mpi_import_statement is not None:
+            try:
+                self.log.info("Initializing MPI:")
+                self.log.info(mpi_import_statement)
+                exec mpi_import_statement in globals()
+            except:
+                mpi = None
+        else:
+            mpi = None
+
+    def start_logging(self):
+        if self.master_config.Global.log_to_file:
+            log_filename = self.name + '-' + str(os.getpid()) + '.log'
+            logfile = os.path.join(self.log_dir, log_filename)
+            open_log_file = open(logfile, 'w')
+        else:
+            open_log_file = sys.stdout
+        log.startLogging(open_log_file)
+
+    def exec_lines(self):
+        for line in self.master_config.Global.exec_lines:
+            try:
+                log.msg("Executing statement: '%s'" % line)
+                self.engine_service.execute(line)
+            except:
+                log.msg("Error executing statement: %s" % line)
+
+    def start_app(self):
+        # Start the controller service and set things running
+        reactor.run()
+
+
+def launch_new_instance():
+    """Create and run the IPython controller"""
+    app = IPEngineApp()
+    app.start()

IPython/config/default/ipengine_config.py

@@ -1,6 +1,6 b''
 c = get_config()
 
-c.MPI.default = 'mpi4py'
+c.MPI.use = 'mpi4py'
 
 c.MPI.mpi4py = """from mpi4py import MPI as mpi
 mpi.size = mpi.COMM_WORLD.Get_size()
@@ -15,5 +15,11 b' mpi.rank = 0'
 mpi.size = 0
 """
 
-c.Global.logfile = ''
-c.Global.furl_file = 'ipcontroller-engine.furl'
+c.Global.log_to_file = False
+c.Global.exec_lines = []
+c.Global.log_dir_name = 'log'
+c.Global.security_dir_name = 'security'
+c.Global.shell_class = 'IPython.kernel.core.interpreter.Interpreter'
+self.default_config.Global.furl_file_name = 'ipcontroller-engine.furl'
+self.default_config.Global.furl_file = ''
+

IPython/core/application.py

@@ -61,7 +61,8 b' class BaseAppArgParseConfigLoader(ArgParseConfigLoader):'
         self.parser.add_argument('-log_level', '--log-level',
             dest="Global.log_level",type=int,
             help='Set the log level (0,10,20,30,40,50).  Default is 30.',
-            default=NoConfigDefault)
+            default=NoConfigDefault,
+            metavar='Global.log_level')
         self.parser.add_argument('-config_file', '--config-file',
             dest='Global.config_file',type=str,
             help='Set the config file name to override default.',
@@ -251,7 +252,7 b' class Application(object):'
         ``CONFIG_FILE`` config variable is set to the resolved config file
         location.  If not successful, an empty config is used.
         """
-        self.log.debug("Attempting to load config file: <%s>" % self.config_file_name)
+        self.log.debug("Attempting to load config file: %s" % self.config_file_name)
         loader = PyFileConfigLoader(self.config_file_name,
                                     path=self.config_file_paths)
         try:
@@ -260,11 +261,11 b' class Application(object):'
         except IOError:
             # Only warn if the default config file was NOT being used.
             if not self.config_file_name==self.default_config_file_name:
-                self.log.warn("Config file not found, skipping: <%s>" % \
+                self.log.warn("Config file not found, skipping: %s" % \
                                self.config_file_name, exc_info=True)
             self.file_config = Config()
         except:
-            self.log.warn("Error loading config file: <%s>" % \
+            self.log.warn("Error loading config file: %s" % \
                            self.config_file_name, exc_info=True)
             self.file_config = Config()
 
@@ -284,7 +285,7 b' class Application(object):'
 
     def log_file_config(self):
         if hasattr(self.file_config.Global, 'config_file'):
-            self.log.debug("Config file loaded: <%s>" % self.file_config.Global.config_file)
+            self.log.debug("Config file loaded: %s" % self.file_config.Global.config_file)
             self.log.debug(repr(self.file_config))
 
     def merge_configs(self):
@@ -441,6 +442,7 b' class ApplicationWithDir(Application):'
         # priority, this will always end up in the master_config.
         self.default_config.Global.app_dir = self.app_dir
         self.command_line_config.Global.app_dir = self.app_dir
+        self.log.info("Application directory set to: %s" % self.app_dir)
 
     def create_app_dir(self):
         """Make sure that the app dir exists."""

IPython/kernel/ipcontrollerapp.py

@@ -171,7 +171,6 b' class IPControllerApp(ApplicationWithDir):'
     app_dir_basename = 'cluster'
     description = 'Start the IPython controller for parallel computing.'
     config_file_name = default_config_file_name
-    default_log_level = logging.WARN
 
     def create_default_config(self):
         super(IPControllerApp, self).create_default_config()
@@ -205,12 +204,13 b' class IPControllerApp(ApplicationWithDir):'
             del self.command_line_config.Global.secure
 
     def pre_construct(self):
+        config = self.master_config
         # Now set the security_dir and log_dir and create them.  We use
         # the names an construct the absolute paths.
-        security_dir = os.path.join(self.master_config.Global.app_dir,
-                                    self.master_config.Global.security_dir_name)
-        log_dir = os.path.join(self.master_config.Global.app_dir, 
-                               self.master_config.Global.log_dir_name)
+        security_dir = os.path.join(config.Global.app_dir,
+                                    config.Global.security_dir_name)
+        log_dir = os.path.join(config.Global.app_dir, 
+                               config.Global.log_dir_name)
         if not os.path.isdir(security_dir):
             os.mkdir(security_dir, 0700)
         else:
@@ -218,8 +218,10 b' class IPControllerApp(ApplicationWithDir):'
         if not os.path.isdir(log_dir):
             os.mkdir(log_dir, 0777)
 
-        self.security_dir = self.master_config.Global.security_dir = security_dir
-        self.log_dir = self.master_config.Global.log_dir = log_dir
+        self.security_dir = config.Global.security_dir = security_dir
+        self.log_dir = config.Global.log_dir = log_dir
+        self.log.info("Log directory set to: %s" % self.log_dir)
+        self.log.info("Security directory set to: %s" % self.security_dir)
 
     def construct(self):
         # I am a little hesitant to put these into InteractiveShell itself.

IPython/kernel/scripts/ipengine

@@ -1,20 +1,20 b''
 #!/usr/bin/env python
 # encoding: utf-8
 
-__docformat__ = "restructuredtext en"
-
-#-------------------------------------------------------------------------------
-#  Copyright (C) 2008  The IPython Development Team
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2009  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
-#-------------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
 
-#-------------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
 # Imports
-#-------------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
+
+
+from IPython.kernel.ipengineapp import launch_new_instance
+
+launch_new_instance()
 
-if __name__ == '__main__':
-	from IPython.kernel.scripts import ipengine
-	ipengine.main()
 

setup.py

@@ -170,7 +170,7 b" if 'setuptools' in sys.modules:"
             'ipython = IPython.core.ipapp:launch_new_instance',
             'pycolor = IPython.utils.PyColorize:main',
             'ipcontroller = IPython.kernel.ipcontrollerapp:launch_new_instance',
-            'ipengine = IPython.kernel.scripts.ipengine:main',
+            'ipengine = IPython.kernel.ipengineapp:launch_new_instance',
             'ipcluster = IPython.kernel.scripts.ipcluster:main',
             'ipythonx = IPython.frontend.wx.ipythonx:main',
             'iptest = IPython.testing.iptest:main',