The ipengine script has been refactored to use the new config system.

It is also now a formal Application: IPython.kernel.ipengineapp

IPython/kernel/ipengineapp.py

@@ -0,0 +1,257 b''
+#!/usr/bin/env python
+# encoding: utf-8
+"""
+The IPython controller application
+"""
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2009  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+import os
+import sys
+
+from twisted.application import service
+from twisted.internet import reactor
+from twisted.python import log
+
+from IPython.config.loader import NoConfigDefault
+
+from IPython.core.application import (
+    ApplicationWithDir, 
+    AppWithDirArgParseConfigLoader
+)
+from IPython.core import release
+
+from IPython.utils.importstring import import_item
+
+from IPython.kernel.engineservice import EngineService
+from IPython.kernel.fcutil import Tub
+from IPython.kernel.engineconnector import EngineConnector
+
+#-----------------------------------------------------------------------------
+# The main application
+#-----------------------------------------------------------------------------
+
+
+cl_args = (
+    # Controller config
+    (('--furl-file',), dict(
+        type=str, dest='Global.furl_file', default=NoConfigDefault,
+        help='The full location of the file containing the FURL of the '
+        'controller. If this is not given, the FURL file must be in the '
+        'security directory of the cluster directory.  This location is '
+        'resolved using the --profile and --app-dir options.',
+        metavar='Global.furl_file')
+    ),
+    # MPI
+    (('--mpi',), dict(
+        type=str, dest='MPI.use', default=NoConfigDefault,
+        help='How to enable MPI (mpi4py, pytrilinos, or empty string to disable).',
+        metavar='MPI.use')
+    ),
+    # Global config
+    (('--log-to-file',), dict(
+        action='store_true', dest='Global.log_to_file', default=NoConfigDefault,
+        help='Log to a file in the log directory (default is stdout)')
+    )
+)
+
+
+class IPEngineAppCLConfigLoader(AppWithDirArgParseConfigLoader):
+
+    arguments = cl_args
+
+
+mpi4py_init = """from mpi4py import MPI as mpi
+mpi.size = mpi.COMM_WORLD.Get_size()
+mpi.rank = mpi.COMM_WORLD.Get_rank()
+"""
+
+pytrilinos_init = """from PyTrilinos import Epetra
+class SimpleStruct:
+pass
+mpi = SimpleStruct()
+mpi.rank = 0
+mpi.size = 0
+"""
+
+
+default_config_file_name = 'ipengine_config.py'
+
+
+class IPEngineApp(ApplicationWithDir):
+
+    name = 'ipengine'
+    app_dir_basename = 'cluster'
+    description = 'Start the IPython engine for parallel computing.'
+    config_file_name = default_config_file_name
+
+    def create_default_config(self):
+        super(IPEngineApp, self).create_default_config()
+
+        # Global config attributes
+        self.default_config.Global.log_to_file = False
+        self.default_config.Global.exec_lines = []
+        # The log and security dir names must match that of the controller
+        self.default_config.Global.log_dir_name = 'log'
+        self.default_config.Global.security_dir_name = 'security'
+        self.default_config.Global.shell_class = 'IPython.kernel.core.interpreter.Interpreter'
+
+        # Configuration related to the controller
+        # This must match the filename (path not included) that the controller
+        # used for the FURL file.
+        self.default_config.Global.furl_file_name = 'ipcontroller-engine.furl'
+        # If given, this is the actual location of the controller's FURL file.
+        # If not, this is computed using the profile, app_dir and furl_file_name
+        self.default_config.Global.furl_file = ''
+
+        # MPI related config attributes
+        self.default_config.MPI.use = ''
+        self.default_config.MPI.mpi4py = mpi4py_init
+        self.default_config.MPI.pytrilinos = pytrilinos_init
+
+    def create_command_line_config(self):
+        """Create and return a command line config loader."""
+        return IPEngineAppCLConfigLoader(
+            description=self.description, 
+            version=release.version
+        )
+
+    def post_load_command_line_config(self):
+        pass
+
+    def pre_construct(self):
+        config = self.master_config
+        # Now set the security_dir and log_dir and create them.  We use
+        # the names an construct the absolute paths.
+        security_dir = os.path.join(config.Global.app_dir,
+                                    config.Global.security_dir_name)
+        log_dir = os.path.join(config.Global.app_dir,
+                               config.Global.log_dir_name)
+        if not os.path.isdir(security_dir):
+            os.mkdir(security_dir, 0700)
+        else:
+            os.chmod(security_dir, 0700)
+        if not os.path.isdir(log_dir):
+            os.mkdir(log_dir, 0777)
+
+        self.security_dir = config.Global.security_dir = security_dir
+        self.log_dir = config.Global.log_dir = log_dir
+        self.log.info("Log directory set to: %s" % self.log_dir)
+        self.log.info("Security directory set to: %s" % self.security_dir)
+
+        self.find_cont_furl_file()
+
+    def find_cont_furl_file(self):
+        config = self.master_config
+        # Find the actual controller FURL file
+        if os.path.isfile(config.Global.furl_file):
+            return
+        else:
+            # We should know what the app dir is
+            try_this = os.path.join(
+                config.Global.app_dir, 
+                config.Global.security_dir,
+                config.Global.furl_file_name
+            )
+            if os.path.isfile(try_this):
+                config.Global.furl_file = try_this
+                return
+            else:
+                self.log.critical("Could not find a valid controller FURL file.")
+                self.abort()
+
+    def construct(self):
+        # I am a little hesitant to put these into InteractiveShell itself.
+        # But that might be the place for them
+        sys.path.insert(0, '')
+
+        self.start_mpi()
+        self.start_logging()
+
+        # Create the underlying shell class and EngineService
+        shell_class = import_item(self.master_config.Global.shell_class)
+        self.engine_service = EngineService(shell_class, mpi=mpi)
+
+        self.exec_lines()
+
+        # Create the service hierarchy
+        self.main_service = service.MultiService()
+        self.engine_service.setServiceParent(self.main_service)
+        self.tub_service = Tub()
+        self.tub_service.setServiceParent(self.main_service)
+        # This needs to be called before the connection is initiated
+        self.main_service.startService()
+
+        # This initiates the connection to the controller and calls
+        # register_engine to tell the controller we are ready to do work
+        self.engine_connector = EngineConnector(self.tub_service)
+
+        log.msg("Using furl file: %s" % self.master_config.Global.furl_file)
+
+        reactor.callWhenRunning(self.call_connect)
+
+    def call_connect(self):
+        d = self.engine_connector.connect_to_controller(
+            self.engine_service, 
+            self.master_config.Global.furl_file
+        )
+
+        def handle_error(f):
+            # If this print statement is replaced by a log.err(f) I get
+            # an unhandled error, which makes no sense.  I shouldn't have
+            # to use a print statement here.  My only thought is that
+            # at the beginning of the process the logging is still starting up
+            print "Error connecting to controller:", f.getErrorMessage()
+            reactor.callLater(0.1, reactor.stop)
+
+        d.addErrback(handle_error)
+
+    def start_mpi(self):
+        global mpi
+        mpikey = self.master_config.MPI.use
+        mpi_import_statement = self.master_config.MPI.get(mpikey, None)
+        if mpi_import_statement is not None:
+            try:
+                self.log.info("Initializing MPI:")
+                self.log.info(mpi_import_statement)
+                exec mpi_import_statement in globals()
+            except:
+                mpi = None
+        else:
+            mpi = None
+
+    def start_logging(self):
+        if self.master_config.Global.log_to_file:
+            log_filename = self.name + '-' + str(os.getpid()) + '.log'
+            logfile = os.path.join(self.log_dir, log_filename)
+            open_log_file = open(logfile, 'w')
+        else:
+            open_log_file = sys.stdout
+        log.startLogging(open_log_file)
+
+    def exec_lines(self):
+        for line in self.master_config.Global.exec_lines:
+            try:
+                log.msg("Executing statement: '%s'" % line)
+                self.engine_service.execute(line)
+            except:
+                log.msg("Error executing statement: %s" % line)
+
+    def start_app(self):
+        # Start the controller service and set things running
+        reactor.run()
+
+
+def launch_new_instance():
+    """Create and run the IPython controller"""
+    app = IPEngineApp()
+    app.start()

IPython/config/default/ipengine_config.py

@@ -1,19 +1,25 b''
 c = get_config()
 
-c.MPI.default = 'mpi4py'
+c.MPI.use = 'mpi4py'
 
 c.MPI.mpi4py = """from mpi4py import MPI as mpi
 mpi.size = mpi.COMM_WORLD.Get_size()
 mpi.rank = mpi.COMM_WORLD.Get_rank()
 """
 
 c.MPI.pytrilinos = """from PyTrilinos import Epetra
 class SimpleStruct:
 pass
 mpi = SimpleStruct()
 mpi.rank = 0
 mpi.size = 0
 """
 
-c.Global.logfile = ''
-c.Global.furl_file = 'ipcontroller-engine.furl'
+c.Global.log_to_file = False
+c.Global.exec_lines = []
+c.Global.log_dir_name = 'log'
+c.Global.security_dir_name = 'security'
+c.Global.shell_class = 'IPython.kernel.core.interpreter.Interpreter'
+self.default_config.Global.furl_file_name = 'ipcontroller-engine.furl'
+self.default_config.Global.furl_file = ''
+

IPython/core/application.py

@@ -1,452 +1,454 b''
 #!/usr/bin/env python
 # encoding: utf-8
 """
 An application for IPython
 
 Authors:
 
 * Brian Granger
 * Fernando Perez
 
 Notes
 -----
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008-2009  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import logging
 import os
 
 import sys
 import traceback
 from copy import deepcopy
 
 from IPython.core import release
 from IPython.utils.genutils import get_ipython_dir, filefind
 from IPython.config.loader import (
     PyFileConfigLoader,
     ArgParseConfigLoader,
     Config,
     NoConfigDefault
 )
 
 #-----------------------------------------------------------------------------
 # Classes and functions
 #-----------------------------------------------------------------------------
 
 
 class BaseAppArgParseConfigLoader(ArgParseConfigLoader):
     """Default command line options for IPython based applications."""
 
     def _add_other_arguments(self):
         self.parser.add_argument('-ipythondir', '--ipython-dir', 
             dest='Global.ipythondir',type=str,
             help='Set to override default location of Global.ipythondir.',
             default=NoConfigDefault,
             metavar='Global.ipythondir')
         self.parser.add_argument('-p','-profile', '--profile',
             dest='Global.profile',type=str,
             help='The string name of the ipython profile to be used.',
             default=NoConfigDefault,
             metavar='Global.profile')
         self.parser.add_argument('-log_level', '--log-level',
             dest="Global.log_level",type=int,
             help='Set the log level (0,10,20,30,40,50).  Default is 30.',
-            default=NoConfigDefault)
+            default=NoConfigDefault,
+            metavar='Global.log_level')
         self.parser.add_argument('-config_file', '--config-file',
             dest='Global.config_file',type=str,
             help='Set the config file name to override default.',
             default=NoConfigDefault,
             metavar='Global.config_file')
 
 
 class ApplicationError(Exception):
     pass
 
 
 class Application(object):
     """Load a config, construct an app and run it.
     """
 
     name = 'ipython'
     description = 'IPython: an enhanced interactive Python shell.'
     config_file_name = 'ipython_config.py'
     default_log_level = logging.WARN
     
 
     def __init__(self):
         self.init_logger()
         self.default_config_file_name = self.config_file_name
 
     def init_logger(self):
         self.log = logging.getLogger(self.__class__.__name__)
         # This is used as the default until the command line arguments are read.
         self.log.setLevel(self.default_log_level)
         self._log_handler = logging.StreamHandler()
         self._log_formatter = logging.Formatter("[%(name)s] %(message)s")
         self._log_handler.setFormatter(self._log_formatter)
         self.log.addHandler(self._log_handler)
 
     def _set_log_level(self, level):
         self.log.setLevel(level)
 
     def _get_log_level(self):
         return self.log.level
 
     log_level = property(_get_log_level, _set_log_level)
 
     def start(self):
         """Start the application."""
         self.attempt(self.create_default_config)
         self.log_default_config()
         self.set_default_config_log_level()
         self.attempt(self.pre_load_command_line_config)
         self.attempt(self.load_command_line_config, action='abort')
         self.set_command_line_config_log_level()
         self.attempt(self.post_load_command_line_config)
         self.log_command_line_config()
         self.attempt(self.find_ipythondir)
         self.attempt(self.find_config_file_name)
         self.attempt(self.find_config_file_paths)
         self.attempt(self.pre_load_file_config)
         self.attempt(self.load_file_config)
         self.set_file_config_log_level()
         self.attempt(self.post_load_file_config)
         self.log_file_config()
         self.attempt(self.merge_configs)
         self.log_master_config()
         self.attempt(self.pre_construct)
         self.attempt(self.construct)
         self.attempt(self.post_construct)
         self.attempt(self.start_app)
 
     #-------------------------------------------------------------------------
     # Various stages of Application creation
     #-------------------------------------------------------------------------
 
     def create_default_config(self):
         """Create defaults that can't be set elsewhere.
 
         For the most part, we try to set default in the class attributes
         of Components.  But, defaults the top-level Application (which is
         not a HasTraitlets or Component) are not set in this way.  Instead
         we set them here.  The Global section is for variables like this that
         don't belong to a particular component.
         """
         self.default_config = Config()
         self.default_config.Global.ipythondir = get_ipython_dir()
         self.default_config.Global.log_level = self.log_level
 
     def log_default_config(self):
         self.log.debug('Default config loaded:')
         self.log.debug(repr(self.default_config))
 
     def set_default_config_log_level(self):
         try:
             self.log_level = self.default_config.Global.log_level
         except AttributeError:
             # Fallback to the default_log_level class attribute
             pass
 
     def create_command_line_config(self):
         """Create and return a command line config loader."""
         return BaseAppArgParseConfigLoader(
             description=self.description, 
             version=release.version
         )
 
     def pre_load_command_line_config(self):
         """Do actions just before loading the command line config."""
         pass
 
     def load_command_line_config(self):
         """Load the command line config."""
         loader = self.create_command_line_config()
         self.command_line_config = loader.load_config()
         self.extra_args = loader.get_extra_args()
 
     def set_command_line_config_log_level(self):
         try:
             self.log_level = self.command_line_config.Global.log_level
         except AttributeError:
             pass
 
     def post_load_command_line_config(self):
         """Do actions just after loading the command line config."""
         pass
 
     def log_command_line_config(self):
         self.log.debug("Command line config loaded:")
         self.log.debug(repr(self.command_line_config))
 
     def find_ipythondir(self):
         """Set the IPython directory.
 
         This sets ``self.ipythondir``, but the actual value that is passed
         to the application is kept in either ``self.default_config`` or
         ``self.command_line_config``.  This also added ``self.ipythondir`` to
         ``sys.path`` so config files there can be references by other config
         files.
         """
 
         try:
             self.ipythondir = self.command_line_config.Global.ipythondir
         except AttributeError:
             self.ipythondir = self.default_config.Global.ipythondir
         sys.path.append(os.path.abspath(self.ipythondir))
         if not os.path.isdir(self.ipythondir):
             os.makedirs(self.ipythondir, mode=0777)
         self.log.debug("IPYTHONDIR set to: %s" % self.ipythondir)
 
     def find_config_file_name(self):
         """Find the config file name for this application.
 
         This must set ``self.config_file_name`` to the filename of the
         config file to use (just the filename). The search paths for the
         config file are set in :meth:`find_config_file_paths` and then passed
         to the config file loader where they are resolved to an absolute path.
 
         If a profile has been set at the command line, this will resolve
         it.
         """
 
         try:
             self.config_file_name = self.command_line_config.Global.config_file
         except AttributeError:
             pass
 
         try:
             self.profile_name = self.command_line_config.Global.profile
             name_parts = self.config_file_name.split('.')
             name_parts.insert(1, '_' + self.profile_name + '.')
             self.config_file_name = ''.join(name_parts)
         except AttributeError:
             pass
 
     def find_config_file_paths(self):
         """Set the search paths for resolving the config file.
 
         This must set ``self.config_file_paths`` to a sequence of search
         paths to pass to the config file loader.
         """
         self.config_file_paths = (os.getcwd(), self.ipythondir)
 
     def pre_load_file_config(self):
         """Do actions before the config file is loaded."""
         pass
 
     def load_file_config(self):
         """Load the config file.
         
         This tries to load the config file from disk.  If successful, the
         ``CONFIG_FILE`` config variable is set to the resolved config file
         location.  If not successful, an empty config is used.
         """
-        self.log.debug("Attempting to load config file: <%s>" % self.config_file_name)
+        self.log.debug("Attempting to load config file: %s" % self.config_file_name)
         loader = PyFileConfigLoader(self.config_file_name,
                                     path=self.config_file_paths)
         try:
             self.file_config = loader.load_config()
             self.file_config.Global.config_file = loader.full_filename
         except IOError:
             # Only warn if the default config file was NOT being used.
             if not self.config_file_name==self.default_config_file_name:
-                self.log.warn("Config file not found, skipping: <%s>" % \
+                self.log.warn("Config file not found, skipping: %s" % \
                                self.config_file_name, exc_info=True)
             self.file_config = Config()
         except:
-            self.log.warn("Error loading config file: <%s>" % \
+            self.log.warn("Error loading config file: %s" % \
                            self.config_file_name, exc_info=True)
             self.file_config = Config()
 
     def set_file_config_log_level(self):
         # We need to keeep self.log_level updated.  But we only use the value
         # of the file_config if a value was not specified at the command
         # line, because the command line overrides everything.
         if not hasattr(self.command_line_config.Global, 'log_level'):
             try:
                 self.log_level = self.file_config.Global.log_level
             except AttributeError:
                 pass # Use existing value
 
     def post_load_file_config(self):
         """Do actions after the config file is loaded."""
         pass
 
     def log_file_config(self):
         if hasattr(self.file_config.Global, 'config_file'):
-            self.log.debug("Config file loaded: <%s>" % self.file_config.Global.config_file)
+            self.log.debug("Config file loaded: %s" % self.file_config.Global.config_file)
             self.log.debug(repr(self.file_config))
 
     def merge_configs(self):
         """Merge the default, command line and file config objects."""
         config = Config()
         config._merge(self.default_config)
         config._merge(self.file_config)
         config._merge(self.command_line_config)
         self.master_config = config
 
     def log_master_config(self):
         self.log.debug("Master config created:")
         self.log.debug(repr(self.master_config))
 
     def pre_construct(self):
         """Do actions after the config has been built, but before construct."""
         pass
 
     def construct(self):
         """Construct the main components that make up this app."""
         self.log.debug("Constructing components for application")
 
     def post_construct(self):
         """Do actions after construct, but before starting the app."""
         pass
 
     def start_app(self):
         """Actually start the app."""
         self.log.debug("Starting application")
 
     #-------------------------------------------------------------------------
     # Utility methods
     #-------------------------------------------------------------------------
 
     def abort(self):
         """Abort the starting of the application."""
         self.log.critical("Aborting application: %s" % self.name, exc_info=True)
         sys.exit(1)
 
     def exit(self):
         self.log.critical("Aborting application: %s" % self.name)
         sys.exit(1)
 
     def attempt(self, func, action='abort'):
         try:
             func()
         except SystemExit:
             self.exit()
         except:
             if action == 'abort':
                 self.abort()
             elif action == 'exit':
                 self.exit()
 
 
 class AppWithDirArgParseConfigLoader(ArgParseConfigLoader):
     """Default command line options for IPython based applications."""
 
     def _add_other_arguments(self):
         self.parser.add_argument('-ipythondir', '--ipython-dir', 
             dest='Global.ipythondir',type=str,
             help='Set to override default location of Global.ipythondir.',
             default=NoConfigDefault,
             metavar='Global.ipythondir')
         self.parser.add_argument('-p','-profile', '--profile',
             dest='Global.profile',type=str,
             help='The string name of the profile to be used. This determines '
             'the name of the application dir: basename_<profile>.  The basename is '
             'determined by the particular application.  The default profile '
             'is named "default".  This convention is used if the -app_dir '
             'option is not used.',
             default=NoConfigDefault,
             metavar='Global.profile')
         self.parser.add_argument('-log_level', '--log-level',
             dest="Global.log_level",type=int,
             help='Set the log level (0,10,20,30,40,50).  Default is 30.',
             default=NoConfigDefault)
         self.parser.add_argument('-app_dir', '--app-dir',
             dest='Global.app_dir',type=str,
             help='Set the application dir where everything for this '
             'application will be found (including the config file). This '
             'overrides the logic used by the profile option.',
             default=NoConfigDefault,
             metavar='Global.app_dir')
 
 
 class ApplicationWithDir(Application):
     """An application that puts everything into a application directory.
 
     Instead of looking for things in the ipythondir, this type of application
     will use its own private directory called the "application directory"
     for things like config files, log files, etc.
 
     The application directory is resolved as follows:
 
     * If the ``--app-dir`` option is given, it is used.
     * If ``--app-dir`` is not given, the application directory is resolve using
       ``app_dir_basename`` and ``profile`` as ``<app_dir_basename>_<profile>``.
       The search path for this directory is then i) cwd if it is found there
       and ii) in ipythondir otherwise.
 
     The config file for the application is to be put in the application
     dir and named the value of the ``config_file_name`` class attribute.
     """
 
     # The basename used for the application dir: <app_dir_basename>_<profile>
     app_dir_basename = 'cluster'
 
     def create_default_config(self):
         super(ApplicationWithDir, self).create_default_config()
         self.default_config.Global.profile = 'default'
         # The application dir.  This is empty initially so the default is to
         # try to resolve this using the profile.
         self.default_config.Global.app_dir = ''
 
     def create_command_line_config(self):
         """Create and return a command line config loader."""
         return AppWithDirArgParseConfigLoader(
             description=self.description, 
             version=release.version
         )
 
     def find_config_file_name(self):
         """Find the config file name for this application."""
         self.find_app_dir()
         self.create_app_dir()
 
     def find_app_dir(self):
         """This resolves the app directory.
 
         This method must set ``self.app_dir`` to the location of the app
         dir.
         """
         # Instead, first look for an explicit app_dir
         try:
             self.app_dir = self.command_line_config.Global.app_dir
         except AttributeError:
             self.app_dir = self.default_config.Global.app_dir
         self.app_dir = os.path.expandvars(os.path.expanduser(self.app_dir))
         if not self.app_dir:
             # Then look for a profile
             try:
                 self.profile = self.command_line_config.Global.profile
             except AttributeError:
                 self.profile = self.default_config.Global.profile
             app_dir_name = self.app_dir_basename + '_' + self.profile
             try_this = os.path.join(os.getcwd(), app_dir_name)
             if os.path.isdir(try_this):
                 self.app_dir = try_this
             else:
                 self.app_dir = os.path.join(self.ipythondir, app_dir_name)
         # These have to be set because they could be different from the one
         # that we just computed.  Because command line has the highest
         # priority, this will always end up in the master_config.
         self.default_config.Global.app_dir = self.app_dir
         self.command_line_config.Global.app_dir = self.app_dir
+        self.log.info("Application directory set to: %s" % self.app_dir)
 
     def create_app_dir(self):
         """Make sure that the app dir exists."""
         if not os.path.isdir(self.app_dir):
             os.makedirs(self.app_dir, mode=0777)
 
     def find_config_file_paths(self):
         """Set the search paths for resolving the config file."""
         self.config_file_paths = (self.app_dir,)

IPython/kernel/ipcontrollerapp.py

@@ -1,273 +1,275 b''
 #!/usr/bin/env python
 # encoding: utf-8
 """
 The IPython controller application
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008-2009  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import copy
 import logging
 import os
 import sys
 
 from twisted.application import service
 from twisted.internet import reactor, defer
 from twisted.python import log
 
 from IPython.config.loader import Config, NoConfigDefault
 
 from IPython.core.application import (
     ApplicationWithDir, 
     AppWithDirArgParseConfigLoader
 )
 
 from IPython.core import release
 
 from IPython.utils.traitlets import Int, Str, Bool, Instance
 from IPython.utils.importstring import import_item
 
 from IPython.kernel import controllerservice
 from IPython.kernel.configobjfactory import (
     ConfiguredObjectFactory,
     AdaptedConfiguredObjectFactory
 )
 
 from IPython.kernel.fcutil import FCServiceFactory
 
 #-----------------------------------------------------------------------------
 # Default interfaces
 #-----------------------------------------------------------------------------
 
 
 # The default client interfaces for FCClientServiceFactory.interfaces
 default_client_interfaces = Config()
 default_client_interfaces.Task.interface_chain = [
     'IPython.kernel.task.ITaskController',
     'IPython.kernel.taskfc.IFCTaskController'
 ]
 
 default_client_interfaces.Task.furl_file = 'ipcontroller-tc.furl'
 
 default_client_interfaces.MultiEngine.interface_chain = [
     'IPython.kernel.multiengine.IMultiEngine',
     'IPython.kernel.multienginefc.IFCSynchronousMultiEngine'
 ]
 
 default_client_interfaces.MultiEngine.furl_file = 'ipcontroller-mec.furl'
 
 # Make this a dict we can pass to Config.__init__ for the default
 default_client_interfaces = dict(copy.deepcopy(default_client_interfaces.items()))
 
 
 
 # The default engine interfaces for FCEngineServiceFactory.interfaces
 default_engine_interfaces = Config()
 default_engine_interfaces.Default.interface_chain = [
     'IPython.kernel.enginefc.IFCControllerBase'
 ]
 
 default_engine_interfaces.Default.furl_file = 'ipcontroller-engine.furl'
 
 # Make this a dict we can pass to Config.__init__ for the default
 default_engine_interfaces = dict(copy.deepcopy(default_engine_interfaces.items()))
 
 
 #-----------------------------------------------------------------------------
 # Service factories
 #-----------------------------------------------------------------------------
 
 
 class FCClientServiceFactory(FCServiceFactory):
     """A Foolscap implementation of the client services."""
 
     cert_file = Str('ipcontroller-client.pem', config=True)
     interfaces = Instance(klass=Config, kw=default_client_interfaces,
                           allow_none=False, config=True)
 
 
 class FCEngineServiceFactory(FCServiceFactory):
     """A Foolscap implementation of the engine services."""
 
     cert_file = Str('ipcontroller-engine.pem', config=True)
     interfaces = Instance(klass=dict, kw=default_engine_interfaces,
                           allow_none=False, config=True)
 
 
 #-----------------------------------------------------------------------------
 # The main application
 #-----------------------------------------------------------------------------
 
 
 cl_args = (
     # Client config
     (('--client-ip',), dict(
         type=str, dest='FCClientServiceFactory.ip', default=NoConfigDefault,
         help='The IP address or hostname the controller will listen on for client connections.',
         metavar='FCClientServiceFactory.ip')
     ),
     (('--client-port',), dict(
         type=int, dest='FCClientServiceFactory.port', default=NoConfigDefault,
         help='The port the controller will listen on for client connections.',
         metavar='FCClientServiceFactory.port')
     ),
     (('--client-location',), dict(
         type=str, dest='FCClientServiceFactory.location', default=NoConfigDefault,
         help='The hostname or ip that clients should connect to.',
         metavar='FCClientServiceFactory.location')
     ),
     # Engine config
     (('--engine-ip',), dict(
         type=str, dest='FCEngineServiceFactory.ip', default=NoConfigDefault,
         help='The IP address or hostname the controller will listen on for engine connections.',
         metavar='FCEngineServiceFactory.ip')
     ),
     (('--engine-port',), dict(
         type=int, dest='FCEngineServiceFactory.port', default=NoConfigDefault,
         help='The port the controller will listen on for engine connections.',
         metavar='FCEngineServiceFactory.port')
     ),
     (('--engine-location',), dict(
         type=str, dest='FCEngineServiceFactory.location', default=NoConfigDefault,
         help='The hostname or ip that engines should connect to.',
         metavar='FCEngineServiceFactory.location')
     ),
     # Global config
     (('--log-to-file',), dict(
         action='store_true', dest='Global.log_to_file', default=NoConfigDefault,
         help='Log to a file in the log directory (default is stdout)')
     ),
     (('-r','--reuse-furls'), dict(
         action='store_true', dest='Global.reuse_furls', default=NoConfigDefault,
         help='Try to reuse all FURL files.')
     ),
     (('-ns','--no-security'), dict(
         action='store_false', dest='Global.secure', default=NoConfigDefault,
         help='Turn off SSL encryption for all connections.')
     )
 )
 
 
 class IPControllerAppCLConfigLoader(AppWithDirArgParseConfigLoader):
 
     arguments = cl_args
 
 
 default_config_file_name = 'ipcontroller_config.py'
 
 
 class IPControllerApp(ApplicationWithDir):
 
     name = 'ipcontroller'
     app_dir_basename = 'cluster'
     description = 'Start the IPython controller for parallel computing.'
     config_file_name = default_config_file_name
-    default_log_level = logging.WARN
 
     def create_default_config(self):
         super(IPControllerApp, self).create_default_config()
         self.default_config.Global.reuse_furls = False
         self.default_config.Global.secure = True
         self.default_config.Global.import_statements = []
         self.default_config.Global.log_dir_name = 'log'
         self.default_config.Global.security_dir_name = 'security'
         self.default_config.Global.log_to_file = False
 
     def create_command_line_config(self):
         """Create and return a command line config loader."""
         return IPControllerAppCLConfigLoader(
             description=self.description, 
             version=release.version
         )
 
     def post_load_command_line_config(self):
         # Now setup reuse_furls
         if hasattr(self.command_line_config.Global, 'reuse_furls'):
             self.command_line_config.FCClientServiceFactory.reuse_furls = \
                 self.command_line_config.Global.reuse_furls
             self.command_line_config.FCEngineServiceFactory.reuse_furls = \
                 self.command_line_config.Global.reuse_furls
             del self.command_line_config.Global.reuse_furls
         if hasattr(self.command_line_config.Global, 'secure'):
             self.command_line_config.FCClientServiceFactory.secure = \
                 self.command_line_config.Global.secure
             self.command_line_config.FCEngineServiceFactory.secure = \
                 self.command_line_config.Global.secure
             del self.command_line_config.Global.secure
 
     def pre_construct(self):
+        config = self.master_config
         # Now set the security_dir and log_dir and create them.  We use
         # the names an construct the absolute paths.
-        security_dir = os.path.join(self.master_config.Global.app_dir,
-                                    self.master_config.Global.security_dir_name)
-        log_dir = os.path.join(self.master_config.Global.app_dir, 
-                               self.master_config.Global.log_dir_name)
+        security_dir = os.path.join(config.Global.app_dir,
+                                    config.Global.security_dir_name)
+        log_dir = os.path.join(config.Global.app_dir, 
+                               config.Global.log_dir_name)
         if not os.path.isdir(security_dir):
             os.mkdir(security_dir, 0700)
         else:
             os.chmod(security_dir, 0700)
         if not os.path.isdir(log_dir):
             os.mkdir(log_dir, 0777)
 
-        self.security_dir = self.master_config.Global.security_dir = security_dir
-        self.log_dir = self.master_config.Global.log_dir = log_dir
+        self.security_dir = config.Global.security_dir = security_dir
+        self.log_dir = config.Global.log_dir = log_dir
+        self.log.info("Log directory set to: %s" % self.log_dir)
+        self.log.info("Security directory set to: %s" % self.security_dir)
 
     def construct(self):
         # I am a little hesitant to put these into InteractiveShell itself.
         # But that might be the place for them
         sys.path.insert(0, '')
 
         self.start_logging()
         self.import_statements()
         
         # Create the service hierarchy
         self.main_service = service.MultiService()
         # The controller service
         controller_service = controllerservice.ControllerService()
         controller_service.setServiceParent(self.main_service)
         # The client tub and all its refereceables
         csfactory = FCClientServiceFactory(self.master_config, controller_service)
         client_service = csfactory.create()
         client_service.setServiceParent(self.main_service)
         # The engine tub
         esfactory = FCEngineServiceFactory(self.master_config, controller_service)
         engine_service = esfactory.create()
         engine_service.setServiceParent(self.main_service)
 
     def start_logging(self):
         if self.master_config.Global.log_to_file:
             log_filename = self.name + '-' + str(os.getpid()) + '.log'
             logfile = os.path.join(self.log_dir, log_filename)
             open_log_file = open(logfile, 'w')
         else:
             open_log_file = sys.stdout
         log.startLogging(open_log_file)
 
     def import_statements(self):
         statements = self.master_config.Global.import_statements
         for s in statements:
             try:
                 log.msg("Executing statement: '%s'" % s)
                 exec s in globals(), locals()
             except:
                 log.msg("Error running statement: %s" % s)
 
     def start_app(self):
         # Start the controller service and set things running
         self.main_service.startService()
         reactor.run()
 
 
 def launch_new_instance():
     """Create and run the IPython controller"""
     app = IPControllerApp()
     app.start()

IPython/kernel/scripts/ipengine

@@ -1,20 +1,20 b''
 #!/usr/bin/env python
 # encoding: utf-8
 
-__docformat__ = "restructuredtext en"
-
-#-------------------------------------------------------------------------------
-#  Copyright (C) 2008  The IPython Development Team
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2009  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
-#-------------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
 
-#-------------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
 # Imports
-#-------------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
+
+
+from IPython.kernel.ipengineapp import launch_new_instance
+
+launch_new_instance()
 
-if __name__ == '__main__':
-	from IPython.kernel.scripts import ipengine
-	ipengine.main()
 

setup.py

@@ -1,210 +1,210 b''
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 """Setup script for IPython.
 
 Under Posix environments it works like a typical setup.py script. 
 Under Windows, the command sdist is not supported, since IPython 
 requires utilities which are not available under Windows."""
 
 #-------------------------------------------------------------------------------
 #  Copyright (C) 2008  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-------------------------------------------------------------------------------
 
 #-------------------------------------------------------------------------------
 # Imports
 #-------------------------------------------------------------------------------
 
 # Stdlib imports
 import os
 import sys
 
 from glob import glob
 
 # BEFORE importing distutils, remove MANIFEST. distutils doesn't properly
 # update it when the contents of directories change.
 if os.path.exists('MANIFEST'): os.remove('MANIFEST')
 
 from distutils.core import setup
 
 from IPython.utils.genutils import target_update
 
 from setupbase import (
     setup_args, 
     find_packages, 
     find_package_data, 
     find_scripts,
     find_data_files,
     check_for_dependencies
 )
 
 isfile = os.path.isfile
 pjoin = os.path.join
 
 #-------------------------------------------------------------------------------
 # Handle OS specific things
 #-------------------------------------------------------------------------------
 
 if os.name == 'posix':
     os_name = 'posix'
 elif os.name in ['nt','dos']:
     os_name = 'windows'
 else:
     print 'Unsupported operating system:',os.name
     sys.exit(1)
 
 # Under Windows, 'sdist' has not been supported.  Now that the docs build with
 # Sphinx it might work, but let's not turn it on until someone confirms that it
 # actually works.
 if os_name == 'windows' and 'sdist' in sys.argv:
     print 'The sdist command is not available under Windows.  Exiting.'
     sys.exit(1)
 
 #-------------------------------------------------------------------------------
 # Things related to the IPython documentation
 #-------------------------------------------------------------------------------
 
 # update the manuals when building a source dist
 if len(sys.argv) >= 2 and sys.argv[1] in ('sdist','bdist_rpm'):
     import textwrap
 
     # List of things to be updated. Each entry is a triplet of args for
     # target_update()
     to_update = [
                   # FIXME - Disabled for now: we need to redo an automatic way
                   # of generating the magic info inside the rst.                
                   #('docs/magic.tex',
                   #['IPython/Magic.py'],
                   #"cd doc && ./update_magic.sh" ),
 
                  ('docs/man/ipcluster.1.gz',
                   ['docs/man/ipcluster.1'],
                   'cd docs/man && gzip -9c ipcluster.1 > ipcluster.1.gz'),
 
                  ('docs/man/ipcontroller.1.gz',
                   ['docs/man/ipcontroller.1'],
                   'cd docs/man && gzip -9c ipcontroller.1 > ipcontroller.1.gz'),
 
                  ('docs/man/ipengine.1.gz',
                   ['docs/man/ipengine.1'],
                   'cd docs/man && gzip -9c ipengine.1 > ipengine.1.gz'),
 
                  ('docs/man/ipython.1.gz',
                   ['docs/man/ipython.1'],
                   'cd docs/man && gzip -9c ipython.1 > ipython.1.gz'),
 
                  ('docs/man/ipython-wx.1.gz',
                   ['docs/man/ipython-wx.1'],
                   'cd docs/man && gzip -9c ipython-wx.1 > ipython-wx.1.gz'),
 
                  ('docs/man/ipythonx.1.gz',
                   ['docs/man/ipythonx.1'],
                   'cd docs/man && gzip -9c ipythonx.1 > ipythonx.1.gz'),
 
                  ('docs/man/irunner.1.gz',
                   ['docs/man/irunner.1'],
                   'cd docs/man && gzip -9c irunner.1 > irunner.1.gz'),
 
                  ('docs/man/pycolor.1.gz',
                   ['docs/man/pycolor.1'],
                   'cd docs/man && gzip -9c pycolor.1 > pycolor.1.gz'),
                  ]
 
     # Only build the docs if sphinx is present
     try:
         import sphinx
     except ImportError:
         pass
     else:
         # The Makefile calls the do_sphinx scripts to build html and pdf, so
         # just one target is enough to cover all manual generation
 
         # First, compute all the dependencies that can force us to rebuild the
         # docs.  Start with the main release file that contains metadata
         docdeps = ['IPython/core/release.py']
         # Inculde all the reST sources
         pjoin = os.path.join
         for dirpath,dirnames,filenames in os.walk('docs/source'):
             if dirpath in ['_static','_templates']:
                 continue
             docdeps += [ pjoin(dirpath,f) for f in filenames
                          if f.endswith('.txt') ]
         # and the examples
         for dirpath,dirnames,filenames in os.walk('docs/example'):
             docdeps += [ pjoin(dirpath,f) for f in filenames
                          if not f.endswith('~') ]
         # then, make them all dependencies for the main PDF (the html will get
         # auto-generated as well).
         to_update.append(
             ('docs/dist/ipython.pdf',
              docdeps,
              "cd docs && make dist")
             )
         
     [ target_update(*t) for t in to_update ]
 
     
 #---------------------------------------------------------------------------
 # Find all the packages, package data, scripts and data_files
 #---------------------------------------------------------------------------
 
 packages = find_packages()
 package_data = find_package_data()
 scripts = find_scripts()
 data_files = find_data_files()
 
 #---------------------------------------------------------------------------
 # Handle dependencies and setuptools specific things
 #---------------------------------------------------------------------------
 
 # This dict is used for passing extra arguments that are setuptools 
 # specific to setup
 setuptools_extra_args = {}
 
 if 'setuptools' in sys.modules:
     setuptools_extra_args['zip_safe'] = False
     setuptools_extra_args['entry_points'] = {
         'console_scripts': [
             'ipython = IPython.core.ipapp:launch_new_instance',
             'pycolor = IPython.utils.PyColorize:main',
             'ipcontroller = IPython.kernel.ipcontrollerapp:launch_new_instance',
-            'ipengine = IPython.kernel.scripts.ipengine:main',
+            'ipengine = IPython.kernel.ipengineapp:launch_new_instance',
             'ipcluster = IPython.kernel.scripts.ipcluster:main',
             'ipythonx = IPython.frontend.wx.ipythonx:main',
             'iptest = IPython.testing.iptest:main',
             'irunner = IPython.lib.irunner:main'
         ]
     }
     setup_args['extras_require'] = dict(
         kernel = [
             'zope.interface>=3.4.1',
             'Twisted>=8.0.1',
             'foolscap>=0.2.6'
         ],
         doc='Sphinx>=0.3',
         test='nose>=0.10.1',
         security='pyOpenSSL>=0.6'
     )
     # Allow setuptools to handle the scripts
     scripts = []
 else:
     # If we are running without setuptools, call this function which will
     # check for dependencies an inform the user what is needed.  This is
     # just to make life easy for users.
     check_for_dependencies()
 
 
 #---------------------------------------------------------------------------
 # Do the actual setup now
 #---------------------------------------------------------------------------
 
 setup_args['packages'] = packages
 setup_args['package_data'] = package_data
 setup_args['scripts'] = scripts
 setup_args['data_files'] = data_files
 setup_args.update(setuptools_extra_args)
 
 if __name__ == '__main__':
     setup(**setup_args)