From de16c0c9aa18c157dcbd30f2c4ee795b5ce67c7e 2009-10-22 03:06:22 From: Brian Granger Date: 2009-10-22 03:06:22 Subject: [PATCH] The ipengine script has been refactored to use the new config system. It is also now a formal Application: IPython.kernel.ipengineapp --- diff --git a/IPython/config/default/ipengine_config.py b/IPython/config/default/ipengine_config.py index f86a407..fafae22 100644 --- a/IPython/config/default/ipengine_config.py +++ b/IPython/config/default/ipengine_config.py @@ -1,6 +1,6 @@ c = get_config() -c.MPI.default = 'mpi4py' +c.MPI.use = 'mpi4py' c.MPI.mpi4py = """from mpi4py import MPI as mpi mpi.size = mpi.COMM_WORLD.Get_size() @@ -15,5 +15,11 @@ mpi.rank = 0 mpi.size = 0 """ -c.Global.logfile = '' -c.Global.furl_file = 'ipcontroller-engine.furl' +c.Global.log_to_file = False +c.Global.exec_lines = [] +c.Global.log_dir_name = 'log' +c.Global.security_dir_name = 'security' +c.Global.shell_class = 'IPython.kernel.core.interpreter.Interpreter' +self.default_config.Global.furl_file_name = 'ipcontroller-engine.furl' +self.default_config.Global.furl_file = '' + diff --git a/IPython/core/application.py b/IPython/core/application.py index 49e22e5..3c47fd7 100644 --- a/IPython/core/application.py +++ b/IPython/core/application.py @@ -61,7 +61,8 @@ class BaseAppArgParseConfigLoader(ArgParseConfigLoader): self.parser.add_argument('-log_level', '--log-level', dest="Global.log_level",type=int, help='Set the log level (0,10,20,30,40,50). Default is 30.', - default=NoConfigDefault) + default=NoConfigDefault, + metavar='Global.log_level') self.parser.add_argument('-config_file', '--config-file', dest='Global.config_file',type=str, help='Set the config file name to override default.', @@ -251,7 +252,7 @@ class Application(object): ``CONFIG_FILE`` config variable is set to the resolved config file location. If not successful, an empty config is used. """ - self.log.debug("Attempting to load config file: <%s>" % self.config_file_name) + self.log.debug("Attempting to load config file: %s" % self.config_file_name) loader = PyFileConfigLoader(self.config_file_name, path=self.config_file_paths) try: @@ -260,11 +261,11 @@ class Application(object): except IOError: # Only warn if the default config file was NOT being used. if not self.config_file_name==self.default_config_file_name: - self.log.warn("Config file not found, skipping: <%s>" % \ + self.log.warn("Config file not found, skipping: %s" % \ self.config_file_name, exc_info=True) self.file_config = Config() except: - self.log.warn("Error loading config file: <%s>" % \ + self.log.warn("Error loading config file: %s" % \ self.config_file_name, exc_info=True) self.file_config = Config() @@ -284,7 +285,7 @@ class Application(object): def log_file_config(self): if hasattr(self.file_config.Global, 'config_file'): - self.log.debug("Config file loaded: <%s>" % self.file_config.Global.config_file) + self.log.debug("Config file loaded: %s" % self.file_config.Global.config_file) self.log.debug(repr(self.file_config)) def merge_configs(self): @@ -441,6 +442,7 @@ class ApplicationWithDir(Application): # priority, this will always end up in the master_config. self.default_config.Global.app_dir = self.app_dir self.command_line_config.Global.app_dir = self.app_dir + self.log.info("Application directory set to: %s" % self.app_dir) def create_app_dir(self): """Make sure that the app dir exists.""" diff --git a/IPython/kernel/ipcontrollerapp.py b/IPython/kernel/ipcontrollerapp.py index dcb85d1..72936b3 100644 --- a/IPython/kernel/ipcontrollerapp.py +++ b/IPython/kernel/ipcontrollerapp.py @@ -171,7 +171,6 @@ class IPControllerApp(ApplicationWithDir): app_dir_basename = 'cluster' description = 'Start the IPython controller for parallel computing.' config_file_name = default_config_file_name - default_log_level = logging.WARN def create_default_config(self): super(IPControllerApp, self).create_default_config() @@ -205,12 +204,13 @@ class IPControllerApp(ApplicationWithDir): del self.command_line_config.Global.secure def pre_construct(self): + config = self.master_config # Now set the security_dir and log_dir and create them. We use # the names an construct the absolute paths. - security_dir = os.path.join(self.master_config.Global.app_dir, - self.master_config.Global.security_dir_name) - log_dir = os.path.join(self.master_config.Global.app_dir, - self.master_config.Global.log_dir_name) + security_dir = os.path.join(config.Global.app_dir, + config.Global.security_dir_name) + log_dir = os.path.join(config.Global.app_dir, + config.Global.log_dir_name) if not os.path.isdir(security_dir): os.mkdir(security_dir, 0700) else: @@ -218,8 +218,10 @@ class IPControllerApp(ApplicationWithDir): if not os.path.isdir(log_dir): os.mkdir(log_dir, 0777) - self.security_dir = self.master_config.Global.security_dir = security_dir - self.log_dir = self.master_config.Global.log_dir = log_dir + self.security_dir = config.Global.security_dir = security_dir + self.log_dir = config.Global.log_dir = log_dir + self.log.info("Log directory set to: %s" % self.log_dir) + self.log.info("Security directory set to: %s" % self.security_dir) def construct(self): # I am a little hesitant to put these into InteractiveShell itself. diff --git a/IPython/kernel/ipengineapp.py b/IPython/kernel/ipengineapp.py new file mode 100644 index 0000000..682bae8 --- /dev/null +++ b/IPython/kernel/ipengineapp.py @@ -0,0 +1,257 @@ +#!/usr/bin/env python +# encoding: utf-8 +""" +The IPython controller application +""" + +#----------------------------------------------------------------------------- +# Copyright (C) 2008-2009 The IPython Development Team +# +# Distributed under the terms of the BSD License. The full license is in +# the file COPYING, distributed as part of this software. +#----------------------------------------------------------------------------- + +#----------------------------------------------------------------------------- +# Imports +#----------------------------------------------------------------------------- + +import os +import sys + +from twisted.application import service +from twisted.internet import reactor +from twisted.python import log + +from IPython.config.loader import NoConfigDefault + +from IPython.core.application import ( + ApplicationWithDir, + AppWithDirArgParseConfigLoader +) +from IPython.core import release + +from IPython.utils.importstring import import_item + +from IPython.kernel.engineservice import EngineService +from IPython.kernel.fcutil import Tub +from IPython.kernel.engineconnector import EngineConnector + +#----------------------------------------------------------------------------- +# The main application +#----------------------------------------------------------------------------- + + +cl_args = ( + # Controller config + (('--furl-file',), dict( + type=str, dest='Global.furl_file', default=NoConfigDefault, + help='The full location of the file containing the FURL of the ' + 'controller. If this is not given, the FURL file must be in the ' + 'security directory of the cluster directory. This location is ' + 'resolved using the --profile and --app-dir options.', + metavar='Global.furl_file') + ), + # MPI + (('--mpi',), dict( + type=str, dest='MPI.use', default=NoConfigDefault, + help='How to enable MPI (mpi4py, pytrilinos, or empty string to disable).', + metavar='MPI.use') + ), + # Global config + (('--log-to-file',), dict( + action='store_true', dest='Global.log_to_file', default=NoConfigDefault, + help='Log to a file in the log directory (default is stdout)') + ) +) + + +class IPEngineAppCLConfigLoader(AppWithDirArgParseConfigLoader): + + arguments = cl_args + + +mpi4py_init = """from mpi4py import MPI as mpi +mpi.size = mpi.COMM_WORLD.Get_size() +mpi.rank = mpi.COMM_WORLD.Get_rank() +""" + +pytrilinos_init = """from PyTrilinos import Epetra +class SimpleStruct: +pass +mpi = SimpleStruct() +mpi.rank = 0 +mpi.size = 0 +""" + + +default_config_file_name = 'ipengine_config.py' + + +class IPEngineApp(ApplicationWithDir): + + name = 'ipengine' + app_dir_basename = 'cluster' + description = 'Start the IPython engine for parallel computing.' + config_file_name = default_config_file_name + + def create_default_config(self): + super(IPEngineApp, self).create_default_config() + + # Global config attributes + self.default_config.Global.log_to_file = False + self.default_config.Global.exec_lines = [] + # The log and security dir names must match that of the controller + self.default_config.Global.log_dir_name = 'log' + self.default_config.Global.security_dir_name = 'security' + self.default_config.Global.shell_class = 'IPython.kernel.core.interpreter.Interpreter' + + # Configuration related to the controller + # This must match the filename (path not included) that the controller + # used for the FURL file. + self.default_config.Global.furl_file_name = 'ipcontroller-engine.furl' + # If given, this is the actual location of the controller's FURL file. + # If not, this is computed using the profile, app_dir and furl_file_name + self.default_config.Global.furl_file = '' + + # MPI related config attributes + self.default_config.MPI.use = '' + self.default_config.MPI.mpi4py = mpi4py_init + self.default_config.MPI.pytrilinos = pytrilinos_init + + def create_command_line_config(self): + """Create and return a command line config loader.""" + return IPEngineAppCLConfigLoader( + description=self.description, + version=release.version + ) + + def post_load_command_line_config(self): + pass + + def pre_construct(self): + config = self.master_config + # Now set the security_dir and log_dir and create them. We use + # the names an construct the absolute paths. + security_dir = os.path.join(config.Global.app_dir, + config.Global.security_dir_name) + log_dir = os.path.join(config.Global.app_dir, + config.Global.log_dir_name) + if not os.path.isdir(security_dir): + os.mkdir(security_dir, 0700) + else: + os.chmod(security_dir, 0700) + if not os.path.isdir(log_dir): + os.mkdir(log_dir, 0777) + + self.security_dir = config.Global.security_dir = security_dir + self.log_dir = config.Global.log_dir = log_dir + self.log.info("Log directory set to: %s" % self.log_dir) + self.log.info("Security directory set to: %s" % self.security_dir) + + self.find_cont_furl_file() + + def find_cont_furl_file(self): + config = self.master_config + # Find the actual controller FURL file + if os.path.isfile(config.Global.furl_file): + return + else: + # We should know what the app dir is + try_this = os.path.join( + config.Global.app_dir, + config.Global.security_dir, + config.Global.furl_file_name + ) + if os.path.isfile(try_this): + config.Global.furl_file = try_this + return + else: + self.log.critical("Could not find a valid controller FURL file.") + self.abort() + + def construct(self): + # I am a little hesitant to put these into InteractiveShell itself. + # But that might be the place for them + sys.path.insert(0, '') + + self.start_mpi() + self.start_logging() + + # Create the underlying shell class and EngineService + shell_class = import_item(self.master_config.Global.shell_class) + self.engine_service = EngineService(shell_class, mpi=mpi) + + self.exec_lines() + + # Create the service hierarchy + self.main_service = service.MultiService() + self.engine_service.setServiceParent(self.main_service) + self.tub_service = Tub() + self.tub_service.setServiceParent(self.main_service) + # This needs to be called before the connection is initiated + self.main_service.startService() + + # This initiates the connection to the controller and calls + # register_engine to tell the controller we are ready to do work + self.engine_connector = EngineConnector(self.tub_service) + + log.msg("Using furl file: %s" % self.master_config.Global.furl_file) + + reactor.callWhenRunning(self.call_connect) + + def call_connect(self): + d = self.engine_connector.connect_to_controller( + self.engine_service, + self.master_config.Global.furl_file + ) + + def handle_error(f): + # If this print statement is replaced by a log.err(f) I get + # an unhandled error, which makes no sense. I shouldn't have + # to use a print statement here. My only thought is that + # at the beginning of the process the logging is still starting up + print "Error connecting to controller:", f.getErrorMessage() + reactor.callLater(0.1, reactor.stop) + + d.addErrback(handle_error) + + def start_mpi(self): + global mpi + mpikey = self.master_config.MPI.use + mpi_import_statement = self.master_config.MPI.get(mpikey, None) + if mpi_import_statement is not None: + try: + self.log.info("Initializing MPI:") + self.log.info(mpi_import_statement) + exec mpi_import_statement in globals() + except: + mpi = None + else: + mpi = None + + def start_logging(self): + if self.master_config.Global.log_to_file: + log_filename = self.name + '-' + str(os.getpid()) + '.log' + logfile = os.path.join(self.log_dir, log_filename) + open_log_file = open(logfile, 'w') + else: + open_log_file = sys.stdout + log.startLogging(open_log_file) + + def exec_lines(self): + for line in self.master_config.Global.exec_lines: + try: + log.msg("Executing statement: '%s'" % line) + self.engine_service.execute(line) + except: + log.msg("Error executing statement: %s" % line) + + def start_app(self): + # Start the controller service and set things running + reactor.run() + + +def launch_new_instance(): + """Create and run the IPython controller""" + app = IPEngineApp() + app.start() diff --git a/IPython/kernel/scripts/ipengine b/IPython/kernel/scripts/ipengine index 92eab1c..2a4e04a 100755 --- a/IPython/kernel/scripts/ipengine +++ b/IPython/kernel/scripts/ipengine @@ -1,20 +1,20 @@ #!/usr/bin/env python # encoding: utf-8 -__docformat__ = "restructuredtext en" - -#------------------------------------------------------------------------------- -# Copyright (C) 2008 The IPython Development Team +#----------------------------------------------------------------------------- +# Copyright (C) 2008-2009 The IPython Development Team # # Distributed under the terms of the BSD License. The full license is in # the file COPYING, distributed as part of this software. -#------------------------------------------------------------------------------- +#----------------------------------------------------------------------------- -#------------------------------------------------------------------------------- +#----------------------------------------------------------------------------- # Imports -#------------------------------------------------------------------------------- +#----------------------------------------------------------------------------- + + +from IPython.kernel.ipengineapp import launch_new_instance + +launch_new_instance() -if __name__ == '__main__': - from IPython.kernel.scripts import ipengine - ipengine.main() diff --git a/IPython/kernel/scripts/ipengine.py b/IPython/kernel/scripts/ipengine.py deleted file mode 100755 index 425c3c1..0000000 --- a/IPython/kernel/scripts/ipengine.py +++ /dev/null @@ -1,193 +0,0 @@ -#!/usr/bin/env python -# encoding: utf-8 - -"""Start the IPython Engine.""" - -__docformat__ = "restructuredtext en" - -#------------------------------------------------------------------------------- -# Copyright (C) 2008 The IPython Development Team -# -# Distributed under the terms of the BSD License. The full license is in -# the file COPYING, distributed as part of this software. -#------------------------------------------------------------------------------- - -#------------------------------------------------------------------------------- -# Imports -#------------------------------------------------------------------------------- - -# Python looks for an empty string at the beginning of sys.path to enable -# importing from the cwd. -import sys -sys.path.insert(0, '') - -from optparse import OptionParser -import os - -from twisted.application import service -from twisted.internet import reactor -from twisted.python import log - -from IPython.kernel.fcutil import Tub, UnauthenticatedTub - -from IPython.kernel.core.config import config_manager as core_config_manager -from IPython.utils.importstring import import_item -from IPython.kernel.engineservice import EngineService - -# Create various ipython directories if they don't exist. -# This must be done before IPython.kernel.config is imported. -from IPython.core.oldusersetup import user_setup -from IPython.utils.genutils import get_ipython_dir, get_log_dir, get_security_dir -if os.name == 'posix': - rc_suffix = '' -else: - rc_suffix = '.ini' -user_setup(get_ipython_dir(), rc_suffix, mode='install', interactive=False) -get_log_dir() -get_security_dir() - -from IPython.kernel.config import config_manager as kernel_config_manager -from IPython.kernel.engineconnector import EngineConnector - - -#------------------------------------------------------------------------------- -# Code -#------------------------------------------------------------------------------- - -def start_engine(): - """ - Start the engine, by creating it and starting the Twisted reactor. - - This method does: - - * If it exists, runs the `mpi_import_statement` to call `MPI_Init` - * Starts the engine logging - * Creates an IPython shell and wraps it in an `EngineService` - * Creates a `foolscap.Tub` to use in connecting to a controller. - * Uses the tub and the `EngineService` along with a Foolscap URL - (or FURL) to connect to the controller and register the engine - with the controller - """ - kernel_config = kernel_config_manager.get_config_obj() - core_config = core_config_manager.get_config_obj() - - - # Execute the mpi import statement that needs to call MPI_Init - global mpi - mpikey = kernel_config['mpi']['default'] - mpi_import_statement = kernel_config['mpi'].get(mpikey, None) - if mpi_import_statement is not None: - try: - exec mpi_import_statement in globals() - except: - mpi = None - else: - mpi = None - - # Start logging - logfile = kernel_config['engine']['logfile'] - if logfile: - logfile = logfile + str(os.getpid()) + '.log' - try: - openLogFile = open(logfile, 'w') - except: - openLogFile = sys.stdout - else: - openLogFile = sys.stdout - log.startLogging(openLogFile) - - # Create the underlying shell class and EngineService - shell_class = import_item(core_config['shell']['shell_class']) - engine_service = EngineService(shell_class, mpi=mpi) - shell_import_statement = core_config['shell']['import_statement'] - if shell_import_statement: - try: - engine_service.execute(shell_import_statement) - except: - log.msg("Error running import_statement: %s" % shell_import_statement) - - # Create the service hierarchy - main_service = service.MultiService() - engine_service.setServiceParent(main_service) - tub_service = Tub() - tub_service.setServiceParent(main_service) - # This needs to be called before the connection is initiated - main_service.startService() - - # This initiates the connection to the controller and calls - # register_engine to tell the controller we are ready to do work - engine_connector = EngineConnector(tub_service) - furl_file = kernel_config['engine']['furl_file'] - log.msg("Using furl file: %s" % furl_file) - - def call_connect(engine_service, furl_file): - d = engine_connector.connect_to_controller(engine_service, furl_file) - def handle_error(f): - # If this print statement is replaced by a log.err(f) I get - # an unhandled error, which makes no sense. I shouldn't have - # to use a print statement here. My only thought is that - # at the beginning of the process the logging is still starting up - print "error connecting to controller:", f.getErrorMessage() - reactor.callLater(0.1, reactor.stop) - d.addErrback(handle_error) - - reactor.callWhenRunning(call_connect, engine_service, furl_file) - reactor.run() - - -def init_config(): - """ - Initialize the configuration using default and command line options. - """ - - parser = OptionParser("""ipengine [options] - -Start an IPython engine. - -Use the IPYTHONDIR environment variable to change your IPython directory -from the default of .ipython or _ipython. The log and security -subdirectories of your IPython directory will be used by this script -for log files and security files.""") - - parser.add_option( - "--furl-file", - type="string", - dest="furl_file", - help="The filename containing the FURL of the controller" - ) - parser.add_option( - "--mpi", - type="string", - dest="mpi", - help="How to enable MPI (mpi4py, pytrilinos, or empty string to disable)" - ) - parser.add_option( - "-l", - "--logfile", - type="string", - dest="logfile", - help="log file name (default is stdout)" - ) - - (options, args) = parser.parse_args() - - kernel_config = kernel_config_manager.get_config_obj() - # Now override with command line options - if options.furl_file is not None: - kernel_config['engine']['furl_file'] = options.furl_file - if options.logfile is not None: - kernel_config['engine']['logfile'] = options.logfile - if options.mpi is not None: - kernel_config['mpi']['default'] = options.mpi - - -def main(): - """ - After creating the configuration information, start the engine. - """ - init_config() - start_engine() - - -if __name__ == "__main__": - main() diff --git a/setup.py b/setup.py index 6170588..7ac0aec 100755 --- a/setup.py +++ b/setup.py @@ -170,7 +170,7 @@ if 'setuptools' in sys.modules: 'ipython = IPython.core.ipapp:launch_new_instance', 'pycolor = IPython.utils.PyColorize:main', 'ipcontroller = IPython.kernel.ipcontrollerapp:launch_new_instance', - 'ipengine = IPython.kernel.scripts.ipengine:main', + 'ipengine = IPython.kernel.ipengineapp:launch_new_instance', 'ipcluster = IPython.kernel.scripts.ipcluster:main', 'ipythonx = IPython.frontend.wx.ipythonx:main', 'iptest = IPython.testing.iptest:main',