diff --git a/IPython/config/default/ipengine_config.py b/IPython/config/default/ipengine_config.py index 8d55db4..ed35df9 100644 --- a/IPython/config/default/ipengine_config.py +++ b/IPython/config/default/ipengine_config.py @@ -1,32 +1,24 @@ c = get_config() -c.MPI.use = '' +# c.Global.log_to_file = False +# c.Global.exec_lines = ['import numpy'] +# c.Global.log_dir_name = 'log' +# c.Global.security_dir_name = 'security' +# c.Global.log_level = 10 +# c.Global.shell_class = 'IPython.kernel.core.interpreter.Interpreter' +# c.Global.furl_file_name = 'ipcontroller-engine.furl' +# c.Global.furl_file = '' + +# c.MPI.use = '' +# c.MPI.mpi4py = """from mpi4py import MPI as mpi +# mpi.size = mpi.COMM_WORLD.Get_size() +# mpi.rank = mpi.COMM_WORLD.Get_rank() +# """ +# c.MPI.pytrilinos = """from PyTrilinos import Epetra +# class SimpleStruct: +# pass +# mpi = SimpleStruct() +# mpi.rank = 0 +# mpi.size = 0 +# """ -c.MPI.mpi4py = """from mpi4py import MPI as mpi -mpi.size = mpi.COMM_WORLD.Get_size() -mpi.rank = mpi.COMM_WORLD.Get_rank() -""" - -c.MPI.pytrilinos = """from PyTrilinos import Epetra -class SimpleStruct: -pass -mpi = SimpleStruct() -mpi.rank = 0 -mpi.size = 0 -""" - -c.Global.log_to_file = False - -c.Global.exec_lines = ['import numpy'] - -c.Global.log_dir_name = 'log' - -c.Global.security_dir_name = 'security' - -c.Global.log_level = 10 - -c.Global.shell_class = 'IPython.kernel.core.interpreter.Interpreter' - -c.Global.furl_file_name = 'ipcontroller-engine.furl' - -c.Global.furl_file = '' diff --git a/IPython/core/application.py b/IPython/core/application.py index 97690cd..ed25a1e 100644 --- a/IPython/core/application.py +++ b/IPython/core/application.py @@ -86,6 +86,7 @@ class Application(object): default_log_level = logging.WARN def __init__(self): + self._exiting = False self.init_logger() # Track the default and actual separately because some messages are # only printed if we aren't using the default. @@ -119,6 +120,7 @@ class Application(object): self.attempt(self.post_load_command_line_config) self.log_command_line_config() self.attempt(self.find_ipythondir) + self.attempt(self.find_resources) self.attempt(self.find_config_file_name) self.attempt(self.find_config_file_paths) self.attempt(self.pre_load_file_config) @@ -211,6 +213,15 @@ class Application(object): os.makedirs(self.ipythondir, mode=0777) self.log.debug("IPYTHONDIR set to: %s" % self.ipythondir) + def find_resources(self): + """Find other resources that need to be in place. + + Things like cluster directories need to be in place to find the + config file. These happen right after the IPython directory has + been set. + """ + pass + def find_config_file_name(self): """Find the config file name for this application. @@ -325,18 +336,26 @@ class Application(object): def abort(self): """Abort the starting of the application.""" - self.log.critical("Aborting application: %s" % self.name, exc_info=True) - sys.exit(1) + if self._exiting: + pass + else: + self.log.critical("Aborting application: %s" % self.name, exc_info=True) + self._exiting = True + sys.exit(1) def exit(self): - self.log.critical("Aborting application: %s" % self.name) - sys.exit(1) + if self._exiting: + pass + else: + self.log.debug("Exiting application: %s" % self.name) + self._exiting = True + sys.exit(1) def attempt(self, func, action='abort'): try: func() except SystemExit: - self.exit() + raise except: if action == 'abort': self.abort() diff --git a/IPython/kernel/clusterdir.py b/IPython/kernel/clusterdir.py index 9de6cdd..25b56d3 100644 --- a/IPython/kernel/clusterdir.py +++ b/IPython/kernel/clusterdir.py @@ -23,13 +23,17 @@ from IPython.config.loader import PyFileConfigLoader from IPython.core.application import Application from IPython.core.component import Component from IPython.config.loader import ArgParseConfigLoader, NoConfigDefault -from IPython.utils.traitlets import Unicode +from IPython.utils.traitlets import Unicode, Bool #----------------------------------------------------------------------------- # Imports #----------------------------------------------------------------------------- +class ClusterDirError(Exception): + pass + + class ClusterDir(Component): """An object to manage the cluster directory and its resources. @@ -117,28 +121,68 @@ class ClusterDir(Component): self.copy_config_file(f, path=path, overwrite=overwrite) @classmethod - def find_cluster_dir_by_profile(cls, path, profile='default'): - """Find/create a cluster dir by profile name and return its ClusterDir. + def create_cluster_dir(csl, cluster_dir): + """Create a new cluster directory given a full path. - This will create the cluster directory if it doesn't exist. + Parameters + ---------- + cluster_dir : str + The full path to the cluster directory. If it does exist, it will + be used. If not, it will be created. + """ + return ClusterDir(cluster_dir) + + @classmethod + def create_cluster_dir_by_profile(cls, path, profile='default'): + """Create a cluster dir by profile name and path. + + Parameters + ---------- + path : str + The path (directory) to put the cluster directory in. + profile : str + The name of the profile. The name of the cluster directory will + be "cluster_". + """ + if not os.path.isdir(path): + raise ClusterDirError('Directory not found: %s' % path) + cluster_dir = os.path.join(path, 'cluster_' + profile) + return ClusterDir(cluster_dir) + + @classmethod + def find_cluster_dir_by_profile(cls, ipythondir, profile='default'): + """Find an existing cluster dir by profile name, return its ClusterDir. + + This searches through a sequence of paths for a cluster dir. If it + is not found, a :class:`ClusterDirError` exception will be raised. + + The search path algorithm is: + 1. ``os.getcwd()`` + 2. ``ipythondir`` + 3. The directories found in the ":" separated + :env:`IPCLUSTERDIR_PATH` environment variable. Parameters ---------- - path : unicode or str - The directory path to look for the cluster directory in. + ipythondir : unicode or str + The IPython directory to use. profile : unicode or str The name of the profile. The name of the cluster directory will be "cluster_". """ dirname = 'cluster_' + profile - cluster_dir = os.path.join(os.getcwd(), dirname) - if os.path.isdir(cluster_dir): - return ClusterDir(cluster_dir) + cluster_dir_paths = os.environ.get('IPCLUSTERDIR_PATH','') + if cluster_dir_paths: + cluster_dir_paths = cluster_dir_paths.split(':') else: - if not os.path.isdir(path): - raise IOError("Directory doesn't exist: %s" % path) - cluster_dir = os.path.join(path, dirname) - return ClusterDir(cluster_dir) + cluster_dir_paths = [] + paths = [os.getcwd(), ipythondir] + cluster_dir_paths + for p in paths: + cluster_dir = os.path.join(p, dirname) + if os.path.isdir(cluster_dir): + return ClusterDir(cluster_dir) + else: + raise ClusterDirError('Cluster directory not found in paths: %s' % dirname) @classmethod def find_cluster_dir(cls, cluster_dir): @@ -153,6 +197,8 @@ class ClusterDir(Component): :func:`os.path.expandvars` and :func:`os.path.expanduser`. """ cluster_dir = os.path.expandvars(os.path.expanduser(cluster_dir)) + if not os.path.isdir(cluster_dir): + raise ClusterDirError('Cluster directory not found: %s' % cluster_dir) return ClusterDir(cluster_dir) @@ -176,7 +222,8 @@ class AppWithClusterDirArgParseConfigLoader(ArgParseConfigLoader): self.parser.add_argument('-log_level', '--log-level', dest="Global.log_level",type=int, help='Set the log level (0,10,20,30,40,50). Default is 30.', - default=NoConfigDefault) + default=NoConfigDefault, + metavar="Global.log_level") self.parser.add_argument('-cluster_dir', '--cluster-dir', dest='Global.cluster_dir',type=str, help='Set the cluster dir. This overrides the logic used by the ' @@ -204,6 +251,8 @@ class ApplicationWithClusterDir(Application): dir and named the value of the ``config_file_name`` class attribute. """ + auto_create_cluster_dir = True + def create_default_config(self): super(ApplicationWithClusterDir, self).create_default_config() self.default_config.Global.profile = 'default' @@ -216,40 +265,68 @@ class ApplicationWithClusterDir(Application): version=release.version ) - def find_config_file_name(self): - """Find the config file name for this application.""" - # For this type of Application it should be set as a class attribute. - if not hasattr(self, 'config_file_name'): - self.log.critical("No config filename found") - - def find_config_file_paths(self): - """This resolves the cluster directory and sets ``config_file_paths``. + def find_resources(self): + """This resolves the cluster directory. - This does the following: - * Create the :class:`ClusterDir` object for the application. - * Set the ``cluster_dir`` attribute of the application and config + This tries to find the cluster directory and if successful, it will + have done: + * Sets ``self.cluster_dir_obj`` to the :class:`ClusterDir` object for + the application. + * Sets ``self.cluster_dir`` attribute of the application and config objects. - * Set ``config_file_paths`` to point to the cluster directory. + + The algorithm used for this is as follows: + 1. Try ``Global.cluster_dir``. + 2. Try using ``Global.profile``. + 3. If both of these fail and ``self.auto_create_cluster_dir`` is + ``True``, then create the new cluster dir in the IPython directory. + 4. If all fails, then raise :class:`ClusterDirError`. """ - # Create the ClusterDir object for managing everything try: cluster_dir = self.command_line_config.Global.cluster_dir except AttributeError: cluster_dir = self.default_config.Global.cluster_dir cluster_dir = os.path.expandvars(os.path.expanduser(cluster_dir)) - if cluster_dir: - # Just use cluster_dir + try: self.cluster_dir_obj = ClusterDir.find_cluster_dir(cluster_dir) + except ClusterDirError: + pass else: - # Then look for a profile - try: - self.profile = self.command_line_config.Global.profile - except AttributeError: - self.profile = self.default_config.Global.profile + self.log.info('Using existing cluster dir: %s' % \ + self.cluster_dir_obj.location + ) + self.finish_cluster_dir() + return + + try: + self.profile = self.command_line_config.Global.profile + except AttributeError: + self.profile = self.default_config.Global.profile + try: self.cluster_dir_obj = ClusterDir.find_cluster_dir_by_profile( self.ipythondir, self.profile) + except ClusterDirError: + pass + else: + self.log.info('Using existing cluster dir: %s' % \ + self.cluster_dir_obj.location + ) + self.finish_cluster_dir() + return + + if self.auto_create_cluster_dir: + self.cluster_dir_obj = ClusterDir.create_cluster_dir_by_profile( + self.ipythondir, self.profile + ) + self.log.info('Creating new cluster dir: %s' % \ + self.cluster_dir_obj.location + ) + self.finish_cluster_dir() + else: + raise ClusterDirError('Could not find a valid cluster directory.') + def finish_cluster_dir(self): # Set the cluster directory self.cluster_dir = self.cluster_dir_obj.location @@ -261,3 +338,15 @@ class ApplicationWithClusterDir(Application): # Set the search path to the cluster directory self.config_file_paths = (self.cluster_dir,) + + def find_config_file_name(self): + """Find the config file name for this application.""" + # For this type of Application it should be set as a class attribute. + if not hasattr(self, 'config_file_name'): + self.log.critical("No config filename found") + + def find_config_file_paths(self): + # Set the search path to the cluster directory + self.config_file_paths = (self.cluster_dir,) + + diff --git a/IPython/kernel/ipcontrollerapp.py b/IPython/kernel/ipcontrollerapp.py index 290c1b3..6c9ea3c 100644 --- a/IPython/kernel/ipcontrollerapp.py +++ b/IPython/kernel/ipcontrollerapp.py @@ -178,6 +178,7 @@ class IPControllerApp(ApplicationWithClusterDir): name = 'ipcontroller' description = 'Start the IPython controller for parallel computing.' config_file_name = default_config_file_name + auto_create_cluster_dir = True def create_default_config(self): super(IPControllerApp, self).create_default_config() diff --git a/IPython/kernel/ipengineapp.py b/IPython/kernel/ipengineapp.py index ade8089..a144795 100644 --- a/IPython/kernel/ipengineapp.py +++ b/IPython/kernel/ipengineapp.py @@ -92,6 +92,7 @@ class IPEngineApp(ApplicationWithClusterDir): name = 'ipengine' description = 'Start the IPython engine for parallel computing.' config_file_name = default_config_file_name + auto_create_cluster_dir = True def create_default_config(self): super(IPEngineApp, self).create_default_config() diff --git a/IPython/kernel/twistedutil.py b/IPython/kernel/twistedutil.py index 17c7dd5..76456d7 100644 --- a/IPython/kernel/twistedutil.py +++ b/IPython/kernel/twistedutil.py @@ -254,3 +254,12 @@ def sleep_deferred(seconds): d = defer.Deferred() reactor.callLater(seconds, d.callback, seconds) return d + + +def make_deferred(func): + """A decorator that calls a function with :func`maybeDeferred`.""" + + def _wrapper(*args, **kwargs): + return defer.maybeDeferred(func, *args, **kwargs) + + return _wrapper \ No newline at end of file