From 86c5a33dda9a68d953cb9e7e245f83485b696af4 2009-11-12 21:08:15 From: Brian Granger Date: 2009-11-12 21:08:15 Subject: [PATCH] Fixing how the working directory is handled in kernel. * Changed working_dir -> work_dir. * Now, ipcluster does not try to set the work dir for the controller and engine. To set it, just use the engine/controller config or pass the command line arg to the engine_args/controller_args attribute of a launcher. --- diff --git a/IPython/config/default/ipcluster_config.py b/IPython/config/default/ipcluster_config.py index 9a90de7..d391510 100644 --- a/IPython/config/default/ipcluster_config.py +++ b/IPython/config/default/ipcluster_config.py @@ -47,22 +47,19 @@ c = get_config() # The working directory for the process. The application will use os.chdir # to change to this directory before starting. -# c.Global.working_dir = os.getcwd() +# c.Global.work_dir = os.getcwd() #----------------------------------------------------------------------------- # Local process launchers #----------------------------------------------------------------------------- -# The working directory for the controller -# c.LocalControllerLauncher.working_dir = u'' - # The command line arguments to call the controller with. # c.LocalControllerLauncher.controller_args = \ # ['--log-to-file','--log-level', '40'] # The working directory for the controller -# c.LocalEngineSetLauncher.working_dir = u'' +# c.LocalEngineSetLauncher.work_dir = u'' # Command line argument passed to the engines. # c.LocalEngineSetLauncher.engine_args = ['--log-to-file','--log-level', '40'] @@ -71,9 +68,6 @@ c = get_config() # MPIExec launchers #----------------------------------------------------------------------------- -# The working directory for the controller -# c.MPIExecControllerLauncher.working_dir = u'' - # The mpiexec/mpirun command to use in started the controller. # c.MPIExecControllerLauncher.mpi_cmd = ['mpiexec'] @@ -85,9 +79,6 @@ c = get_config() # ['--log-to-file','--log-level', '40'] -# The working directory for the controller -# c.MPIExecEngineSetLauncher.working_dir = u'' - # The mpiexec/mpirun command to use in started the controller. # c.MPIExecEngineSetLauncher.mpi_cmd = ['mpiexec'] @@ -111,9 +102,6 @@ c = get_config() # Unix batch (PBS) schedulers launchers #----------------------------------------------------------------------------- -# The working directory for the controller -# c.PBSControllerLauncher.working_dir = u'' - # The command line program to use to submit a PBS job. # c.PBSControllerLauncher.submit_command = 'qsub' @@ -125,8 +113,8 @@ c = get_config() # The batch submission script used to start the controller. This is where # environment variables would be setup, etc. This string is interpolated using -# the Itpl module in IPython.external. Basically, you can use ${profile} for -# the controller profile or ${cluster_dir} for the cluster_dir. +# the Itpl module in IPython.external. Basically, you can use ${n} for the +# number of engine and ${cluster_dir} for the cluster_dir. # c.PBSControllerLauncher.batch_template = """""" # The name of the instantiated batch script that will actually be used to @@ -134,9 +122,6 @@ c = get_config() # c.PBSControllerLauncher.batch_file_name = u'pbs_batch_script_controller' -# The working directory for the controller -# c.PBSEngineSetLauncher.working_dir = u'' - # The command line program to use to submit a PBS job. # c.PBSEngineSetLauncher.submit_command = 'qsub' @@ -149,8 +134,7 @@ c = get_config() # The batch submission script used to start the engines. This is where # environment variables would be setup, etc. This string is interpolated using # the Itpl module in IPython.external. Basically, you can use ${n} for the -# number of engine, ${profile} or the engine profile and ${cluster_dir} -# for the cluster_dir. +# number of engine and ${cluster_dir} for the cluster_dir. # c.PBSEngineSetLauncher.batch_template = """""" # The name of the instantiated batch script that will actually be used to @@ -173,7 +157,6 @@ c = get_config() # c.IPControllerTask.controller_args = ['--log-to-file', '--log-level', '40'] # c.IPControllerTask.environment_variables = {} -# c.WindowsHPCControllerLauncher.working_dir = u'' # c.WindowsHPCControllerLauncher.scheduler = 'HEADNODE' # c.WindowsHPCControllerLauncher.job_file_name = u'ipcontroller_job.xml' @@ -190,7 +173,6 @@ c = get_config() # c.IPEngineTask.engine_args = ['--log-to-file', '--log-level', '40'] # c.IPEngineTask.environment_variables = {} -# c.WindowsHPCEngineSetLauncher.working_dir = u'' # c.WindowsHPCEngineSetLauncher.scheduler = 'HEADNODE' # c.WindowsHPCEngineSetLauncher.job_file_name = u'ipengineset_job.xml' diff --git a/IPython/config/default/ipcontroller_config.py b/IPython/config/default/ipcontroller_config.py index 5ecd2aa..c1d0bce 100644 --- a/IPython/config/default/ipcontroller_config.py +++ b/IPython/config/default/ipcontroller_config.py @@ -37,7 +37,7 @@ c = get_config() # The working directory for the process. The application will use os.chdir # to change to this directory before starting. -# c.Global.working_dir = os.getcwd() +# c.Global.work_dir = os.getcwd() #----------------------------------------------------------------------------- # Configure the client services diff --git a/IPython/config/default/ipengine_config.py b/IPython/config/default/ipengine_config.py index 08d4206..42483ed 100644 --- a/IPython/config/default/ipengine_config.py +++ b/IPython/config/default/ipengine_config.py @@ -36,7 +36,7 @@ c = get_config() # The working directory for the process. The application will use os.chdir # to change to this directory before starting. -# c.Global.working_dir = os.getcwd() +# c.Global.work_dir = os.getcwd() #----------------------------------------------------------------------------- # MPI configuration diff --git a/IPython/kernel/clusterdir.py b/IPython/kernel/clusterdir.py index 356cdf6..7168ea4 100644 --- a/IPython/kernel/clusterdir.py +++ b/IPython/kernel/clusterdir.py @@ -257,11 +257,11 @@ class AppWithClusterDirArgParseConfigLoader(ArgParseConfigLoader): default=NoConfigDefault, metavar='Global.cluster_dir' ), - self.parser.add_argument('--working-dir', - dest='Global.working_dir',type=unicode, + self.parser.add_argument('--work-dir', + dest='Global.work_dir',type=unicode, help='Set the working dir for the process.', default=NoConfigDefault, - metavar='Global.working_dir' + metavar='Global.work_dir' ) self.parser.add_argument('--clean-logs', dest='Global.clean_logs', action='store_true', @@ -299,7 +299,7 @@ class ApplicationWithClusterDir(Application): super(ApplicationWithClusterDir, self).create_default_config() self.default_config.Global.profile = u'default' self.default_config.Global.cluster_dir = u'' - self.default_config.Global.working_dir = os.getcwd() + self.default_config.Global.work_dir = os.getcwd() self.default_config.Global.log_to_file = False self.default_config.Global.clean_logs = False @@ -405,13 +405,13 @@ class ApplicationWithClusterDir(Application): pdir = self.cluster_dir_obj.pid_dir self.pid_dir = config.Global.pid_dir = pdir self.log.info("Cluster directory set to: %s" % self.cluster_dir) - config.Global.working_dir = unicode(genutils.expand_path(config.Global.working_dir)) + config.Global.work_dir = unicode(genutils.expand_path(config.Global.work_dir)) # Change to the working directory. We do this just before construct # is called so all the components there have the right working dir. - self.to_working_dir() + self.to_work_dir() - def to_working_dir(self): - wd = self.master_config.Global.working_dir + def to_work_dir(self): + wd = self.master_config.Global.work_dir if unicode(wd) != unicode(os.getcwd()): os.chdir(wd) self.log.info("Changing to working dir: %s" % wd) diff --git a/IPython/kernel/ipclusterapp.py b/IPython/kernel/ipclusterapp.py index 783833e..71fcaa7 100644 --- a/IPython/kernel/ipclusterapp.py +++ b/IPython/kernel/ipclusterapp.py @@ -86,11 +86,11 @@ class IPClusterCLLoader(ArgParseConfigLoader): '--profile option.', default=NoConfigDefault, metavar='Global.cluster_dir'), - parent_parser2.add_argument('--working-dir', - dest='Global.working_dir',type=unicode, + parent_parser2.add_argument('--work-dir', + dest='Global.work_dir',type=unicode, help='Set the working dir for the process.', default=NoConfigDefault, - metavar='Global.working_dir') + metavar='Global.work_dir') parent_parser2.add_argument('--log-to-file', action='store_true', dest='Global.log_to_file', default=NoConfigDefault, @@ -247,7 +247,7 @@ class IPClusterApp(ApplicationWithClusterDir): print start_cmd + " ==> " + full_path def pre_construct(self): - # This is where we cd to the working directory. + # IPClusterApp.pre_construct() is where we cd to the working directory. super(IPClusterApp, self).pre_construct() config = self.master_config try: @@ -272,14 +272,17 @@ class IPClusterApp(ApplicationWithClusterDir): def start_launchers(self): config = self.master_config - # Create the launchers + # Create the launchers. In both bases, we set the work_dir of + # the launcher to the cluster_dir. This is where the launcher's + # subprocesses will be launched. It is not where the controller + # and engine will be launched. el_class = import_item(config.Global.engine_launcher) self.engine_launcher = el_class( - self.cluster_dir, config=config + work_dir=self.cluster_dir, config=config ) cl_class = import_item(config.Global.controller_launcher) self.controller_launcher = cl_class( - self.cluster_dir, config=config + work_dir=self.cluster_dir, config=config ) # Setup signals diff --git a/IPython/kernel/launcher.py b/IPython/kernel/launcher.py index 7c9e4aa..e14d9d1 100644 --- a/IPython/kernel/launcher.py +++ b/IPython/kernel/launcher.py @@ -99,11 +99,18 @@ class UnknownStatus(LauncherError): class BaseLauncher(Component): """An asbtraction for starting, stopping and signaling a process.""" - working_dir = Unicode(u'') - - def __init__(self, working_dir, parent=None, name=None, config=None): + # In all of the launchers, the work_dir is where child processes will be + # run. This will usually be the cluster_dir, but may not be. any work_dir + # passed into the __init__ method will override the config value. + # This should not be used to set the work_dir for the actual engine + # and controller. Instead, use their own config files or the + # controller_args, engine_args attributes of the launchers to add + # the --work-dir option. + work_dir = Unicode(u'') + + def __init__(self, work_dir, parent=None, name=None, config=None): super(BaseLauncher, self).__init__(parent, name, config) - self.working_dir = working_dir + self.work_dir = work_dir self.state = 'before' # can be before, running, after self.stop_deferreds = [] self.start_data = None @@ -270,16 +277,16 @@ class LocalProcessLauncher(BaseLauncher): """Start and stop an external process in an asynchronous manner. This will launch the external process with a working directory of - ``self.working_dir``. + ``self.work_dir``. """ # This is used to to construct self.args, which is passed to # spawnProcess. cmd_and_args = List([]) - def __init__(self, working_dir, parent=None, name=None, config=None): + def __init__(self, work_dir, parent=None, name=None, config=None): super(LocalProcessLauncher, self).__init__( - working_dir, parent, name, config + work_dir, parent, name, config ) self.process_protocol = None self.start_deferred = None @@ -296,7 +303,7 @@ class LocalProcessLauncher(BaseLauncher): str(self.args[0]), # twisted expects these to be str, not unicode [str(a) for a in self.args], # str expected, not unicode env=os.environ, - path=self.working_dir # start in the working_dir + path=self.work_dir # start in the work_dir ) return self.start_deferred else: @@ -331,8 +338,7 @@ class LocalControllerLauncher(LocalProcessLauncher): controller_args = List(['--log-to-file','--log-level', '40'], config=True) def find_args(self): - return self.controller_cmd + self.controller_args + \ - ['--working-dir', self.working_dir] + return self.controller_cmd + self.controller_args def start(self, cluster_dir): """Start the controller by cluster_dir.""" @@ -352,8 +358,7 @@ class LocalEngineLauncher(LocalProcessLauncher): ) def find_args(self): - return self.engine_cmd + self.engine_args + \ - ['--working-dir', self.working_dir] + return self.engine_cmd + self.engine_args def start(self, cluster_dir): """Start the engine by cluster_dir.""" @@ -370,9 +375,9 @@ class LocalEngineSetLauncher(BaseLauncher): ['--log-to-file','--log-level', '40'], config=True ) - def __init__(self, working_dir, parent=None, name=None, config=None): + def __init__(self, work_dir, parent=None, name=None, config=None): super(LocalEngineSetLauncher, self).__init__( - working_dir, parent, name, config + work_dir, parent, name, config ) self.launchers = [] @@ -381,7 +386,7 @@ class LocalEngineSetLauncher(BaseLauncher): self.cluster_dir = unicode(cluster_dir) dlist = [] for i in range(n): - el = LocalEngineLauncher(self.working_dir, self) + el = LocalEngineLauncher(self.work_dir, self) # Copy the engine args over to each engine launcher. import copy el.engine_args = copy.deepcopy(self.engine_args) @@ -471,8 +476,7 @@ class MPIExecControllerLauncher(MPIExecLauncher): def find_args(self): return self.mpi_cmd + ['-n', self.n] + self.mpi_args + \ - self.controller_cmd + self.controller_args + \ - ['--working-dir', self.working_dir] + self.controller_cmd + self.controller_args class MPIExecEngineSetLauncher(MPIExecLauncher): @@ -481,20 +485,20 @@ class MPIExecEngineSetLauncher(MPIExecLauncher): # Command line arguments for ipengine. engine_args = List( ['--log-to-file','--log-level', '40'], config=True - ) + ) n = Int(1, config=True) def start(self, n, cluster_dir): """Start n engines by profile or cluster_dir.""" self.engine_args.extend(['--cluster-dir', cluster_dir]) self.cluster_dir = unicode(cluster_dir) + self.n = n log.msg('Starting MPIExecEngineSetLauncher: %r' % self.args) return super(MPIExecEngineSetLauncher, self).start(n) def find_args(self): return self.mpi_cmd + ['-n', self.n] + self.mpi_args + \ - self.engine_cmd + self.engine_args + \ - ['--working-dir', self.working_dir] + self.engine_cmd + self.engine_args #----------------------------------------------------------------------------- @@ -565,20 +569,20 @@ class WindowsHPCLauncher(BaseLauncher): # The filename of the instantiated job script. job_file_name = Unicode(u'ipython_job.xml', config=True) # The full path to the instantiated job script. This gets made dynamically - # by combining the working_dir with the job_file_name. + # by combining the work_dir with the job_file_name. job_file = Unicode(u'') # The hostname of the scheduler to submit the job to scheduler = Str('', config=True) job_cmd = Str(find_job_cmd(), config=True) - def __init__(self, working_dir, parent=None, name=None, config=None): + def __init__(self, work_dir, parent=None, name=None, config=None): super(WindowsHPCLauncher, self).__init__( - working_dir, parent, name, config + work_dir, parent, name, config ) @property def job_file(self): - return os.path.join(self.working_dir, self.job_file_name) + return os.path.join(self.work_dir, self.job_file_name) def write_job_file(self, n): raise NotImplementedError("Implement write_job_file in a subclass.") @@ -611,7 +615,7 @@ class WindowsHPCLauncher(BaseLauncher): output = yield getProcessOutput(str(self.job_cmd), [str(a) for a in args], env=dict((str(k),str(v)) for k,v in os.environ.items()), - path=self.working_dir + path=self.work_dir ) job_id = self.parse_job_id(output) self.notify_start(job_id) @@ -630,7 +634,7 @@ class WindowsHPCLauncher(BaseLauncher): output = yield getProcessOutput(str(self.job_cmd), [str(a) for a in args], env=dict((str(k),str(v)) for k,v in os.environ.items()), - path=self.working_dir + path=self.work_dir ) except: output = 'The job already appears to be stoppped: %r' % self.job_id @@ -651,7 +655,7 @@ class WindowsHPCControllerLauncher(WindowsHPCLauncher): # the controller. It is used as the base path for the stdout/stderr # files that the scheduler redirects to. t.work_directory = self.cluster_dir - # Add the --cluster-dir and --working-dir from self.start(). + # Add the --cluster-dir and from self.start(). t.controller_args.extend(self.extra_args) job.add_task(t) @@ -664,9 +668,7 @@ class WindowsHPCControllerLauncher(WindowsHPCLauncher): def start(self, cluster_dir): """Start the controller by cluster_dir.""" - self.extra_args = [ - '--cluster-dir', cluster_dir, '--working-dir', self.working_dir - ] + self.extra_args = ['--cluster-dir', cluster_dir] self.cluster_dir = unicode(cluster_dir) return super(WindowsHPCControllerLauncher, self).start(1) @@ -685,7 +687,7 @@ class WindowsHPCEngineSetLauncher(WindowsHPCLauncher): # the engine. It is used as the base path for the stdout/stderr # files that the scheduler redirects to. t.work_directory = self.cluster_dir - # Add the --cluster-dir and --working-dir from self.start(). + # Add the --cluster-dir and from self.start(). t.engine_args.extend(self.extra_args) job.add_task(t) @@ -698,9 +700,7 @@ class WindowsHPCEngineSetLauncher(WindowsHPCLauncher): def start(self, n, cluster_dir): """Start the controller by cluster_dir.""" - self.extra_args = [ - '--cluster-dir', cluster_dir, '--working-dir', self.working_dir - ] + self.extra_args = ['--cluster-dir', cluster_dir] self.cluster_dir = unicode(cluster_dir) return super(WindowsHPCEngineSetLauncher, self).start(n) @@ -739,11 +739,11 @@ class BatchSystemLauncher(BaseLauncher): # The full path to the instantiated batch script. batch_file = Unicode(u'') - def __init__(self, working_dir, parent=None, name=None, config=None): + def __init__(self, work_dir, parent=None, name=None, config=None): super(BatchSystemLauncher, self).__init__( - working_dir, parent, name, config + work_dir, parent, name, config ) - self.batch_file = os.path.join(self.working_dir, self.batch_file_name) + self.batch_file = os.path.join(self.work_dir, self.batch_file_name) self.context = {} def parse_job_id(self, output): @@ -758,7 +758,7 @@ class BatchSystemLauncher(BaseLauncher): return job_id def write_batch_script(self, n): - """Instantiate and write the batch script to the working_dir.""" + """Instantiate and write the batch script to the work_dir.""" self.context['n'] = n script_as_string = Itpl.itplns(self.batch_template, self.context) log.msg('Writing instantiated batch script: %s' % self.batch_file) diff --git a/IPython/quarantine/InterpreterExec.py b/IPython/quarantine/InterpreterExec.py index f3f4ede..70bebac 100644 --- a/IPython/quarantine/InterpreterExec.py +++ b/IPython/quarantine/InterpreterExec.py @@ -14,7 +14,7 @@ executed. # the file COPYING, distributed as part of this software. #***************************************************************************** -# TODO: deprecated + def prefilter_shell(self,line,continuation): """Alternate prefilter, modified for shell-like functionality.