adjustments to PBS/SGE, SSH Launchers + docs update

IPython/config/default/ipclusterz_config.py

@@ -1,192 +1,227 b''
 import os
 
 c = get_config()
 
 #-----------------------------------------------------------------------------
 # Select which launchers to use
 #-----------------------------------------------------------------------------
 
 # This allows you to control what method is used to start the controller
 # and engines.  The following methods are currently supported:
 # - Start as a regular process on localhost.
 # - Start using mpiexec.
 # - Start using the Windows HPC Server 2008 scheduler
 # - Start using PBS
 # - Start using SSH
 
 
 # The selected launchers can be configured below.
 
 # Options are:
 # - LocalControllerLauncher
 # - MPIExecControllerLauncher
 # - PBSControllerLauncher
 # - WindowsHPCControllerLauncher
 # c.Global.controller_launcher = 'IPython.zmq.parallel.launcher.LocalControllerLauncher'
 
 # Options are:
 # - LocalEngineSetLauncher
 # - MPIExecEngineSetLauncher
 # - PBSEngineSetLauncher
 # - WindowsHPCEngineSetLauncher
 # c.Global.engine_launcher = 'IPython.zmq.parallel.launcher.LocalEngineSetLauncher'
 
 #-----------------------------------------------------------------------------
 # Global configuration
 #-----------------------------------------------------------------------------
 
 # The default number of engines that will be started. This is overridden by
 # the -n command line option: "ipcluster start -n 4"
 # c.Global.n = 2
 
 # Log to a file in cluster_dir/log, otherwise just log to sys.stdout.
 # c.Global.log_to_file = False
 
 # Remove old logs from cluster_dir/log before starting.
 # c.Global.clean_logs = True
 
 # The working directory for the process. The application will use os.chdir
 # to change to this directory before starting.
 # c.Global.work_dir = os.getcwd()
 
 
 #-----------------------------------------------------------------------------
 # Local process launchers
 #-----------------------------------------------------------------------------
 
 # The command line arguments to call the controller with.
 # c.LocalControllerLauncher.controller_args = \
 #    ['--log-to-file','--log-level', '40']
 
 # The working directory for the controller
 # c.LocalEngineSetLauncher.work_dir = u''
 
 # Command line argument passed to the engines.
 # c.LocalEngineSetLauncher.engine_args = ['--log-to-file','--log-level', '40']
 
 #-----------------------------------------------------------------------------
 # MPIExec launchers
 #-----------------------------------------------------------------------------
 
-# The mpiexec/mpirun command to use in started the controller.
-# c.MPIExecControllerLauncher.mpi_cmd = ['mpiexec']
+# The mpiexec/mpirun command to use in both the controller and engines.
+# c.MPIExecLauncher.mpi_cmd = ['mpiexec']
 
 # Additional arguments to pass to the actual mpiexec command.
+# c.MPIExecLauncher.mpi_args = []
+
+# The mpiexec/mpirun command and args can be overridden if they should be different
+# for controller and engines.
+# c.MPIExecControllerLauncher.mpi_cmd = ['mpiexec']
 # c.MPIExecControllerLauncher.mpi_args = []
+# c.MPIExecEngineSetLauncher.mpi_cmd = ['mpiexec']
+# c.MPIExecEngineSetLauncher.mpi_args = []
 
 # The command line argument to call the controller with.
 # c.MPIExecControllerLauncher.controller_args = \
 #     ['--log-to-file','--log-level', '40']
 
-
-# The mpiexec/mpirun command to use in started the controller.
-# c.MPIExecEngineSetLauncher.mpi_cmd = ['mpiexec']
-
-# Additional arguments to pass to the actual mpiexec command.
-# c.MPIExecEngineSetLauncher.mpi_args = []
-
 # Command line argument passed to the engines.
 # c.MPIExecEngineSetLauncher.engine_args = ['--log-to-file','--log-level', '40']
 
 # The default number of engines to start if not given elsewhere.
 # c.MPIExecEngineSetLauncher.n = 1
 
 #-----------------------------------------------------------------------------
 # SSH launchers
 #-----------------------------------------------------------------------------
 
-# Todo
+# ipclusterz can be used to launch controller and engines remotely via ssh.
+# Note that currently ipclusterz does not do any file distribution, so if
+# machines are not on a shared filesystem, config and json files must be
+# distributed.  For this reason, the reuse_files defaults to True on an
+# ssh-launched Controller.  This flag can be overridded by the program_args
+# attribute of c.SSHControllerLauncher.
+
+# set the ssh cmd for launching remote commands. The default is ['ssh']
+# c.SSHLauncher.ssh_cmd = ['ssh']
+
+# set the ssh cmd for launching remote commands. The default is ['ssh']
+# c.SSHLauncher.ssh_args = ['tt']
+
+# Set the user and hostname for the controller
+# c.SSHControllerLauncher.hostname = 'controller.example.com'
+# c.SSHControllerLauncher.user = os.environ.get('USER','username')
 
+# Set the arguments to be passed to ipcontrollerz
+# note that remotely launched ipcontrollerz will not get the contents of 
+# the local ipcontrollerz_config.py unless it resides on the *remote host*
+# in the location specified by the --cluster_dir argument.
+# c.SSHControllerLauncher.program_args = ['-r', '-ip', '0.0.0.0', '--cluster_dir', '/path/to/cd']
+
+# Set the default args passed to ipenginez for SSH launched engines
+# c.SSHEngineSetLauncher.engine_args = ['--mpi', 'mpi4py']
+
+# SSH engines are launched as a dict of locations/n-engines.
+# if a value is a tuple instead of an int, it is assumed to be of the form
+# (n, [args]), setting the arguments to passed to ipenginez on `host`.
+# otherwise, c.SSHEngineSetLauncher.engine_args will be used as the default.
+
+# In this case, there will be 3 engines at my.example.com, and
+# 2 at [e-mail] with a special json connector location.
+# c.SSHEngineSetLauncher.engines = {'my.example.com' : 3,
+#                                   '[e-mail]' : (2, ['-f', '/path/to/ipcontroller-engine.json']}
+#                                  }
 
 #-----------------------------------------------------------------------------
 # Unix batch (PBS) schedulers launchers
 #-----------------------------------------------------------------------------
 
 # The command line program to use to submit a PBS job.
-# c.PBSControllerLauncher.submit_command = 'qsub'
+# c.PBSControllerLauncher.submit_command = ['qsub']
 
 # The command line program to use to delete a PBS job.
-# c.PBSControllerLauncher.delete_command = 'qdel'
+# c.PBSControllerLauncher.delete_command = ['qdel']
 
 # A regular expression that takes the output of qsub and find the job id.
 # c.PBSControllerLauncher.job_id_regexp = r'\d+'
 
 # The batch submission script used to start the controller. This is where
-# environment variables would be setup, etc. This string is interpolated using
+# environment variables would be setup, etc. This string is interpreted using
 # the Itpl module in IPython.external. Basically, you can use ${n} for the
 # number of engine and ${cluster_dir} for the cluster_dir.
 # c.PBSControllerLauncher.batch_template = """
-# #PBS -l nprocs=$n
+# #PBS -N ipcontroller
 # 
 # ipcontrollerz --cluster-dir $cluster_dir
 # """
 
 # The name of the instantiated batch script that will actually be used to
 # submit the job. This will be written to the cluster directory.
 # c.PBSControllerLauncher.batch_file_name = u'pbs_batch_script_controller'
 
 
 # The command line program to use to submit a PBS job.
 # c.PBSEngineSetLauncher.submit_command = 'qsub'
 
 # The command line program to use to delete a PBS job.
 # c.PBSEngineSetLauncher.delete_command = 'qdel'
 
 # A regular expression that takes the output of qsub and find the job id.
 # c.PBSEngineSetLauncher.job_id_regexp = r'\d+'
 
 # The batch submission script used to start the engines. This is where
-# environment variables would be setup, etc. This string is interpolated using
+# environment variables would be setup, etc. This string is interpreted using
 # the Itpl module in IPython.external. Basically, you can use ${n} for the
 # number of engine and ${cluster_dir} for the cluster_dir.
 # c.PBSEngineSetLauncher.batch_template = """
+# #PBS -N ipcontroller
 # #PBS -l nprocs=$n
 # 
 # ipenginez --cluster-dir $cluster_dir$s
 # """
 
 # The name of the instantiated batch script that will actually be used to
 # submit the job. This will be written to the cluster directory.
 # c.PBSEngineSetLauncher.batch_file_name = u'pbs_batch_script_engines'
 
 #-----------------------------------------------------------------------------
 # Windows HPC Server 2008 launcher configuration
 #-----------------------------------------------------------------------------
 
 # c.IPControllerJob.job_name = 'IPController'
 # c.IPControllerJob.is_exclusive = False
 # c.IPControllerJob.username = r'USERDOMAIN\USERNAME'
 # c.IPControllerJob.priority = 'Highest'
 # c.IPControllerJob.requested_nodes = ''
 # c.IPControllerJob.project = 'MyProject'
 
 # c.IPControllerTask.task_name = 'IPController'
 # c.IPControllerTask.controller_cmd = [u'ipcontroller.exe']
 # c.IPControllerTask.controller_args = ['--log-to-file', '--log-level', '40']
 # c.IPControllerTask.environment_variables = {}
 
 # c.WindowsHPCControllerLauncher.scheduler = 'HEADNODE'
 # c.WindowsHPCControllerLauncher.job_file_name = u'ipcontroller_job.xml'
 
 
 # c.IPEngineSetJob.job_name = 'IPEngineSet'
 # c.IPEngineSetJob.is_exclusive = False
 # c.IPEngineSetJob.username = r'USERDOMAIN\USERNAME'
 # c.IPEngineSetJob.priority = 'Highest'
 # c.IPEngineSetJob.requested_nodes = ''
 # c.IPEngineSetJob.project = 'MyProject'
 
 # c.IPEngineTask.task_name = 'IPEngine'
 # c.IPEngineTask.engine_cmd = [u'ipengine.exe']
 # c.IPEngineTask.engine_args = ['--log-to-file', '--log-level', '40']
 # c.IPEngineTask.environment_variables = {}
 
 # c.WindowsHPCEngineSetLauncher.scheduler = 'HEADNODE'
 # c.WindowsHPCEngineSetLauncher.job_file_name = u'ipengineset_job.xml'
 
 
 
 
 
 
 

IPython/zmq/parallel/__init__.py

@@ -1,18 +1,18 b''
 """The IPython ZMQ-based parallel computing interface."""
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2011 The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
-from .asyncresult import *
-from .client import Client
-from .dependency import *
-from .remotefunction import *
-from .view import *
+# from .asyncresult import *
+# from .client import Client
+# from .dependency import *
+# from .remotefunction import *
+# from .view import *
 

IPython/zmq/parallel/launcher.py

@@ -1,847 +1,879 b''
 #!/usr/bin/env python
 # encoding: utf-8
 """
 Facilities for launching IPython processes asynchronously.
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008-2009  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
+import copy
 import logging
 import os
 import re
-import sys
 
 from signal import SIGINT, SIGTERM
 try:
     from signal import SIGKILL
 except ImportError:
     SIGKILL=SIGTERM
 
 from subprocess import Popen, PIPE, STDOUT
 try:
     from subprocess import check_output
 except ImportError:
-    # pre-2.7:
-    from StringIO import StringIO
-    
+    # pre-2.7, define check_output with Popen
     def check_output(*args, **kwargs):
-        sio = StringIO()
-        kwargs.update(dict(stdout=PIPE, stderr=STDOUT))
+        kwargs.update(dict(stdout=PIPE))
         p = Popen(*args, **kwargs)
         out,err = p.communicate()
         return out
 
 from zmq.eventloop import ioloop
 
 from IPython.external import Itpl
 # from IPython.config.configurable import Configurable
-from IPython.utils.traitlets import Str, Int, List, Unicode, Dict, Instance
+from IPython.utils.traitlets import Any, Str, Int, List, Unicode, Dict, Instance
 from IPython.utils.path import get_ipython_module_path
 from IPython.utils.process import find_cmd, pycmd2argv, FindCmdError
 
 from .factory import LoggingFactory
 
 # load winhpcjob from IPython.kernel
 try:
     from IPython.kernel.winhpcjob import (
         IPControllerTask, IPEngineTask,
         IPControllerJob, IPEngineSetJob
     )
 except ImportError:
     pass
 
 
 #-----------------------------------------------------------------------------
 # Paths to the kernel apps
 #-----------------------------------------------------------------------------
 
 
 ipcluster_cmd_argv = pycmd2argv(get_ipython_module_path(
     'IPython.zmq.parallel.ipclusterapp'
 ))
 
 ipengine_cmd_argv = pycmd2argv(get_ipython_module_path(
     'IPython.zmq.parallel.ipengineapp'
 ))
 
 ipcontroller_cmd_argv = pycmd2argv(get_ipython_module_path(
     'IPython.zmq.parallel.ipcontrollerapp'
 ))
 
 #-----------------------------------------------------------------------------
 # Base launchers and errors
 #-----------------------------------------------------------------------------
 
 
 class LauncherError(Exception):
     pass
 
 
 class ProcessStateError(LauncherError):
     pass
 
 
 class UnknownStatus(LauncherError):
     pass
 
 
 class BaseLauncher(LoggingFactory):
     """An asbtraction for starting, stopping and signaling a process."""
 
     # In all of the launchers, the work_dir is where child processes will be
     # run. This will usually be the cluster_dir, but may not be. any work_dir
     # passed into the __init__ method will override the config value.
     # This should not be used to set the work_dir for the actual engine
     # and controller. Instead, use their own config files or the
     # controller_args, engine_args attributes of the launchers to add
     # the --work-dir option.
     work_dir = Unicode(u'.')
     loop = Instance('zmq.eventloop.ioloop.IOLoop')
+    
+    start_data = Any()
+    stop_data = Any()
+    
     def _loop_default(self):
         return ioloop.IOLoop.instance()
 
     def __init__(self, work_dir=u'.', config=None, **kwargs):
         super(BaseLauncher, self).__init__(work_dir=work_dir, config=config, **kwargs)
         self.state = 'before' # can be before, running, after
         self.stop_callbacks = []
         self.start_data = None
         self.stop_data = None
 
     @property
     def args(self):
         """A list of cmd and args that will be used to start the process.
 
         This is what is passed to :func:`spawnProcess` and the first element
         will be the process name.
         """
         return self.find_args()
 
     def find_args(self):
         """The ``.args`` property calls this to find the args list.
 
         Subcommand should implement this to construct the cmd and args.
         """
         raise NotImplementedError('find_args must be implemented in a subclass')
 
     @property
     def arg_str(self):
         """The string form of the program arguments."""
         return ' '.join(self.args)
 
     @property
     def running(self):
         """Am I running."""
         if self.state == 'running':
             return True
         else:
             return False
 
     def start(self):
         """Start the process.
 
         This must return a deferred that fires with information about the
         process starting (like a pid, job id, etc.).
         """
         raise NotImplementedError('start must be implemented in a subclass')
 
     def stop(self):
         """Stop the process and notify observers of stopping.
 
         This must return a deferred that fires with information about the
         processing stopping, like errors that occur while the process is
         attempting to be shut down. This deferred won't fire when the process
         actually stops. To observe the actual process stopping, see
         :func:`observe_stop`.
         """
         raise NotImplementedError('stop must be implemented in a subclass')
 
     def on_stop(self, f):
         """Get a deferred that will fire when the process stops.
 
         The deferred will fire with data that contains information about
         the exit status of the process.
         """
         if self.state=='after':
             return f(self.stop_data)
         else:
             self.stop_callbacks.append(f)
 
     def notify_start(self, data):
         """Call this to trigger startup actions.
 
         This logs the process startup and sets the state to 'running'.  It is
         a pass-through so it can be used as a callback.
         """
 
         self.log.info('Process %r started: %r' % (self.args[0], data))
         self.start_data = data
         self.state = 'running'
         return data
 
     def notify_stop(self, data):
         """Call this to trigger process stop actions.
 
         This logs the process stopping and sets the state to 'after'. Call
         this to trigger all the deferreds from :func:`observe_stop`."""
 
         self.log.info('Process %r stopped: %r' % (self.args[0], data))
         self.stop_data = data
         self.state = 'after'
         for i in range(len(self.stop_callbacks)):
             d = self.stop_callbacks.pop()
             d(data)
         return data
 
     def signal(self, sig):
         """Signal the process.
 
         Return a semi-meaningless deferred after signaling the process.
 
         Parameters
         ----------
         sig : str or int
             'KILL', 'INT', etc., or any signal number
         """
         raise NotImplementedError('signal must be implemented in a subclass')
 
 
 #-----------------------------------------------------------------------------
 # Local process launchers
 #-----------------------------------------------------------------------------
 
 
 class LocalProcessLauncher(BaseLauncher):
     """Start and stop an external process in an asynchronous manner.
 
     This will launch the external process with a working directory of
     ``self.work_dir``.
     """
 
     # This is used to to construct self.args, which is passed to 
     # spawnProcess.
     cmd_and_args = List([])
     poll_frequency = Int(100) # in ms
 
     def __init__(self, work_dir=u'.', config=None, **kwargs):
         super(LocalProcessLauncher, self).__init__(
             work_dir=work_dir, config=config, **kwargs
         )
         self.process = None
         self.start_deferred = None
         self.poller = None
 
     def find_args(self):
         return self.cmd_and_args
 
     def start(self):
         if self.state == 'before':
             self.process = Popen(self.args,
                 stdout=PIPE,stderr=PIPE,stdin=PIPE,
                 env=os.environ,
                 cwd=self.work_dir
             )
             
             self.loop.add_handler(self.process.stdout.fileno(), self.handle_stdout, self.loop.READ)
             self.loop.add_handler(self.process.stderr.fileno(), self.handle_stderr, self.loop.READ)
             self.poller = ioloop.PeriodicCallback(self.poll, self.poll_frequency, self.loop)
             self.poller.start()
             self.notify_start(self.process.pid)
         else:
             s = 'The process was already started and has state: %r' % self.state
             raise ProcessStateError(s)
 
     def stop(self):
         return self.interrupt_then_kill()
 
     def signal(self, sig):
         if self.state == 'running':
             self.process.send_signal(sig)
 
     def interrupt_then_kill(self, delay=2.0):
         """Send INT, wait a delay and then send KILL."""
         self.signal(SIGINT)
         self.killer  = ioloop.DelayedCallback(lambda : self.signal(SIGKILL), delay*1000, self.loop)
         self.killer.start()
 
     # callbacks, etc:
     
     def handle_stdout(self, fd, events):
         line = self.process.stdout.readline()
         # a stopped process will be readable but return empty strings
         if line:
             self.log.info(line[:-1])
         else:
             self.poll()
     
     def handle_stderr(self, fd, events):
         line = self.process.stderr.readline()
         # a stopped process will be readable but return empty strings
         if line:
             self.log.error(line[:-1])
         else:
             self.poll()
     
     def poll(self):
         status = self.process.poll()
         if status is not None:
             self.poller.stop()
             self.loop.remove_handler(self.process.stdout.fileno())
             self.loop.remove_handler(self.process.stderr.fileno())
             self.notify_stop(dict(exit_code=status, pid=self.process.pid))
         return status
 
 class LocalControllerLauncher(LocalProcessLauncher):
     """Launch a controller as a regular external process."""
 
     controller_cmd = List(ipcontroller_cmd_argv, config=True)
     # Command line arguments to ipcontroller.
     controller_args = List(['--log-to-file','--log-level', str(logging.INFO)], config=True)
 
     def find_args(self):
         return self.controller_cmd + self.controller_args
 
     def start(self, cluster_dir):
         """Start the controller by cluster_dir."""
         self.controller_args.extend(['--cluster-dir', cluster_dir])
         self.cluster_dir = unicode(cluster_dir)
         self.log.info("Starting LocalControllerLauncher: %r" % self.args)
         return super(LocalControllerLauncher, self).start()
 
 
 class LocalEngineLauncher(LocalProcessLauncher):
     """Launch a single engine as a regular externall process."""
 
     engine_cmd = List(ipengine_cmd_argv, config=True)
     # Command line arguments for ipengine.
     engine_args = List(
         ['--log-to-file','--log-level', str(logging.INFO)], config=True
     )
 
     def find_args(self):
         return self.engine_cmd + self.engine_args
 
     def start(self, cluster_dir):
         """Start the engine by cluster_dir."""
         self.engine_args.extend(['--cluster-dir', cluster_dir])
         self.cluster_dir = unicode(cluster_dir)
         return super(LocalEngineLauncher, self).start()
 
 
 class LocalEngineSetLauncher(BaseLauncher):
     """Launch a set of engines as regular external processes."""
 
     # Command line arguments for ipengine.
     engine_args = List(
         ['--log-to-file','--log-level', str(logging.INFO)], config=True
     )
     # launcher class
     launcher_class = LocalEngineLauncher
     
+    launchers = Dict()
+    stop_data = Dict()
+    
     def __init__(self, work_dir=u'.', config=None, **kwargs):
         super(LocalEngineSetLauncher, self).__init__(
             work_dir=work_dir, config=config, **kwargs
         )
-        self.launchers = {}
         self.stop_data = {}
 
     def start(self, n, cluster_dir):
         """Start n engines by profile or cluster_dir."""
         self.cluster_dir = unicode(cluster_dir)
         dlist = []
         for i in range(n):
             el = self.launcher_class(work_dir=self.work_dir, config=self.config, logname=self.log.name)
             # Copy the engine args over to each engine launcher.
-            import copy
             el.engine_args = copy.deepcopy(self.engine_args)
             el.on_stop(self._notice_engine_stopped)
             d = el.start(cluster_dir)
             if i==0:
                 self.log.info("Starting LocalEngineSetLauncher: %r" % el.args)
             self.launchers[i] = el
             dlist.append(d)
         self.notify_start(dlist)
         # The consumeErrors here could be dangerous
         # dfinal = gatherBoth(dlist, consumeErrors=True)
         # dfinal.addCallback(self.notify_start)
         return dlist
 
     def find_args(self):
         return ['engine set']
 
     def signal(self, sig):
         dlist = []
         for el in self.launchers.itervalues():
             d = el.signal(sig)
             dlist.append(d)
         # dfinal = gatherBoth(dlist, consumeErrors=True)
         return dlist
 
     def interrupt_then_kill(self, delay=1.0):
         dlist = []
         for el in self.launchers.itervalues():
             d = el.interrupt_then_kill(delay)
             dlist.append(d)
         # dfinal = gatherBoth(dlist, consumeErrors=True)
         return dlist
 
     def stop(self):
         return self.interrupt_then_kill()
     
     def _notice_engine_stopped(self, data):
-        print "notice", data
         pid = data['pid']
         for idx,el in self.launchers.iteritems():
             if el.process.pid == pid:
                 break
         self.launchers.pop(idx)
         self.stop_data[idx] = data
         if not self.launchers:
             self.notify_stop(self.stop_data)
 
 
 #-----------------------------------------------------------------------------
 # MPIExec launchers
 #-----------------------------------------------------------------------------
 
 
 class MPIExecLauncher(LocalProcessLauncher):
     """Launch an external process using mpiexec."""
 
     # The mpiexec command to use in starting the process.
     mpi_cmd = List(['mpiexec'], config=True)
     # The command line arguments to pass to mpiexec.
     mpi_args = List([], config=True)
     # The program to start using mpiexec.
     program = List(['date'], config=True)
     # The command line argument to the program.
     program_args = List([], config=True)
     # The number of instances of the program to start.
     n = Int(1, config=True)
 
     def find_args(self):
         """Build self.args using all the fields."""
-        return self.mpi_cmd + ['-n', self.n] + self.mpi_args + \
+        return self.mpi_cmd + ['-n', str(self.n)] + self.mpi_args + \
                self.program + self.program_args
 
     def start(self, n):
         """Start n instances of the program using mpiexec."""
         self.n = n
         return super(MPIExecLauncher, self).start()
 
 
 class MPIExecControllerLauncher(MPIExecLauncher):
     """Launch a controller using mpiexec."""
 
     controller_cmd = List(ipcontroller_cmd_argv, config=True)
     # Command line arguments to ipcontroller.
     controller_args = List(['--log-to-file','--log-level', str(logging.INFO)], config=True)
     n = Int(1, config=False)
 
     def start(self, cluster_dir):
         """Start the controller by cluster_dir."""
         self.controller_args.extend(['--cluster-dir', cluster_dir])
         self.cluster_dir = unicode(cluster_dir)
         self.log.info("Starting MPIExecControllerLauncher: %r" % self.args)
         return super(MPIExecControllerLauncher, self).start(1)
 
     def find_args(self):
         return self.mpi_cmd + ['-n', self.n] + self.mpi_args + \
                self.controller_cmd + self.controller_args
 
 
 class MPIExecEngineSetLauncher(MPIExecLauncher):
 
-    engine_cmd = List(ipengine_cmd_argv, config=True)
+    program = List(ipengine_cmd_argv, config=True)
     # Command line arguments for ipengine.
-    engine_args = List(
+    program_args = List(
         ['--log-to-file','--log-level', str(logging.INFO)], config=True
     )
     n = Int(1, config=True)
 
     def start(self, n, cluster_dir):
         """Start n engines by profile or cluster_dir."""
-        self.engine_args.extend(['--cluster-dir', cluster_dir])
+        self.program_args.extend(['--cluster-dir', cluster_dir])
         self.cluster_dir = unicode(cluster_dir)
         self.n = n
         self.log.info('Starting MPIExecEngineSetLauncher: %r' % self.args)
         return super(MPIExecEngineSetLauncher, self).start(n)
 
-    def find_args(self):
-        return self.mpi_cmd + ['-n', self.n] + self.mpi_args + \
-               self.engine_cmd + self.engine_args
-
-
 #-----------------------------------------------------------------------------
 # SSH launchers
 #-----------------------------------------------------------------------------
 
 # TODO: Get SSH Launcher working again.
 
 class SSHLauncher(LocalProcessLauncher):
     """A minimal launcher for ssh.
 
     To be useful this will probably have to be extended to use the ``sshx``
     idea for environment variables.  There could be other things this needs
     as well.
     """
 
     ssh_cmd = List(['ssh'], config=True)
     ssh_args = List(['-tt'], config=True)
     program = List(['date'], config=True)
     program_args = List([], config=True)
     hostname = Str('', config=True)
-    user = Str(os.environ.get('USER','username'), config=True)
+    user = Str('', config=True)
     location = Str('')
 
     def _hostname_changed(self, name, old, new):
-        self.location = '%s@%s' % (self.user, new)
+        if self.user:
+            self.location = '%s@%s' % (self.user, new)
+        else:
+            self.location = new
 
     def _user_changed(self, name, old, new):
         self.location = '%s@%s' % (new, self.hostname)
 
     def find_args(self):
         return self.ssh_cmd + self.ssh_args + [self.location] + \
                self.program + self.program_args
 
     def start(self, cluster_dir, hostname=None, user=None):
-        print self.config
+        self.cluster_dir = unicode(cluster_dir)
         if hostname is not None:
             self.hostname = hostname
         if user is not None:
             self.user = user
-        print (self.location, hostname, user)
+        
         return super(SSHLauncher, self).start()
     
     def signal(self, sig):
         if self.state == 'running':
             # send escaped ssh connection-closer
             self.process.stdin.write('~.')
             self.process.stdin.flush()
         
 
 
 class SSHControllerLauncher(SSHLauncher):
 
     program = List(ipcontroller_cmd_argv, config=True)
     # Command line arguments to ipcontroller.
-    program_args = List(['--log-to-file','--log-level', str(logging.INFO)], config=True)
+    program_args = List(['-r', '--log-to-file','--log-level', str(logging.INFO)], config=True)
 
 
 class SSHEngineLauncher(SSHLauncher):
     program = List(ipengine_cmd_argv, config=True)
     # Command line arguments for ipengine.
     program_args = List(
         ['--log-to-file','--log-level', str(logging.INFO)], config=True
     )
     
 class SSHEngineSetLauncher(LocalEngineSetLauncher):
     launcher_class = SSHEngineLauncher
+    engines = Dict(config=True)
+    
+    def start(self, n, cluster_dir):
+        """Start engines by profile or cluster_dir.
+        `n` is ignored, and the `engines` config property is used instead.
+        """
+        
+        self.cluster_dir = unicode(cluster_dir)
+        dlist = []
+        for host, n in self.engines.iteritems():
+            if isinstance(n, (tuple, list)):
+                n, args = n
+            else:
+                args = copy.deepcopy(self.engine_args)
+            
+            if '@' in host:
+                user,host = host.split('@',1)
+            else:
+                user=None
+            for i in range(n):
+                el = self.launcher_class(work_dir=self.work_dir, config=self.config, logname=self.log.name)
+                
+                # Copy the engine args over to each engine launcher.
+                i
+                el.program_args = args
+                el.on_stop(self._notice_engine_stopped)
+                d = el.start(cluster_dir, user=user, hostname=host)
+                if i==0:
+                    self.log.info("Starting SSHEngineSetLauncher: %r" % el.args)
+                self.launchers[host+str(i)] = el
+                dlist.append(d)
+        self.notify_start(dlist)
+        return dlist
+    
 
 
 #-----------------------------------------------------------------------------
 # Windows HPC Server 2008 scheduler launchers
 #-----------------------------------------------------------------------------
 
 
 # This is only used on Windows.
 def find_job_cmd():
     if os.name=='nt':
         try:
             return find_cmd('job')
         except FindCmdError:
             return 'job'
     else:
         return 'job'
 
 
 class WindowsHPCLauncher(BaseLauncher):
 
     # A regular expression used to get the job id from the output of the 
     # submit_command.
     job_id_regexp = Str(r'\d+', config=True)
     # The filename of the instantiated job script.
     job_file_name = Unicode(u'ipython_job.xml', config=True)
     # The full path to the instantiated job script. This gets made dynamically
     # by combining the work_dir with the job_file_name.
     job_file = Unicode(u'')
     # The hostname of the scheduler to submit the job to
     scheduler = Str('', config=True)
     job_cmd = Str(find_job_cmd(), config=True)
 
     def __init__(self, work_dir=u'.', config=None, **kwargs):
         super(WindowsHPCLauncher, self).__init__(
             work_dir=work_dir, config=config, **kwargs
         )
 
     @property
     def job_file(self):
         return os.path.join(self.work_dir, self.job_file_name)
 
     def write_job_file(self, n):
         raise NotImplementedError("Implement write_job_file in a subclass.")
 
     def find_args(self):
         return ['job.exe']
         
     def parse_job_id(self, output):
         """Take the output of the submit command and return the job id."""
         m = re.search(self.job_id_regexp, output)
         if m is not None:
             job_id = m.group()
         else:
             raise LauncherError("Job id couldn't be determined: %s" % output)
         self.job_id = job_id
         self.log.info('Job started with job id: %r' % job_id)
         return job_id
 
     def start(self, n):
         """Start n copies of the process using the Win HPC job scheduler."""
         self.write_job_file(n)
         args = [
             'submit',
             '/jobfile:%s' % self.job_file,
             '/scheduler:%s' % self.scheduler
         ]
         self.log.info("Starting Win HPC Job: %s" % (self.job_cmd + ' ' + ' '.join(args),))
         # Twisted will raise DeprecationWarnings if we try to pass unicode to this
         output = check_output([self.job_cmd]+args,
             env=os.environ,
             cwd=self.work_dir,
             stderr=STDOUT
         )
         job_id = self.parse_job_id(output)
-        # self.notify_start(job_id)
+        self.notify_start(job_id)
         return job_id
 
     def stop(self):
         args = [
             'cancel',
             self.job_id,
             '/scheduler:%s' % self.scheduler
         ]
         self.log.info("Stopping Win HPC Job: %s" % (self.job_cmd + ' ' + ' '.join(args),))
         try:
             output = check_output([self.job_cmd]+args,
                 env=os.environ,
                 cwd=self.work_dir,
                 stderr=STDOUT
             )
         except:
             output = 'The job already appears to be stoppped: %r' % self.job_id
-        self.notify_stop(output)  # Pass the output of the kill cmd
+        self.notify_stop(dict(job_id=self.job_id, output=output))  # Pass the output of the kill cmd
         return output
 
 
 class WindowsHPCControllerLauncher(WindowsHPCLauncher):
 
     job_file_name = Unicode(u'ipcontroller_job.xml', config=True)
     extra_args = List([], config=False)
 
     def write_job_file(self, n):
         job = IPControllerJob(config=self.config)
 
         t = IPControllerTask(config=self.config)
         # The tasks work directory is *not* the actual work directory of 
         # the controller. It is used as the base path for the stdout/stderr
         # files that the scheduler redirects to.
         t.work_directory = self.cluster_dir
         # Add the --cluster-dir and from self.start().
         t.controller_args.extend(self.extra_args)
         job.add_task(t)
 
         self.log.info("Writing job description file: %s" % self.job_file)
         job.write(self.job_file)
 
     @property
     def job_file(self):
         return os.path.join(self.cluster_dir, self.job_file_name)
 
     def start(self, cluster_dir):
         """Start the controller by cluster_dir."""
         self.extra_args = ['--cluster-dir', cluster_dir]
         self.cluster_dir = unicode(cluster_dir)
         return super(WindowsHPCControllerLauncher, self).start(1)
 
 
 class WindowsHPCEngineSetLauncher(WindowsHPCLauncher):
 
     job_file_name = Unicode(u'ipengineset_job.xml', config=True)
     extra_args = List([], config=False)
 
     def write_job_file(self, n):
         job = IPEngineSetJob(config=self.config)
 
         for i in range(n):
             t = IPEngineTask(config=self.config)
             # The tasks work directory is *not* the actual work directory of 
             # the engine. It is used as the base path for the stdout/stderr
             # files that the scheduler redirects to.
             t.work_directory = self.cluster_dir
             # Add the --cluster-dir and from self.start().
             t.engine_args.extend(self.extra_args)
             job.add_task(t)
 
         self.log.info("Writing job description file: %s" % self.job_file)
         job.write(self.job_file)
 
     @property
     def job_file(self):
         return os.path.join(self.cluster_dir, self.job_file_name)
 
     def start(self, n, cluster_dir):
         """Start the controller by cluster_dir."""
         self.extra_args = ['--cluster-dir', cluster_dir]
         self.cluster_dir = unicode(cluster_dir)
         return super(WindowsHPCEngineSetLauncher, self).start(n)
 
 
 #-----------------------------------------------------------------------------
 # Batch (PBS) system launchers
 #-----------------------------------------------------------------------------
 
-# TODO: Get PBS launcher working again.
-
 class BatchSystemLauncher(BaseLauncher):
     """Launch an external process using a batch system.
 
     This class is designed to work with UNIX batch systems like PBS, LSF,
     GridEngine, etc.  The overall model is that there are different commands
     like qsub, qdel, etc. that handle the starting and stopping of the process.
 
     This class also has the notion of a batch script. The ``batch_template``
     attribute can be set to a string that is a template for the batch script.
     This template is instantiated using Itpl. Thus the template can use
     ${n} fot the number of instances. Subclasses can add additional variables
     to the template dict.
     """
 
     # Subclasses must fill these in.  See PBSEngineSet
     # The name of the command line program used to submit jobs.
     submit_command = Str('', config=True)
     # The name of the command line program used to delete jobs.
     delete_command = Str('', config=True)
     # A regular expression used to get the job id from the output of the 
     # submit_command.
     job_id_regexp = Str('', config=True)
     # The string that is the batch script template itself.
     batch_template = Str('', config=True)
     # The filename of the instantiated batch script.
     batch_file_name = Unicode(u'batch_script', config=True)
     # The full path to the instantiated batch script.
     batch_file = Unicode(u'')
     # the format dict used with batch_template:
     context = Dict()
 
     
     def find_args(self):
-        return [self.submit_command]
+        return [self.submit_command, self.batch_file]
     
     def __init__(self, work_dir=u'.', config=None, **kwargs):
         super(BatchSystemLauncher, self).__init__(
             work_dir=work_dir, config=config, **kwargs
         )
         self.batch_file = os.path.join(self.work_dir, self.batch_file_name)
 
     def parse_job_id(self, output):
         """Take the output of the submit command and return the job id."""
-        m = re.match(self.job_id_regexp, output)
+        m = re.search(self.job_id_regexp, output)
         if m is not None:
             job_id = m.group()
         else:
             raise LauncherError("Job id couldn't be determined: %s" % output)
         self.job_id = job_id
-        self.log.info('Job started with job id: %r' % job_id)
+        self.log.info('Job submitted with job id: %r' % job_id)
         return job_id
 
     def write_batch_script(self, n):
         """Instantiate and write the batch script to the work_dir."""
         self.context['n'] = n
         script_as_string = Itpl.itplns(self.batch_template, self.context)
         self.log.info('Writing instantiated batch script: %s' % self.batch_file)
         f = open(self.batch_file, 'w')
         f.write(script_as_string)
         f.close()
 
     def start(self, n, cluster_dir):
         """Start n copies of the process using a batch system."""
         # Here we save profile and cluster_dir in the context so they
         # can be used in the batch script template as ${profile} and
         # ${cluster_dir}
         self.context['cluster_dir'] = cluster_dir
         self.cluster_dir = unicode(cluster_dir)
         self.write_batch_script(n)
-        output = check_output([self.submit_command, self.batch_file], env=os.environ, stdout=STDOUT)
+        output = check_output(self.args, env=os.environ)
+        
         job_id = self.parse_job_id(output)
-        # self.notify_start(job_id)
+        self.notify_start(job_id)
         return job_id
 
     def stop(self):
-        output = check_output([self.delete_command, self.job_id], env=os.environ, stderr=STDOUT)
-        self.notify_stop(output)  # Pass the output of the kill cmd
+        output = check_output([self.delete_command, self.job_id], env=os.environ)
+        self.notify_stop(dict(job_id=self.job_id, output=output)) # Pass the output of the kill cmd
         return output
 
 
 class PBSLauncher(BatchSystemLauncher):
     """A BatchSystemLauncher subclass for PBS."""
 
     submit_command = Str('qsub', config=True)
     delete_command = Str('qdel', config=True)
     job_id_regexp = Str(r'\d+', config=True)
     batch_template = Str('', config=True)
     batch_file_name = Unicode(u'pbs_batch_script', config=True)
     batch_file = Unicode(u'')
 
 
 class PBSControllerLauncher(PBSLauncher):
     """Launch a controller using PBS."""
 
     batch_file_name = Unicode(u'pbs_batch_script_controller', config=True)
 
     def start(self, cluster_dir):
         """Start the controller by profile or cluster_dir."""
         self.log.info("Starting PBSControllerLauncher: %r" % self.args)
         return super(PBSControllerLauncher, self).start(1, cluster_dir)
 
 
 class PBSEngineSetLauncher(PBSLauncher):
 
     batch_file_name = Unicode(u'pbs_batch_script_engines', config=True)
 
     def start(self, n, cluster_dir):
         """Start n engines by profile or cluster_dir."""
         self.log.info('Starting PBSEngineSetLauncher: %r' % self.args)
         return super(PBSEngineSetLauncher, self).start(n, cluster_dir)
 
 
 #-----------------------------------------------------------------------------
 # A launcher for ipcluster itself!
 #-----------------------------------------------------------------------------
 
 
 class IPClusterLauncher(LocalProcessLauncher):
     """Launch the ipcluster program in an external process."""
 
     ipcluster_cmd = List(ipcluster_cmd_argv, config=True)
     # Command line arguments to pass to ipcluster.
     ipcluster_args = List(
         ['--clean-logs', '--log-to-file', '--log-level', str(logging.INFO)], config=True)
     ipcluster_subcommand = Str('start')
     ipcluster_n = Int(2)
 
     def find_args(self):
         return self.ipcluster_cmd + [self.ipcluster_subcommand] + \
             ['-n', repr(self.ipcluster_n)] + self.ipcluster_args
 
     def start(self):
         self.log.info("Starting ipcluster: %r" % self.args)
         return super(IPClusterLauncher, self).start()
 

docs/source/parallelz/parallel_process.txt

@@ -1,403 +1,490 b''
 .. _parallel_process:
 
 ===========================================
 Starting the IPython controller and engines
 ===========================================
 
 To use IPython for parallel computing, you need to start one instance of
 the controller and one or more instances of the engine. The controller
 and each engine can run on different machines or on the same machine.
 Because of this, there are many different possibilities.
 
 Broadly speaking, there are two ways of going about starting a controller and engines:
 
 * In an automated manner using the :command:`ipclusterz` command.
 * In a more manual way using the :command:`ipcontrollerz` and 
   :command:`ipenginez` commands.
 
 This document describes both of these methods. We recommend that new users
 start with the :command:`ipclusterz` command as it simplifies many common usage
 cases.
 
 General considerations
 ======================
 
 Before delving into the details about how you can start a controller and
 engines using the various methods, we outline some of the general issues that
 come up when starting the controller and engines. These things come up no
 matter which method you use to start your IPython cluster.
 
 Let's say that you want to start the controller on ``host0`` and engines on
 hosts ``host1``-``hostn``. The following steps are then required:
 
 1. Start the controller on ``host0`` by running :command:`ipcontrollerz` on
    ``host0``.
 2. Move the JSON file (:file:`ipcontroller-engine.json`) created by the 
    controller from ``host0`` to hosts ``host1``-``hostn``.
 3. Start the engines on hosts ``host1``-``hostn`` by running 
    :command:`ipenginez`.  This command has to be told where the JSON file 
    (:file:`ipcontroller-engine.json`) is located.
 
 At this point, the controller and engines will be connected. By default, the JSON files
 created by the controller are put into the :file:`~/.ipython/clusterz_default/security`
 directory. If the engines share a filesystem with the controller, step 2 can be skipped as
 the engines will automatically look at that location.
 
 The final step required to actually use the running controller from a client is to move
 the JSON file :file:`ipcontroller-client.json` from ``host0`` to any host where clients
 will be run. If these file are put into the :file:`~/.ipython/clusterz_default/security`
 directory of the client's host, they will be found automatically. Otherwise, the full path
 to them has to be passed to the client's constructor.
 
 Using :command:`ipclusterz`
 ==========================
 
 The :command:`ipclusterz` command provides a simple way of starting a
 controller and engines in the following situations:
 
 1. When the controller and engines are all run on localhost. This is useful
    for testing or running on a multicore computer.
 2. When engines are started using the :command:`mpirun` command that comes 
    with most MPI [MPI]_ implementations
-3. When engines are started using the PBS [PBS]_ batch system.
+3. When engines are started using the PBS [PBS]_ batch system 
+   (or other `qsub` systems, such as SGE).
 4. When the controller is started on localhost and the engines are started on 
    remote nodes using :command:`ssh`.
-
-.. note::
-
-    It is also possible for advanced users to add support to
-    :command:`ipclusterz` for starting controllers and engines using other
-    methods (like Sun's Grid Engine for example).
+5. When engines are started using the Windows HPC Server batch system.
 
 .. note::
 
     Currently :command:`ipclusterz` requires that the
     :file:`~/.ipython/cluster_<profile>/security` directory live on a shared filesystem that is
     seen by both the controller and engines. If you don't have a shared file
     system you will need to use :command:`ipcontrollerz` and
-    :command:`ipenginez` directly.  This constraint can be relaxed if you are 
-    using the :command:`ssh` method to start the cluster.
+    :command:`ipenginez` directly.
 
 Under the hood, :command:`ipclusterz` just uses :command:`ipcontrollerz`
 and :command:`ipenginez` to perform the steps described above.
 
-Using :command:`ipclusterz` in local mode
-----------------------------------------
-
-To start one controller and 4 engines on localhost, just do::
+The simplest way to use ipclusterz requires no configuration, and will
+launch a controller and a number of engines on the local machine. For instance, 
+to start one controller and 4 engines on localhost, just do::
 
     $ ipclusterz start -n 4
 
 To see other command line options for the local mode, do::
 
     $ ipclusterz -h
 
-.. note::
 
-    The remainder of this section refers to the 0.10 clusterfile model, no longer in use.
-    skip to 
+Configuring an IPython cluster
+==============================
 
-Using :command:`ipclusterz` in mpiexec/mpirun mode
--------------------------------------------------
+Cluster configurations are stored as `profiles`.  You can create a new profile with::
+
+    $ ipclusterz create -p myprofile
+
+This will create the directory :file:`IPYTHONDIR/clusterz_myprofile`, and populate it
+with the default configuration files for the three IPython cluster commands. Once
+you edit those files, you can continue to call ipclusterz/ipcontrollerz/ipenginez
+with no arguments beyond ``-p myprofile``, and any configuration will be maintained.
+
+There is no limit to the number of profiles you can have, so you can maintain a profile for each
+of your common use cases. The default profile will be used whenever the
+profile argument is not specified, so edit :file:`IPYTHONDIR/clusterz_default/*_config.py` to
+represent your most common use case.
+
+The configuration files are loaded with commented-out settings and explanations,
+which should cover most of the available possibilities.
+
+Using various batch systems with :command:`ipclusterz`
+------------------------------------------------------
+
+:command:`ipclusterz` has a notion of Launchers that can start controllers
+and engines with various remote execution schemes.  Currently supported
+models include `mpiexec`, PBS-style (Torque, SGE), and Windows HPC Server.
 
 .. note::
 
-    This section is out of date for IPython 0.11
+    The Launchers and configuration are designed in such a way that advanced
+    users can subclass and configure them to fit their own system that we
+    have not yet supported (such as Condor)
+
+Using :command:`ipclusterz` in mpiexec/mpirun mode
+--------------------------------------------------
 
 
 The mpiexec/mpirun mode is useful if you:
 
 1. Have MPI installed.
 2. Your systems are configured to use the :command:`mpiexec` or
    :command:`mpirun` commands to start MPI processes.
 
-.. note::
+If these are satisfied, you can create a new profile::
+
+    $ ipclusterz create -p mpi
+
+and edit the file :file:`IPYTHONDIR/clusterz_mpi/ipclusterz_config.py`.
 
-    The preferred command to use is :command:`mpiexec`.  However, we also 
-    support :command:`mpirun` for backwards compatibility.  The underlying
-    logic used is exactly the same, the only difference being the name of the 
-    command line program that is called.
+There, instruct ipclusterz to use the MPIExec launchers by adding the lines:
 
-If these are satisfied, you can start an IPython cluster using::
+.. sourcecode:: python
+
+    c.Global.engine_launcher = 'IPython.zmq.parallel.launcher.MPIExecEngineSetLauncher'
+
+If the default MPI configuration is correct, then you can now start your cluster, with::
 
-    $ ipclusterz mpiexec -n 4
+    $ ipclusterz start -n 4 -p mpi
 
 This does the following:
 
 1. Starts the IPython controller on current host.
 2. Uses :command:`mpiexec` to start 4 engines.
 
+If you have a reason to also start the Controller with mpi, you can specify:
+
+.. sourcecode:: python
+
+    c.Global.controller_launcher = 'IPython.zmq.parallel.launcher.MPIExecControllerLauncher'
+
+.. note::
+
+    The Controller *will not* be in the same MPI universe as the engines, so there is not
+    much reason to do this unless sysadmins demand it.
+
 On newer MPI implementations (such as OpenMPI), this will work even if you
 don't make any calls to MPI or call :func:`MPI_Init`. However, older MPI
 implementations actually require each process to call :func:`MPI_Init` upon
 starting. The easiest way of having this done is to install the mpi4py
-[mpi4py]_ package and then call ipclusterz with the ``--mpi`` option::
+[mpi4py]_ package and then specify the ``c.MPI.use`` option in :file:`ipenginez_config.py`:
+
+.. sourcecode:: python
 
-    $ ipclusterz mpiexec -n 4 --mpi=mpi4py
+    c.MPI.use = 'mpi4py'
 
 Unfortunately, even this won't work for some MPI implementations. If you are
 having problems with this, you will likely have to use a custom Python
 executable that itself calls :func:`MPI_Init` at the appropriate time.
 Fortunately, mpi4py comes with such a custom Python executable that is easy to
 install and use. However, this custom Python executable approach will not work
 with :command:`ipclusterz` currently.
 
-Additional command line options for this mode can be found by doing::
-
-    $ ipclusterz mpiexec -h
-
 More details on using MPI with IPython can be found :ref:`here <parallelmpi>`.
 
 
 Using :command:`ipclusterz` in PBS mode
---------------------------------------
+---------------------------------------
 
-.. note::
+The PBS mode uses the Portable Batch System [PBS]_ to start the engines. 
+
+As usual, we will start by creating a fresh profile::
+
+    $ ipclusterz create -p pbs
+
+And in :file:`ipclusterz_config.py`, we will select the PBS launchers for the controller
+and engines:
 
-    This section is out of date for IPython 0.11
+.. sourcecode:: python
 
+    c.Global.controller_launcher = 'IPython.zmq.parallel.launcher.PBSControllerLauncher'
+    c.Global.engine_launcher = 'IPython.zmq.parallel.launcher.PBSEngineSetLauncher'
 
-The PBS mode uses the Portable Batch System [PBS]_ to start the engines. To
-use this mode, you first need to create a PBS script template that will be
+To use this mode, you first need to create a PBS script template that will be
 used to start the engines. Here is a sample PBS script template:
 
 .. sourcecode:: bash
 
     #PBS -N ipython
     #PBS -j oe
     #PBS -l walltime=00:10:00
     #PBS -l nodes=${n/4}:ppn=4
     #PBS -q parallel
 
     cd $$PBS_O_WORKDIR
     export PATH=$$HOME/usr/local/bin
-    export PYTHONPATH=$$HOME/usr/local/lib/python2.4/site-packages
-    /usr/local/bin/mpiexec -n ${n} ipengine --logfile=$$PBS_O_WORKDIR/ipengine
+    export PYTHONPATH=$$HOME/usr/local/lib/python2.7/site-packages
+    /usr/local/bin/mpiexec -n ${n} ipenginez --cluster_dir=${cluster_dir}
 
 There are a few important points about this template:
 
 1. This template will be rendered at runtime using IPython's :mod:`Itpl` 
    template engine.
 
 2. Instead of putting in the actual number of engines, use the notation
    ``${n}`` to indicate the number of engines to be started. You can also uses
    expressions like ``${n/4}`` in the template to indicate the number of 
-   nodes.
+   nodes. There will always be a ${n} and ${cluster_dir} variable passed to the template.
+   These allow the batch system to know how many engines, and where the configuration
+   files reside.
 
 3. Because ``$`` is a special character used by the template engine, you must  
    escape any ``$`` by using ``$$``.  This is important when referring to 
    environment variables in the template.
 
-4. Any options to :command:`ipenginez` should be given in the batch script 
-   template.
+4. Any options to :command:`ipenginez` can be given in the batch script 
+   template, or in :file:`ipenginez_config.py`.
 
 5. Depending on the configuration of you system, you may have to set 
    environment variables in the script template.
 
-Once you have created such a script, save it with a name like
-:file:`pbs.template`. Now you are ready to start your job::
+The controller template should be similar, but simpler:
+
+.. sourcecode:: bash
+
+    #PBS -N ipython
+    #PBS -j oe
+    #PBS -l walltime=00:10:00
+    #PBS -l nodes=1:ppn=4
+    #PBS -q parallel
 
-    $ ipclusterz pbs -n 128 --pbs-script=pbs.template
+    cd $$PBS_O_WORKDIR
+    export PATH=$$HOME/usr/local/bin
+    export PYTHONPATH=$$HOME/usr/local/lib/python2.7/site-packages
+    ipcontrollerz --cluster_dir=${cluster_dir}
 
-Additional command line options for this mode can be found by doing::
 
-    $ ipclusterz pbs -h
+Once you have created these scripts, save them with names like
+:file:`pbs.engine.template`. Now you can load them into the :file:`ipclusterz_config` with:
 
-Using :command:`ipclusterz` in SSH mode
---------------------------------------
+.. sourcecode:: python
+
+    with open("pbs.engine.template") as f:
+        c.PBSEngineSetLauncher.batch_template = f.read()
+    
+    with open("pbs.controller.template") as f:
+        c.PBSControllerLauncher.batch_template = f.read()
+        
+
+Alternately, you can just define the templates as strings inside :file:`ipclusterz_config`.
+
+Note that assuming you are running PBS on a multi-node cluster, the Controller's default behavior
+of listening only on localhost is likely too restrictive.  In this case, also assuming the
+nodes are safely behind a firewall, you can simply instruct the Controller to listen for
+connections on all its interfaces, by adding in :file:`ipcontrollerz_config`:
+
+.. sourcecode:: python
+
+    c.HubFactory.client_ip = '*'
+    c.HubFactory.engine_ip = '*'
+
+You can now run the cluster with::
+
+    $ ipclusterz start -p pbs -n 128
+
+Additional configuration options can be found in the PBS section of :file:`ipclusterz_config`.
 
 .. note::
 
-    This section is out of date for IPython 0.11
+    Due to the flexibility of configuration, the PBS launchers work with simple changes
+    to the template for other :command:`qsub`-using systems, such as Sun Grid Engine,
+    and with further configuration in similar batch systems like Condor.
+
+
+Using :command:`ipclusterz` in SSH mode
+---------------------------------------
 
 
 The SSH mode uses :command:`ssh` to execute :command:`ipenginez` on remote
-nodes and the :command:`ipcontrollerz` on localhost.
+nodes and :command:`ipcontrollerz` can be run remotely as well, or on localhost.
 
-When using using this mode it highly recommended that you have set up SSH keys
-and are using ssh-agent [SSH]_ for password-less logins.
+.. note::
 
-To use this mode you need a python file describing the cluster, here is an
-example of such a "clusterfile":
+    When using this mode it highly recommended that you have set up SSH keys
+    and are using ssh-agent [SSH]_ for password-less logins.
 
-.. sourcecode:: python
-    
-    send_furl = True
-    engines = { 'host1.example.com' : 2,
-                'host2.example.com' : 5,
-                'host3.example.com' : 1,
-                'host4.example.com' : 8 }
+As usual, we start by creating a clean profile::
 
-Since this is a regular python file usual python syntax applies. Things to
-note:
+    $ ipclusterz create -p ssh
 
-* The `engines` dict, where the keys is the host we want to run engines on and
-  the value is the number of engines to run on that host.
-* send_furl can either be `True` or `False`, if `True` it will copy over the
-  furl needed for :command:`ipenginez` to each host.
+To use this mode, select the SSH launchers in :file:`ipclusterz_config.py`:
 
-The ``--clusterfile`` command line option lets you specify the file to use for
-the cluster definition. Once you have your cluster file and you can
-:command:`ssh` into the remote hosts with out an password you are ready to
-start your cluster like so:
+.. sourcecode:: python
 
-.. sourcecode:: bash 
+    c.Global.engine_launcher = 'IPython.zmq.parallel.launcher.PBSEngineSetLauncher'
+    # and if the Controller is also to be remote:
+    c.Global.controller_launcher = 'IPython.zmq.parallel.launcher.SSHControllerLauncher'
+    
 
-   $ ipclusterz ssh --clusterfile /path/to/my/clusterfile.py
+The controller's remote location and configuration can be specified:
 
+.. sourcecode:: python
 
-Two helper shell scripts are used to start and stop :command:`ipenginez` on
-remote hosts:
+    # Set the user and hostname for the controller
+    # c.SSHControllerLauncher.hostname = 'controller.example.com'
+    # c.SSHControllerLauncher.user = os.environ.get('USER','username')
 
-* sshx.sh 
-* engine_killer.sh
+    # Set the arguments to be passed to ipcontrollerz
+    # note that remotely launched ipcontrollerz will not get the contents of 
+    # the local ipcontrollerz_config.py unless it resides on the *remote host*
+    # in the location specified by the --cluster_dir argument.
+    # c.SSHControllerLauncher.program_args = ['-r', '-ip', '0.0.0.0', '--cluster_dir', '/path/to/cd']
 
-Defaults for both of these are contained in the source code for
-:command:`ipclusterz`. The default scripts are written to a local file in a
-tmep directory and then copied to a temp directory on the remote host and
-executed from there. On most Unix, Linux and OS X systems this is /tmp.
+.. note::
 
-The default sshx.sh is the following:
+    SSH mode does not do any file movement, so you will need to distribute configuration
+    files manually.  To aid in this, the `reuse_files` flag defaults to True for ssh-launched
+    Controllers, so you will only need to do this once, unless you override this flag back
+    to False.
 
-.. sourcecode:: bash
+Engines are specified in a dictionary, by hostname and the number of engines to be run
+on that host.
 
-   #!/bin/sh
-   "$@" &> /dev/null &
-   echo $!
+.. sourcecode:: python
+
+    c.SSHEngineSetLauncher.engines = { 'host1.example.com' : 2,
+                'host2.example.com' : 5,
+                'host3.example.com' : (1, ['--cluster_dir', '/home/different/location']),
+                'host4.example.com' : 8 }
 
-If you want to use a custom sshx.sh script you need to use the ``--sshx``
-option and specify the file to use. Using a custom sshx.sh file could be
-helpful when you need to setup the environment on the remote host before
-executing :command:`ipenginez`.
+* The `engines` dict, where the keys are the host we want to run engines on and
+  the value is the number of engines to run on that host.
+* on host3, the value is a tuple, where the number of engines is first, and the arguments
+  to be passed to :command:`ipenginez` are the second element.
 
-For a detailed options list:
+For engines without explicitly specified arguments, the default arguments are set in
+a single location:
 
-.. sourcecode:: bash
+.. sourcecode:: python
 
-   $ ipclusterz ssh -h
+    c.SSHEngineSetLauncher.engine_args = ['--cluster_dir', '/path/to/clusterz_ssh']
 
 Current limitations of the SSH mode of :command:`ipclusterz` are:
 
 * Untested on Windows. Would require a working :command:`ssh` on Windows.
   Also, we are using shell scripts to setup and execute commands on remote
   hosts.
-* :command:`ipcontrollerz` is started on localhost, with no option to start it
-  on a remote node.
+* No file movement - 
 
 Using the :command:`ipcontrollerz` and :command:`ipenginez` commands
 ====================================================================
 
 It is also possible to use the :command:`ipcontrollerz` and :command:`ipenginez`
 commands to start your controller and engines. This approach gives you full
 control over all aspects of the startup process.
 
 Starting the controller and engine on your local machine
 --------------------------------------------------------
 
 To use :command:`ipcontrollerz` and :command:`ipenginez` to start things on your
 local machine, do the following.
 
 First start the controller::
 
-	$ ipcontrollerz
-	
+    $ ipcontrollerz
+    
 Next, start however many instances of the engine you want using (repeatedly)
 the command::
 
-	$ ipenginez
+    $ ipenginez
 
 The engines should start and automatically connect to the controller using the
-JSON files in :file:`~/.ipython/cluster_<profile>/security`. You are now ready to use the
+JSON files in :file:`~/.ipython/clusterz_default/security`. You are now ready to use the
 controller and engines from IPython.
 
 .. warning:: 
-	
-	The order of the above operations may be important.  You *must*
-	start the controller before the engines, unless you are manually specifying
-	the ports on which to connect, in which case ordering is not important.
+    
+    The order of the above operations may be important.  You *must*
+    start the controller before the engines, unless you are reusing connection
+    information (via `-r`), in which case ordering is not important.
 
 .. note::
 
     On some platforms (OS X), to put the controller and engine into the
     background you may need to give these commands in the form ``(ipcontroller
     &)`` and ``(ipengine &)`` (with the parentheses) for them to work
     properly.
 
 Starting the controller and engines on different hosts
 ------------------------------------------------------
 
 When the controller and engines are running on different hosts, things are
 slightly more complicated, but the underlying ideas are the same:
 
 1. Start the controller on a host using :command:`ipcontrollerz`.
 2. Copy :file:`ipcontroller-engine.json` from :file:`~/.ipython/cluster_<profile>/security` on
    the controller's host to the host where the engines will run.
 3. Use :command:`ipenginez` on the engine's hosts to start the engines.
 
 The only thing you have to be careful of is to tell :command:`ipenginez` where
 the :file:`ipcontroller-engine.json` file is located. There are two ways you
 can do this:
 
 * Put :file:`ipcontroller-engine.json` in the :file:`~/.ipython/cluster_<profile>/security`
   directory on the engine's host, where it will be found automatically.
 * Call :command:`ipenginez` with the ``--file=full_path_to_the_file``
   flag.
 
 The ``--file`` flag works like this::
 
     $ ipengine --file=/path/to/my/ipcontroller-engine.json
 
 .. note::
     
     If the controller's and engine's hosts all have a shared file system  
     (:file:`~/.ipython/cluster_<profile>/security` is the same on all of them), then things
     will just work!
 
 Make JSON files persistent
----------------------------
+--------------------------
 
 At fist glance it may seem that that managing the JSON files is a bit
 annoying. Going back to the house and key analogy, copying the JSON around
 each time you start the controller is like having to make a new key every time
 you want to unlock the door and enter your house. As with your house, you want
 to be able to create the key (or JSON file) once, and then simply use it at
 any point in the future.
 
-This is possible, but before you do this, you **must** remove any old JSON
-files in the :file:`~/.ipython/cluster_<profile>/security` directory.
-
-.. warning::
-
-    You **must** remove old JSON files before using persistent JSON files.
-
-Then, the only thing you have to do is specify the registration port, so that
+To do this, the only thing you have to do is specify the `-r` flag, so that
 the connection information in the JSON files remains accurate::
 
     $ ipcontrollerz -r --regport 12345
 
-
 Then, just copy the JSON files over the first time and you are set. You can
 start and stop the controller and engines any many times as you want in the
-future, just make sure to tell the controller to use the *same* ports.
+future, just make sure to tell the controller to reuse the file.
 
 .. note::
 
     You may ask the question: what ports does the controller listen on if you
     don't tell is to use specific ones? The default is to use high random port
     numbers. We do this for two reasons: i) to increase security through
     obscurity and ii) to multiple controllers on a given host to start and
     automatically use different ports.
 
 Log files
 ---------
 
 All of the components of IPython have log files associated with them.
 These log files can be extremely useful in debugging problems with
 IPython and can be found in the directory :file:`~/.ipython/cluster_<profile>/log`.
 Sending the log files to us will often help us to debug any problems.
 
 
 .. [PBS] Portable Batch System.  http://www.openpbs.org/
 .. [SSH] SSH-Agent http://en.wikipedia.org/wiki/Ssh-agent
 
+Configuring `ipcontrollerz`
+---------------------------
+
+.. note::
+
+    TODO
+
+Configuring `ipenginez`
+-----------------------
+
+.. note::
+
+    TODO
+