updated sge to launch multiple engines - start cleanly but does not shutdown cleanly

IPython/kernel/scripts/ipcluster.py

@@ -1,874 +1,914 b''
  #!/usr/bin/env python
 # encoding: utf-8
 
 """Start an IPython cluster = (controller + engines)."""
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import os
 import re
 import sys
 import signal
 import tempfile
 pjoin = os.path.join
 
 from twisted.internet import reactor, defer
 from twisted.internet.protocol import ProcessProtocol
 from twisted.internet.error import ProcessDone, ProcessTerminated
 from twisted.internet.utils import getProcessOutput
 from twisted.python import failure, log
 
 from IPython.external import argparse
 from IPython.external import Itpl
 from IPython.genutils import (
     get_ipython_dir, 
     get_log_dir, 
     get_security_dir, 
     num_cpus
 )
 from IPython.kernel.fcutil import have_crypto
 
 # Create various ipython directories if they don't exist.
 # This must be done before IPython.kernel.config is imported.
 from IPython.iplib import user_setup
 if os.name == 'posix':
     rc_suffix = ''
 else:
     rc_suffix = '.ini'
 user_setup(get_ipython_dir(), rc_suffix, mode='install', interactive=False)
 get_log_dir()
 get_security_dir()
 
 from IPython.kernel.config import config_manager as kernel_config_manager
 from IPython.kernel.error import SecurityError, FileTimeoutError
 from IPython.kernel.fcutil import have_crypto
 from IPython.kernel.twistedutil import gatherBoth, wait_for_file
 from IPython.kernel.util import printer
 
 #-----------------------------------------------------------------------------
 # General process handling code
 #-----------------------------------------------------------------------------
 
 
 class ProcessStateError(Exception):
     pass
 
 class UnknownStatus(Exception):
     pass
 
 class LauncherProcessProtocol(ProcessProtocol):
     """
     A ProcessProtocol to go with the ProcessLauncher.
     """
     def __init__(self, process_launcher):
         self.process_launcher = process_launcher
     
     def connectionMade(self):
         self.process_launcher.fire_start_deferred(self.transport.pid)
     
     def processEnded(self, status):
         value = status.value
         if isinstance(value, ProcessDone):
             self.process_launcher.fire_stop_deferred(0)
         elif isinstance(value, ProcessTerminated):
             self.process_launcher.fire_stop_deferred(
                 {'exit_code':value.exitCode,
                  'signal':value.signal,
                  'status':value.status
                 }
             )
         else:
             raise UnknownStatus("unknown exit status, this is probably a bug in Twisted")
     
     def outReceived(self, data):
         log.msg(data)
     
     def errReceived(self, data):
         log.err(data)
 
 class ProcessLauncher(object):
     """
     Start and stop an external process in an asynchronous manner.
     
     Currently this uses deferreds to notify other parties of process state
     changes.  This is an awkward design and should be moved to using
     a formal NotificationCenter.
     """
     def __init__(self, cmd_and_args):
         self.cmd = cmd_and_args[0]
         self.args = cmd_and_args
         self._reset()
     
     def _reset(self):
         self.process_protocol = None
         self.pid = None
         self.start_deferred = None
         self.stop_deferreds = []
         self.state = 'before' # before, running, or after
     
     @property
     def running(self):
         if self.state == 'running':
             return True
         else:
             return False
     
     def fire_start_deferred(self, pid):
         self.pid = pid
         self.state = 'running'
         log.msg('Process %r has started with pid=%i' % (self.args, pid))
         self.start_deferred.callback(pid)
     
     def start(self):
         if self.state == 'before':
             self.process_protocol = LauncherProcessProtocol(self)
             self.start_deferred = defer.Deferred()
             self.process_transport = reactor.spawnProcess(
                 self.process_protocol,
                 self.cmd,
                 self.args,
                 env=os.environ
             )
             return self.start_deferred
         else:
             s = 'the process has already been started and has state: %r' % \
                 self.state
             return defer.fail(ProcessStateError(s))
     
     def get_stop_deferred(self):
         if self.state == 'running' or self.state == 'before':
             d = defer.Deferred()
             self.stop_deferreds.append(d)
             return d
         else:
             s = 'this process is already complete'
             return defer.fail(ProcessStateError(s))
     
     def fire_stop_deferred(self, exit_code):
         log.msg('Process %r has stopped with %r' % (self.args, exit_code))
         self.state = 'after'
         for d in self.stop_deferreds:
             d.callback(exit_code)
     
     def signal(self, sig):
         """
         Send a signal to the process.
         
         The argument sig can be ('KILL','INT', etc.) or any signal number.
         """
         if self.state == 'running':
             self.process_transport.signalProcess(sig)
     
     # def __del__(self):
     #     self.signal('KILL')
     
     def interrupt_then_kill(self, delay=1.0):
         self.signal('INT')
         reactor.callLater(delay, self.signal, 'KILL')
 
 
 #-----------------------------------------------------------------------------
 # Code for launching controller and engines
 #-----------------------------------------------------------------------------
 
 
 class ControllerLauncher(ProcessLauncher):
     
     def __init__(self, extra_args=None):
         if sys.platform == 'win32':
             # This logic is needed because the ipcontroller script doesn't
             # always get installed in the same way or in the same location.
             from IPython.kernel.scripts import ipcontroller
             script_location = ipcontroller.__file__.replace('.pyc', '.py')
             # The -u option here turns on unbuffered output, which is required
             # on Win32 to prevent wierd conflict and problems with Twisted.
             # Also, use sys.executable to make sure we are picking up the 
             # right python exe.
             args = [sys.executable, '-u', script_location]
         else:
             args = ['ipcontroller']
         self.extra_args = extra_args
         if extra_args is not None:
             args.extend(extra_args)
         
         ProcessLauncher.__init__(self, args)
 
 
 class EngineLauncher(ProcessLauncher):
     
     def __init__(self, extra_args=None):
         if sys.platform == 'win32':
             # This logic is needed because the ipcontroller script doesn't
             # always get installed in the same way or in the same location.
             from IPython.kernel.scripts import ipengine
             script_location = ipengine.__file__.replace('.pyc', '.py')
             # The -u option here turns on unbuffered output, which is required
             # on Win32 to prevent wierd conflict and problems with Twisted.
             # Also, use sys.executable to make sure we are picking up the 
             # right python exe.
             args = [sys.executable, '-u', script_location]
         else:
             args = ['ipengine']
         self.extra_args = extra_args
         if extra_args is not None:
             args.extend(extra_args)
         
         ProcessLauncher.__init__(self, args)
 
 
 class LocalEngineSet(object):
     
     def __init__(self, extra_args=None):
         self.extra_args = extra_args
         self.launchers = []
     
     def start(self, n):
         dlist = []
         for i in range(n):
+            print "starting engine:", i
             el = EngineLauncher(extra_args=self.extra_args)
             d = el.start()
             self.launchers.append(el)
             dlist.append(d)
         dfinal = gatherBoth(dlist, consumeErrors=True)
         dfinal.addCallback(self._handle_start)
         return dfinal
     
     def _handle_start(self, r):
         log.msg('Engines started with pids: %r' % r)
         return r
     
     def _handle_stop(self, r):
         log.msg('Engines received signal: %r' % r)
         return r
     
     def signal(self, sig):
         dlist = []
         for el in self.launchers:
             d = el.get_stop_deferred()
             dlist.append(d)
             el.signal(sig)
         dfinal = gatherBoth(dlist, consumeErrors=True)
         dfinal.addCallback(self._handle_stop)
         return dfinal
     
     def interrupt_then_kill(self, delay=1.0):
         dlist = []
         for el in self.launchers:
             d = el.get_stop_deferred()
             dlist.append(d)
             el.interrupt_then_kill(delay)
         dfinal = gatherBoth(dlist, consumeErrors=True)
         dfinal.addCallback(self._handle_stop)
         return dfinal
 
 
 class BatchEngineSet(object):
     
     # Subclasses must fill these in.  See PBSEngineSet
     submit_command = ''
     delete_command = ''
     job_id_regexp = ''
     
     def __init__(self, template_file, **kwargs):
         self.template_file = template_file
         self.context = {}
         self.context.update(kwargs)
         self.batch_file = self.template_file+'-run'
     
     def parse_job_id(self, output):
         m = re.match(self.job_id_regexp, output)
         if m is not None:
             job_id = m.group()
         else:
             raise Exception("job id couldn't be determined: %s" % output)
         self.job_id = job_id
         log.msg('Job started with job id: %r' % job_id)
         return job_id
     
     def write_batch_script(self, n):
         self.context['n'] = n
         template = open(self.template_file, 'r').read()
         log.msg('Using template for batch script: %s' % self.template_file)
         script_as_string = Itpl.itplns(template, self.context)
         log.msg('Writing instantiated batch script: %s' % self.batch_file)
         f = open(self.batch_file,'w')
         f.write(script_as_string)
         f.close()
     
     def handle_error(self, f):
         f.printTraceback()
         f.raiseException()
     
     def start(self, n):
         self.write_batch_script(n)
         d = getProcessOutput(self.submit_command,
             [self.batch_file],env=os.environ)
         d.addCallback(self.parse_job_id)
         d.addErrback(self.handle_error)
         return d
     
     def kill(self):
         d = getProcessOutput(self.delete_command,
             [self.job_id],env=os.environ)
         return d
 
 class PBSEngineSet(BatchEngineSet):
     
     submit_command = 'qsub'
     delete_command = 'qdel'
     job_id_regexp = '\d+'
     
     def __init__(self, template_file, **kwargs):
         BatchEngineSet.__init__(self, template_file, **kwargs)
 
 class SGEEngineSet(BatchEngineSet):
     
     submit_command = 'qsub'
     delete_command = 'qdel'
     job_id_regexp = '\d+'
     
     def __init__(self, template_file, **kwargs):
         BatchEngineSet.__init__(self, template_file, **kwargs)
+        self.num_engines = None
 
     def parse_job_id(self, output):
         m = re.search(self.job_id_regexp, output)
         if m is not None:
             job_id = m.group()
         else:
             raise Exception("job id couldn't be determined: %s" % output)
-        self.job_id = job_id
+        self.job_id.append(job_id)
         log.msg('Job started with job id: %r' % job_id)
         return job_id
-
+    
+    def kill_job(self, output):
+        log.msg(output)
+        return output
+    
+    def write_batch_script(self, i):
+        context = {'eid':i}
+        template = open(self.template_file, 'r').read()
+        log.msg('Using template for batch script: %s' % self.template_file)
+        script_as_string = Itpl.itplns(template, context)
+        log.msg('Writing instantiated batch script: %s' % self.batch_file+str(i))
+        f = open(self.batch_file+str(i),'w')
+        f.write(script_as_string)
+        f.close()
+        
+    def start(self, n):
+        dlist = []
+        self.num_engines = 0
+        self.job_id = []
+        for i in range(n):
+            log.msg("starting engine: %d"%i)
+            self.write_batch_script(i)
+            d = getProcessOutput(self.submit_command,
+                                 [self.batch_file+str(i)],env=os.environ)
+            d.addCallback(self.parse_job_id)
+            d.addErrback(self.handle_error)
+            dlist.append(d)
+        return gatherBoth(dlist, consumeErrors=True)
+    
+    def kill(self):
+        dlist = []
+        for i in range(self.num_engines):
+            log.msg("killing job id: %d"%self.job_id[i])
+            d = getProcessOutput(self.delete_command,
+                                 [self.job_id[i]],env=os.environ)
+            d.addCallback(self.kill_job)
+            dlist.append(d)
+        return gatherBoth(dlist, consumeErrors=True)
+    
 sshx_template="""#!/bin/sh
 "$@" &> /dev/null &
 echo $!
 """
 
 engine_killer_template="""#!/bin/sh
 ps -fu `whoami` | grep '[i]pengine' | awk '{print $2}' | xargs kill -TERM
 """
 
 class SSHEngineSet(object):
     sshx_template=sshx_template
     engine_killer_template=engine_killer_template
     
     def __init__(self, engine_hosts, sshx=None, ipengine="ipengine"):
         """Start a controller on localhost and engines using ssh.
         
         The engine_hosts argument is a dict with hostnames as keys and
         the number of engine (int) as values.  sshx is the name of a local
         file that will be used to run remote commands.  This file is used
         to setup the environment properly.
         """
         
         self.temp_dir = tempfile.gettempdir()
         if sshx is not None:
             self.sshx = sshx
         else:
             # Write the sshx.sh file locally from our template.
             self.sshx = os.path.join(
                 self.temp_dir,
                 '%s-main-sshx.sh' % os.environ['USER']
             )
             f = open(self.sshx, 'w')
             f.writelines(self.sshx_template)
             f.close()
         self.engine_command = ipengine
         self.engine_hosts = engine_hosts
         # Write the engine killer script file locally from our template.
         self.engine_killer = os.path.join(
             self.temp_dir,  
             '%s-local-engine_killer.sh' % os.environ['USER']
         )
         f = open(self.engine_killer, 'w')
         f.writelines(self.engine_killer_template)
         f.close()
     
     def start(self, send_furl=False):
         dlist = []
         for host in self.engine_hosts.keys():
             count = self.engine_hosts[host]
             d = self._start(host, count, send_furl)
             dlist.append(d)
         return gatherBoth(dlist, consumeErrors=True)
     
     def _start(self, hostname, count=1, send_furl=False):
         if send_furl:
             d = self._scp_furl(hostname)
         else:
             d = defer.succeed(None)
         d.addCallback(lambda r: self._scp_sshx(hostname))
         d.addCallback(lambda r: self._ssh_engine(hostname, count))
         return d
         
     def _scp_furl(self, hostname):
         scp_cmd = "scp ~/.ipython/security/ipcontroller-engine.furl %s:.ipython/security/" % (hostname)
         cmd_list = scp_cmd.split()
         cmd_list[1] = os.path.expanduser(cmd_list[1])
         log.msg('Copying furl file: %s' % scp_cmd)
         d = getProcessOutput(cmd_list[0], cmd_list[1:], env=os.environ) 
         return d
     
     def _scp_sshx(self, hostname):
         scp_cmd = "scp %s %s:%s/%s-sshx.sh" % (
             self.sshx, hostname, 
             self.temp_dir, os.environ['USER']
         )
         print
         log.msg("Copying sshx: %s" % scp_cmd)
         sshx_scp = scp_cmd.split()
         d = getProcessOutput(sshx_scp[0], sshx_scp[1:], env=os.environ)
         return d
     
     def _ssh_engine(self, hostname, count):
         exec_engine = "ssh %s sh %s/%s-sshx.sh %s" % (
             hostname, self.temp_dir, 
             os.environ['USER'], self.engine_command
         )
         cmds = exec_engine.split()
         dlist = []
         log.msg("about to start engines...")
         for i in range(count):
             log.msg('Starting engines: %s' % exec_engine)
             d = getProcessOutput(cmds[0], cmds[1:], env=os.environ)
             dlist.append(d)
         return gatherBoth(dlist, consumeErrors=True)
     
     def kill(self):
         dlist = []
         for host in self.engine_hosts.keys():
             d = self._killall(host)
             dlist.append(d)
         return gatherBoth(dlist, consumeErrors=True)
     
     def _killall(self, hostname):
         d = self._scp_engine_killer(hostname)
         d.addCallback(lambda r: self._ssh_kill(hostname))
         # d.addErrback(self._exec_err)
         return d
 
     def _scp_engine_killer(self, hostname):
         scp_cmd = "scp %s %s:%s/%s-engine_killer.sh" % (
             self.engine_killer, 
             hostname, 
             self.temp_dir, 
             os.environ['USER']
         )
         cmds = scp_cmd.split()
         log.msg('Copying engine_killer: %s' % scp_cmd)
         d = getProcessOutput(cmds[0], cmds[1:], env=os.environ)
         return d
     
     def _ssh_kill(self, hostname):
         kill_cmd = "ssh %s sh %s/%s-engine_killer.sh" % (
             hostname,
             self.temp_dir, 
             os.environ['USER']
         )
         log.msg('Killing engine: %s' % kill_cmd)
         kill_cmd = kill_cmd.split()
         d = getProcessOutput(kill_cmd[0], kill_cmd[1:], env=os.environ)
         return d
 
     def _exec_err(self, r):
         log.msg(r)
 
 #-----------------------------------------------------------------------------
 # Main functions for the different types of clusters
 #-----------------------------------------------------------------------------
 
 # TODO:
 # The logic in these codes should be moved into classes like LocalCluster
 # MpirunCluster, PBSCluster, etc.  This would remove alot of the duplications.
 # The main functions should then just parse the command line arguments, create
 # the appropriate class and call a 'start' method.
 
 
 def check_security(args, cont_args):
     """Check to see if we should run with SSL support."""
     if (not args.x or not args.y) and not have_crypto:
         log.err("""
 OpenSSL/pyOpenSSL is not available, so we can't run in secure mode.
 Try running ipcluster with the -xy flags:  ipcluster local -xy -n 4""")
         reactor.stop()
         return False
     if args.x:
         cont_args.append('-x')
     if args.y:
         cont_args.append('-y')
     return True
 
 
 def check_reuse(args, cont_args):
     """Check to see if we should try to resuse FURL files."""
     if args.r:
         cont_args.append('-r')
         if args.client_port == 0 or args.engine_port == 0:
             log.err("""
 To reuse FURL files, you must also set the client and engine ports using
 the --client-port and --engine-port options.""")
             reactor.stop()
             return False
         cont_args.append('--client-port=%i' % args.client_port)
         cont_args.append('--engine-port=%i' % args.engine_port)
     return True
 
 
 def _err_and_stop(f):
     """Errback to log a failure and halt the reactor on a fatal error."""
     log.err(f)
     reactor.stop()
 
 
 def _delay_start(cont_pid, start_engines, furl_file, reuse):
     """Wait for controller to create FURL files and the start the engines."""
     if not reuse:
         if os.path.isfile(furl_file):
             os.unlink(furl_file)
     log.msg('Waiting for controller to finish starting...')
     d = wait_for_file(furl_file, delay=0.2, max_tries=50)
     d.addCallback(lambda _: log.msg('Controller started'))
     d.addCallback(lambda _: start_engines(cont_pid))
     return d
 
 
 def main_local(args):
     cont_args = []
     cont_args.append('--logfile=%s' % pjoin(args.logdir,'ipcontroller'))
     
     # Check security settings before proceeding
     if not check_security(args, cont_args):
         return
     
     # See if we are reusing FURL files
     if not check_reuse(args, cont_args):
         return
     
     cl = ControllerLauncher(extra_args=cont_args)
     dstart = cl.start()
     def start_engines(cont_pid):
         engine_args = []
         engine_args.append('--logfile=%s' % \
             pjoin(args.logdir,'ipengine%s-' % cont_pid))
         eset = LocalEngineSet(extra_args=engine_args)
         def shutdown(signum, frame):
             log.msg('Stopping local cluster')
             # We are still playing with the times here, but these seem
             # to be reliable in allowing everything to exit cleanly.
             eset.interrupt_then_kill(0.5)
             cl.interrupt_then_kill(0.5)
             reactor.callLater(1.0, reactor.stop)
         signal.signal(signal.SIGINT,shutdown)
         d = eset.start(args.n)
         return d
     config = kernel_config_manager.get_config_obj()
     furl_file = config['controller']['engine_furl_file']
     dstart.addCallback(_delay_start, start_engines, furl_file, args.r)
     dstart.addErrback(_err_and_stop)
 
 
 def main_mpi(args):
     cont_args = []
     cont_args.append('--logfile=%s' % pjoin(args.logdir,'ipcontroller'))
     
     # Check security settings before proceeding
     if not check_security(args, cont_args):
         return
     
     # See if we are reusing FURL files
     if not check_reuse(args, cont_args):
         return
     
     cl = ControllerLauncher(extra_args=cont_args)
     dstart = cl.start()
     def start_engines(cont_pid):
         raw_args = [args.cmd]
         raw_args.extend(['-n',str(args.n)])
         raw_args.append('ipengine')
         raw_args.append('-l')
         raw_args.append(pjoin(args.logdir,'ipengine%s-' % cont_pid))
         if args.mpi:
             raw_args.append('--mpi=%s' % args.mpi)
         eset = ProcessLauncher(raw_args)
         def shutdown(signum, frame):
             log.msg('Stopping local cluster')
             # We are still playing with the times here, but these seem
             # to be reliable in allowing everything to exit cleanly.
             eset.interrupt_then_kill(1.0)
             cl.interrupt_then_kill(1.0)
             reactor.callLater(2.0, reactor.stop)
         signal.signal(signal.SIGINT,shutdown)
         d = eset.start()
         return d
     config = kernel_config_manager.get_config_obj()
     furl_file = config['controller']['engine_furl_file']
     dstart.addCallback(_delay_start, start_engines, furl_file, args.r)
     dstart.addErrback(_err_and_stop)
 
 
 def main_pbs(args):
     cont_args = []
     cont_args.append('--logfile=%s' % pjoin(args.logdir,'ipcontroller'))
     
     # Check security settings before proceeding
     if not check_security(args, cont_args):
         return
     
     # See if we are reusing FURL files
     if not check_reuse(args, cont_args):
         return
     
     cl = ControllerLauncher(extra_args=cont_args)
     dstart = cl.start()
     def start_engines(r):
         pbs_set =  PBSEngineSet(args.pbsscript)
         def shutdown(signum, frame):
             log.msg('Stopping pbs cluster')
             d = pbs_set.kill()
             d.addBoth(lambda _: cl.interrupt_then_kill(1.0))
             d.addBoth(lambda _: reactor.callLater(2.0, reactor.stop))
         signal.signal(signal.SIGINT,shutdown)
         d = pbs_set.start(args.n)
         return d
     config = kernel_config_manager.get_config_obj()
     furl_file = config['controller']['engine_furl_file']
     dstart.addCallback(_delay_start, start_engines, furl_file, args.r)
     dstart.addErrback(_err_and_stop)
 
 def main_sge(args):
     cont_args = []
     cont_args.append('--logfile=%s' % pjoin(args.logdir,'ipcontroller'))
     
     # Check security settings before proceeding
     if not check_security(args, cont_args):
         return
     
     # See if we are reusing FURL files
     if not check_reuse(args, cont_args):
         return
     
     cl = ControllerLauncher(extra_args=cont_args)
     dstart = cl.start()
     def start_engines(r):
         sge_set =  SGEEngineSet(args.sgescript)
         def shutdown(signum, frame):
             log.msg('Stopping sge cluster')
             d = sge_set.kill()
             d.addBoth(lambda _: cl.interrupt_then_kill(1.0))
             d.addBoth(lambda _: reactor.callLater(2.0, reactor.stop))
         signal.signal(signal.SIGINT,shutdown)
         d = sge_set.start(args.n)
         return d
     config = kernel_config_manager.get_config_obj()
     furl_file = config['controller']['engine_furl_file']
     dstart.addCallback(_delay_start, start_engines, furl_file, args.r)
     dstart.addErrback(_err_and_stop)
 
 
 def main_ssh(args):
     """Start a controller on localhost and engines using ssh.
     
     Your clusterfile should look like::
     
         send_furl = False # True, if you want 
         engines = {
             'engine_host1' : engine_count, 
             'engine_host2' : engine_count2
         } 
     """
     clusterfile = {}
     execfile(args.clusterfile, clusterfile)
     if not clusterfile.has_key('send_furl'):
         clusterfile['send_furl'] = False
         
     cont_args = []
     cont_args.append('--logfile=%s' % pjoin(args.logdir,'ipcontroller'))
     
     # Check security settings before proceeding
     if not check_security(args, cont_args):
         return
     
     # See if we are reusing FURL files
     if not check_reuse(args, cont_args):
         return
     
     cl = ControllerLauncher(extra_args=cont_args)
     dstart = cl.start()
     def start_engines(cont_pid):
         ssh_set = SSHEngineSet(clusterfile['engines'], sshx=args.sshx)
         def shutdown(signum, frame):
             d = ssh_set.kill()
             cl.interrupt_then_kill(1.0)
             reactor.callLater(2.0, reactor.stop)
         signal.signal(signal.SIGINT,shutdown)
         d = ssh_set.start(clusterfile['send_furl'])
         return d
     config = kernel_config_manager.get_config_obj()
     furl_file = config['controller']['engine_furl_file']
     dstart.addCallback(_delay_start, start_engines, furl_file, args.r)
     dstart.addErrback(_err_and_stop)
 
 
 def get_args():
     base_parser = argparse.ArgumentParser(add_help=False)
     base_parser.add_argument(
         '-r',
         action='store_true',
         dest='r',
         help='try to reuse FURL files.  Use with --client-port and --engine-port'
     )
     base_parser.add_argument(
         '--client-port',
         type=int,
         dest='client_port',
         help='the port the controller will listen on for client connections',
         default=0
     )
     base_parser.add_argument(
         '--engine-port',
         type=int,
         dest='engine_port',
         help='the port the controller will listen on for engine connections',
         default=0
     )
     base_parser.add_argument(
         '-x',
         action='store_true',
         dest='x',
         help='turn off client security'
     )
     base_parser.add_argument(
         '-y',
         action='store_true',
         dest='y',
         help='turn off engine security'
     )
     base_parser.add_argument(
         "--logdir", 
         type=str,
         dest="logdir",
         help="directory to put log files (default=$IPYTHONDIR/log)",
         default=pjoin(get_ipython_dir(),'log')
     )
     base_parser.add_argument(
         "-n",
         "--num", 
         type=int,
         dest="n",
         default=2,
         help="the number of engines to start"
     )
     
     parser = argparse.ArgumentParser(
         description='IPython cluster startup.  This starts a controller and\
         engines using various approaches.  Use the IPYTHONDIR environment\
         variable to change your IPython directory from the default of\
         .ipython or _ipython.  The log and security subdirectories of your\
         IPython directory will be used by this script for log files and\
         security files.'
     )
     subparsers = parser.add_subparsers(
         help='available cluster types.  For help, do "ipcluster TYPE --help"')
     
     parser_local = subparsers.add_parser(
         'local',
         help='run a local cluster',
         parents=[base_parser]
     )
     parser_local.set_defaults(func=main_local)
     
     parser_mpirun = subparsers.add_parser(
         'mpirun',
         help='run a cluster using mpirun (mpiexec also works)',
         parents=[base_parser]
     )
     parser_mpirun.add_argument(
         "--mpi",
         type=str,
         dest="mpi", # Don't put a default here to allow no MPI support
         help="how to call MPI_Init (default=mpi4py)"
     )
     parser_mpirun.set_defaults(func=main_mpi, cmd='mpirun')
 
     parser_mpiexec = subparsers.add_parser(
         'mpiexec',
         help='run a cluster using mpiexec (mpirun also works)',
         parents=[base_parser]
     )
     parser_mpiexec.add_argument(
         "--mpi",
         type=str,
         dest="mpi", # Don't put a default here to allow no MPI support
         help="how to call MPI_Init (default=mpi4py)"
     )
     parser_mpiexec.set_defaults(func=main_mpi, cmd='mpiexec')
     
     parser_pbs = subparsers.add_parser(
         'pbs', 
         help='run a pbs cluster',
         parents=[base_parser]
     )
     parser_pbs.add_argument(
         '--pbs-script',
         type=str, 
         dest='pbsscript',
         help='PBS script template',
         default='pbs.template'
     )
     parser_pbs.set_defaults(func=main_pbs)
     
     parser_sge = subparsers.add_parser(
         'sge', 
         help='run an sge cluster',
         parents=[base_parser]
     )
     parser_sge.add_argument(
         '--sge-script',
         type=str, 
         dest='sgescript',
         help='SGE script template',
         default='template.sge'
     )
     parser_sge.set_defaults(func=main_sge)
     
     parser_ssh = subparsers.add_parser(
         'ssh',
         help='run a cluster using ssh, should have ssh-keys setup',
         parents=[base_parser]
     )
     parser_ssh.add_argument(
         '--clusterfile', 
         type=str,
         dest='clusterfile',
         help='python file describing the cluster',
         default='clusterfile.py',
     )
     parser_ssh.add_argument(
         '--sshx', 
         type=str,
         dest='sshx',
         help='sshx launcher helper'
     )
     parser_ssh.set_defaults(func=main_ssh)
     
     args = parser.parse_args()
     return args
 
 def main():
     args = get_args()
     reactor.callWhenRunning(args.func, args)
     log.startLogging(sys.stdout)
     reactor.run()
 
 if __name__ == '__main__':
     main()

docs/source/parallel/parallel_process.txt

@@ -1,400 +1,397 b''
 .. _parallel_process:
 
 ===========================================
 Starting the IPython controller and engines
 ===========================================
 
 To use IPython for parallel computing, you need to start one instance of
 the controller and one or more instances of the engine. The controller
 and each engine can run on different machines or on the same machine.
 Because of this, there are many different possibilities.
 
 Broadly speaking, there are two ways of going about starting a controller and engines:
 
 * In an automated manner using the :command:`ipcluster` command.
 * In a more manual way using the :command:`ipcontroller` and 
   :command:`ipengine` commands.
 
 This document describes both of these methods.  We recommend that new users start with the :command:`ipcluster` command as it simplifies many common usage cases.
 
 General considerations
 ======================
 
 Before delving into the details about how you can start a controller and engines using the various methods, we outline some of the general issues that come up when starting the controller and engines.  These things come up no matter which method you use to start your IPython cluster.
 
 Let's say that you want to start the controller on ``host0`` and engines on hosts ``host1``-``hostn``.  The following steps are then required:
 
 1. Start the controller on ``host0`` by running :command:`ipcontroller` on
    ``host0``.
 2. Move the FURL file (:file:`ipcontroller-engine.furl`) created by the 
    controller from ``host0`` to hosts ``host1``-``hostn``.
 3. Start the engines on hosts ``host1``-``hostn`` by running 
    :command:`ipengine`.  This command has to be told where the FURL file 
    (:file:`ipcontroller-engine.furl`) is located.
 
 At this point, the controller and engines will be connected. By default, the
 FURL files created by the controller are put into the
 :file:`~/.ipython/security` directory. If the engines share a filesystem with
 the controller, step 2 can be skipped as the engines will automatically look
 at that location.
 
 The final step required required to actually use the running controller from a
 client is to move the FURL files :file:`ipcontroller-mec.furl` and
 :file:`ipcontroller-tc.furl` from ``host0`` to the host where the clients will
 be run.  If these file are put into the :file:`~/.ipython/security` directory of the client's host, they will be found automatically.  Otherwise, the full path to them has to be passed to the client's constructor.
 
 Using :command:`ipcluster`
 ==========================
 
 The :command:`ipcluster` command provides a simple way of starting a controller and engines in the following situations:
 
 1. When the controller and engines are all run on localhost. This is useful
    for testing or running on a multicore computer.
 2. When engines are started using the :command:`mpirun` command that comes 
    with most MPI [MPI]_ implementations
 3. When engines are started using the PBS [PBS]_ batch system.
 4. When engines are started using the SGE [SGE]_ batch system.
 5. When the controller is started on localhost and the engines are started on 
    remote nodes using :command:`ssh`.
 
 .. note::
 
     It is also possible for advanced users to add support to
     :command:`ipcluster` for starting controllers and engines using other
     methods (like Sun's Grid Engine for example).
 
 .. note::
 
     Currently :command:`ipcluster` requires that the
     :file:`~/.ipython/security` directory live on a shared filesystem that is
     seen by both the controller and engines. If you don't have a shared file
     system you will need to use :command:`ipcontroller` and
     :command:`ipengine` directly.  This constraint can be relaxed if you are 
     using the :command:`ssh` method to start the cluster.
 
 Underneath the hood, :command:`ipcluster` just uses :command:`ipcontroller`
 and :command:`ipengine` to perform the steps described above.
 
 Using :command:`ipcluster` in local mode
 ----------------------------------------
 
 To start one controller and 4 engines on localhost, just do::
 
     $ ipcluster local -n 4
 
 To see other command line options for the local mode, do::
 
     $ ipcluster local -h
 
 Using :command:`ipcluster` in mpiexec/mpirun mode
 -------------------------------------------------
 
 The mpiexec/mpirun mode is useful if you:
 
 1. Have MPI installed.
 2. Your systems are configured to use the :command:`mpiexec` or
    :command:`mpirun` commands to start MPI processes.
 
 .. note::
 
     The preferred command to use is :command:`mpiexec`.  However, we also 
     support :command:`mpirun` for backwards compatibility.  The underlying
     logic used is exactly the same, the only difference being the name of the 
     command line program that is called.
 
 If these are satisfied, you can start an IPython cluster using::
 
     $ ipcluster mpiexec -n 4
 
 This does the following:
 
 1. Starts the IPython controller on current host.
 2. Uses :command:`mpiexec` to start 4 engines.
 
 On newer MPI implementations (such as OpenMPI), this will work even if you don't make any calls to MPI or call :func:`MPI_Init`.  However, older MPI implementations actually require each process to call :func:`MPI_Init` upon starting.  The easiest way of having this done is to install the mpi4py [mpi4py]_ package and then call ipcluster with the ``--mpi`` option::
 
     $ ipcluster mpiexec -n 4 --mpi=mpi4py
 
 Unfortunately, even this won't work for some MPI implementations.  If you are having problems with this, you will likely have to use a custom Python executable that itself calls :func:`MPI_Init` at the appropriate time.  Fortunately, mpi4py comes with such a custom Python executable that is easy to install and use.  However, this custom Python executable approach will not work with :command:`ipcluster` currently.
 
 Additional command line options for this mode can be found by doing::
 
     $ ipcluster mpiexec -h
 
 More details on using MPI with IPython can be found :ref:`here <parallelmpi>`.
 
 
 Using :command:`ipcluster` in PBS mode
 --------------------------------------
 
 The PBS mode uses the Portable Batch System [PBS]_ to start the engines.  To use this mode, you first need to create a PBS script template that will be used to start the engines.  Here is a sample PBS script template:
 
 .. sourcecode:: bash
 
     #PBS -N ipython
     #PBS -j oe
     #PBS -l walltime=00:10:00
     #PBS -l nodes=${n/4}:ppn=4
     #PBS -q parallel
 
     cd $$PBS_O_WORKDIR
     export PATH=$$HOME/usr/local/bin
     export PYTHONPATH=$$HOME/usr/local/lib/python2.4/site-packages
     /usr/local/bin/mpiexec -n ${n} ipengine --logfile=$$PBS_O_WORKDIR/ipengine
 
 There are a few important points about this template:
 
 1. This template will be rendered at runtime using IPython's :mod:`Itpl` 
    template engine.
 
 2. Instead of putting in the actual number of engines, use the notation
    ``${n}`` to indicate the number of engines to be started. You can also uses
    expressions like ``${n/4}`` in the template to indicate the number of 
    nodes.
 
 3. Because ``$`` is a special character used by the template engine, you must  
    escape any ``$`` by using ``$$``.  This is important when referring to 
    environment variables in the template.
 
 4. Any options to :command:`ipengine` should be given in the batch script 
    template.
 
 5. Depending on the configuration of you system, you may have to set 
    environment variables in the script template.
 
 Once you have created such a script, save it with a name like :file:`pbs.template`.  Now you are ready to start your job::
 
     $ ipcluster pbs -n 128 --pbs-script=pbs.template
 
 Additional command line options for this mode can be found by doing::
 
     $ ipcluster pbs -h
 
 Using :command:`ipcluster` in SGE mode
 --------------------------------------
 
 The SGE mode uses the Sun Grid Engine [SGE]_ to start the engines.  To use this mode, you first need to create a SGE script template that will be used to start the engines.  Here is a sample SGE script template:
 
 .. sourcecode:: bash
 
     #!/bin/bash
     #$ -V
-    #$ -cwd
     #$ -m n
-    #$ -N satra-ipython
+    #$ -N ipengine-${eid}
     #$ -r y
     #$ -q sub
     #$ -S /bin/bash
 
     cd $$HOME/sge
-    ipengine --logfile=ipengine
+    ipengine --logfile=ipengine${eid}
 
 There are a few important points about this template:
 
 1. This template will be rendered at runtime using IPython's :mod:`Itpl` 
    template engine.
 
-2. Instead of putting in the actual number of engines, use the notation
-   ``${n}`` to indicate the number of engines to be started. You can also uses
-   expressions like ``${n/4}`` in the template to indicate the number of 
-   nodes.
+2. Instead of putting in the actual id of engines, use the notation
+   ``${eid}`` to indicate where engine id should be inserted.
 
 3. Because ``$`` is a special character used by the template engine, you must  
    escape any ``$`` by using ``$$``.  This is important when referring to 
    environment variables in the template.
 
 4. Any options to :command:`ipengine` should be given in the batch script 
    template.
 
 5. Depending on the configuration of you system, you may have to set 
    environment variables in the script template.
 
 Once you have created such a script, save it with a name like :file:`sge.template`.  Now you are ready to start your job::
 
-    $ ipcluster sge -n 128 --sge-script=sge.template
+    $ ipcluster sge -n 12 --sge-script=sge.template
 
 Additional command line options for this mode can be found by doing::
 
     $ ipcluster sge -h
 
 Using :command:`ipcluster` in SSH mode
 --------------------------------------
 
 The SSH mode uses :command:`ssh` to execute :command:`ipengine` on remote
 nodes and the :command:`ipcontroller` on localhost.
 
 When using using this mode it highly recommended that you have set up SSH keys and are using ssh-agent [SSH]_ for password-less logins.
 
 To use this mode you need a python file describing the cluster, here is an example of such a "clusterfile":
 
 .. sourcecode:: python
     
     send_furl = True
     engines = { 'host1.example.com' : 2,
                 'host2.example.com' : 5,
                 'host3.example.com' : 1,
                 'host4.example.com' : 8 }
 
 Since this is a regular python file usual python syntax applies. Things to note:
 
 * The `engines` dict, where the keys is the host we want to run engines on and
   the value is the number of engines to run on that host.
 * send_furl can either be `True` or `False`, if `True` it will copy over the
   furl needed for :command:`ipengine` to each host.
 
 The ``--clusterfile`` command line option lets you specify the file to use for
 the cluster definition. Once you have your cluster file and you can
 :command:`ssh` into the remote hosts with out an password you are ready to
 start your cluster like so:
 
 .. sourcecode:: bash 
 
    $ ipcluster ssh --clusterfile /path/to/my/clusterfile.py
 
 
 Two helper shell scripts are used to start and stop :command:`ipengine` on remote hosts:
 
 * sshx.sh 
 * engine_killer.sh
 
 Defaults for both of these are contained in the source code for :command:`ipcluster`. The default scripts are written to a local file in a tmep directory and then copied to a temp directory on the remote host and executed from there.  On most Unix, Linux and OS X systems this is /tmp.
 
 The default sshx.sh is the following:
 
 .. sourcecode:: bash
 
    #!/bin/sh
    "$@" &> /dev/null &
    echo $!
 
 If you want to use a custom sshx.sh script you need to use the ``--sshx``
 option and specify the file to use. Using a custom sshx.sh file could be
 helpful when you need to setup the environment on the remote host before
 executing :command:`ipengine`.
 
 For a detailed options list:
 
 .. sourcecode:: bash
 
    $ ipcluster ssh -h
 
 Current limitations of the SSH mode of :command:`ipcluster` are:
 
 * Untested on Windows. Would require a working :command:`ssh` on Windows.
   Also, we are using shell scripts to setup and execute commands on remote
   hosts.
 * :command:`ipcontroller` is started on localhost, with no option to start it
   on a remote node.
 
 Using the :command:`ipcontroller` and :command:`ipengine` commands
 ==================================================================
 
 It is also possible to use the :command:`ipcontroller` and :command:`ipengine` commands to start your controller and engines.  This approach gives you full control over all aspects of the startup process.
 
 Starting the controller and engine on your local machine
 --------------------------------------------------------
 
 To use :command:`ipcontroller` and :command:`ipengine` to start things on your
 local machine, do the following.
 
 First start the controller::
 
 	$ ipcontroller
 	
 Next, start however many instances of the engine you want using (repeatedly) the command::
 
 	$ ipengine
 
 The engines should start and automatically connect to the controller using the FURL files in :file:`~./ipython/security`. You are now ready to use the controller and engines from IPython.
 
 .. warning:: 
 	
 	The order of the above operations is very important.  You *must*
  	start the controller before the engines, since the engines connect 
 	to the controller as they get started.
 
 .. note::
 
     On some platforms (OS X), to put the controller and engine into the
     background you may need to give these commands in the form ``(ipcontroller
     &)`` and ``(ipengine &)`` (with the parentheses) for them to work
     properly.
 
 Starting the controller and engines on different hosts
 ------------------------------------------------------
 
 When the controller and engines are running on different hosts, things are
 slightly more complicated, but the underlying ideas are the same:
 
 1. Start the controller on a host using :command:`ipcontroller`.
 2. Copy :file:`ipcontroller-engine.furl` from :file:`~./ipython/security` on the controller's host to the host where the engines will run.
 3. Use :command:`ipengine` on the engine's hosts to start the engines.
 
 The only thing you have to be careful of is to tell :command:`ipengine` where the :file:`ipcontroller-engine.furl` file is located.  There are two ways you can do this:
 
 * Put :file:`ipcontroller-engine.furl` in the :file:`~./ipython/security`
   directory on the engine's host, where it will be found automatically.
 * Call :command:`ipengine` with the ``--furl-file=full_path_to_the_file``
   flag.
 
 The ``--furl-file`` flag works like this::
 
     $ ipengine --furl-file=/path/to/my/ipcontroller-engine.furl
 
 .. note::
     
     If the controller's and engine's hosts all have a shared file system  
     (:file:`~./ipython/security` is the same on all of them), then things
     will just work!
 
 Make FURL files persistent
 ---------------------------
 
 At fist glance it may seem that that managing the FURL files is a bit
 annoying. Going back to the house and key analogy, copying the FURL around
 each time you start the controller is like having to make a new key every time
 you want to unlock the door and enter your house. As with your house, you want
 to be able to create the key (or FURL file) once, and then simply use it at
 any point in the future.
 
 This is possible, but before you do this, you **must** remove any old FURL
 files in the :file:`~/.ipython/security` directory.
 
 .. warning::
 
     You **must** remove old FURL files before using persistent FURL files.
 
 Then, The only thing you have to do is decide what ports the controller will
 listen on for the engines and clients. This is done as follows::
 
     $ ipcontroller -r --client-port=10101 --engine-port=10102
 
 These options also work with all of the various modes of
 :command:`ipcluster`::
 
     $ ipcluster local -n 2 -r --client-port=10101 --engine-port=10102
 
 Then, just copy the furl files over the first time and you are set. You can
 start and stop the controller and engines any many times as you want in the
 future, just make sure to tell the controller to use the *same* ports.
 
 .. note::
 
     You may ask the question: what ports does the controller listen on if you
     don't tell is to use specific ones? The default is to use high random port
     numbers. We do this for two reasons: i) to increase security through
     obscurity and ii) to multiple controllers on a given host to start and
     automatically use different ports.
 
 Log files
 ---------
 
 All of the components of IPython have log files associated with them.
 These log files can be extremely useful in debugging problems with
 IPython and can be found in the directory :file:`~/.ipython/log`.  Sending
 the log files to us will often help us to debug any problems.
 
 
 .. [PBS] Portable Batch System.  http://www.openpbs.org/
 .. [SGE] Sun Grid Engine.  http://www.sun.com/software/sge/
 .. [SSH] SSH-Agent http://en.wikipedia.org/wiki/Ssh-agent