Merge remote branch 'trunk/newparallel'

IPython/external/ssh/forward.py

@@ -0,0 +1,90 b''
+#!/usr/bin/env python
+
+#
+# This file is adapted from a paramiko demo, and thus licensed under LGPL 2.1.
+# Original Copyright (C) 2003-2007  Robey Pointer <[e-mail]>
+# Edits Copyright (C) 2010 The IPython Team
+#
+# Paramiko is free software; you can redistribute it and/or modify it under the
+# terms of the GNU Lesser General Public License as published by the Free
+# Software Foundation; either version 2.1 of the License, or (at your option)
+# any later version.
+#
+# Paramiko is distrubuted in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+# A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with Paramiko; if not, write to the Free Software Foundation, Inc.,
+# 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA.
+
+"""
+Sample script showing how to do local port forwarding over paramiko.
+
+This script connects to the requested SSH server and sets up local port
+forwarding (the openssh -L option) from a local port through a tunneled
+connection to a destination reachable from the SSH server machine.
+"""
+
+from __future__ import print_function
+
+import logging
+import select
+import SocketServer
+
+logger = logging.getLogger('ssh')
+
+class ForwardServer (SocketServer.ThreadingTCPServer):
+    daemon_threads = True
+    allow_reuse_address = True
+    
+
+class Handler (SocketServer.BaseRequestHandler):
+
+    def handle(self):
+        try:
+            chan = self.ssh_transport.open_channel('direct-tcpip',
+                                                   (self.chain_host, self.chain_port),
+                                                   self.request.getpeername())
+        except Exception, e:
+            logger.debug('Incoming request to %s:%d failed: %s' % (self.chain_host,
+                                                              self.chain_port,
+                                                              repr(e)))
+            return
+        if chan is None:
+            logger.debug('Incoming request to %s:%d was rejected by the SSH server.' %
+                    (self.chain_host, self.chain_port))
+            return
+
+        logger.debug('Connected!  Tunnel open %r -> %r -> %r' % (self.request.getpeername(),
+                                                            chan.getpeername(), (self.chain_host, self.chain_port)))
+        while True:
+            r, w, x = select.select([self.request, chan], [], [])
+            if self.request in r:
+                data = self.request.recv(1024)
+                if len(data) == 0:
+                    break
+                chan.send(data)
+            if chan in r:
+                data = chan.recv(1024)
+                if len(data) == 0:
+                    break
+                self.request.send(data)
+        chan.close()
+        self.request.close()
+        logger.debug('Tunnel closed ')
+
+
+def forward_tunnel(local_port, remote_host, remote_port, transport):
+    # this is a little convoluted, but lets me configure things for the Handler
+    # object.  (SocketServer doesn't give Handlers any way to access the outer
+    # server normally.)
+    class SubHander (Handler):
+        chain_host = remote_host
+        chain_port = remote_port
+        ssh_transport = transport
+    ForwardServer(('127.0.0.1', local_port), SubHander).serve_forever()
+
+
+__all__ = ['forward_tunnel']
  No newline at end of file

IPython/external/ssh/tunnel.py

@@ -0,0 +1,295 b''
+"""Basic ssh tunneling utilities."""
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2010  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+from __future__ import print_function
+
+import os,sys, atexit
+from multiprocessing import Process
+from getpass import getpass, getuser
+import warnings
+
+try:
+    with warnings.catch_warnings():
+        warnings.simplefilter('ignore', DeprecationWarning)
+        import paramiko
+except ImportError:
+    paramiko = None
+else:
+    from forward import forward_tunnel
+
+try:
+    from IPython.external import pexpect
+except ImportError:
+    pexpect = None
+
+from IPython.parallel.util import select_random_ports
+
+#-----------------------------------------------------------------------------
+# Code
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Check for passwordless login
+#-----------------------------------------------------------------------------
+
+def try_passwordless_ssh(server, keyfile, paramiko=None):
+    """Attempt to make an ssh connection without a password.
+    This is mainly used for requiring password input only once
+    when many tunnels may be connected to the same server.
+    
+    If paramiko is None, the default for the platform is chosen.
+    """
+    if paramiko is None:
+        paramiko = sys.platform == 'win32'
+    if not paramiko:
+        f = _try_passwordless_openssh
+    else:
+        f = _try_passwordless_paramiko
+    return f(server, keyfile)
+
+def _try_passwordless_openssh(server, keyfile):
+    """Try passwordless login with shell ssh command."""
+    if pexpect is None:
+        raise ImportError("pexpect unavailable, use paramiko")
+    cmd = 'ssh -f '+ server
+    if keyfile:
+        cmd += ' -i ' + keyfile
+    cmd += ' exit'
+    p = pexpect.spawn(cmd)
+    while True:
+        try:
+            p.expect('[Ppassword]:', timeout=.1)
+        except pexpect.TIMEOUT:
+            continue
+        except pexpect.EOF:
+            return True
+        else:
+            return False
+
+def _try_passwordless_paramiko(server, keyfile):
+    """Try passwordless login with paramiko."""
+    if paramiko is None:
+        raise ImportError("paramiko unavailable, use openssh")
+    username, server, port = _split_server(server)
+    client = paramiko.SSHClient()
+    client.load_system_host_keys()
+    client.set_missing_host_key_policy(paramiko.WarningPolicy())
+    try:
+        client.connect(server, port, username=username, key_filename=keyfile,
+               look_for_keys=True)
+    except paramiko.AuthenticationException:
+        return False
+    else:
+        client.close()
+        return True
+
+
+def tunnel_connection(socket, addr, server, keyfile=None, password=None, paramiko=None):
+    """Connect a socket to an address via an ssh tunnel.
+    
+    This is a wrapper for socket.connect(addr), when addr is not accessible
+    from the local machine.  It simply creates an ssh tunnel using the remaining args,
+    and calls socket.connect('tcp://localhost:lport') where lport is the randomly
+    selected local port of the tunnel.
+    
+    """
+    lport = select_random_ports(1)[0]
+    transport, addr = addr.split('://')
+    ip,rport = addr.split(':')
+    rport = int(rport)
+    if paramiko is None:
+        paramiko = sys.platform == 'win32'
+    if paramiko:
+        tunnelf = paramiko_tunnel
+    else:
+        tunnelf = openssh_tunnel
+    tunnel = tunnelf(lport, rport, server, remoteip=ip, keyfile=keyfile, password=password)
+    socket.connect('tcp://127.0.0.1:%i'%lport)
+    return tunnel
+
+def openssh_tunnel(lport, rport, server, remoteip='127.0.0.1', keyfile=None, password=None, timeout=15):
+    """Create an ssh tunnel using command-line ssh that connects port lport
+    on this machine to localhost:rport on server.  The tunnel
+    will automatically close when not in use, remaining open
+    for a minimum of timeout seconds for an initial connection.
+    
+    This creates a tunnel redirecting `localhost:lport` to `remoteip:rport`,
+    as seen from `server`.
+    
+    keyfile and password may be specified, but ssh config is checked for defaults.
+    
+    Parameters
+    ----------
+    
+        lport : int
+            local port for connecting to the tunnel from this machine.
+        rport : int
+            port on the remote machine to connect to.
+        server : str
+            The ssh server to connect to. The full ssh server string will be parsed.
+            user@server:port
+        remoteip : str [Default: 127.0.0.1]
+            The remote ip, specifying the destination of the tunnel.
+            Default is localhost, which means that the tunnel would redirect
+            localhost:lport on this machine to localhost:rport on the *server*.
+        
+        keyfile : str; path to public key file
+            This specifies a key to be used in ssh login, default None.
+            Regular default ssh keys will be used without specifying this argument.
+        password : str; 
+            Your ssh password to the ssh server. Note that if this is left None,
+            you will be prompted for it if passwordless key based login is unavailable.
+    
+    """
+    if pexpect is None:
+        raise ImportError("pexpect unavailable, use paramiko_tunnel")
+    ssh="ssh "
+    if keyfile:
+        ssh += "-i " + keyfile 
+    cmd = ssh + " -f -L 127.0.0.1:%i:%s:%i %s sleep %i"%(lport, remoteip, rport, server, timeout)
+    tunnel = pexpect.spawn(cmd)
+    failed = False
+    while True:
+        try:
+            tunnel.expect('[Pp]assword:', timeout=.1)
+        except pexpect.TIMEOUT:
+            continue
+        except pexpect.EOF:
+            if tunnel.exitstatus:
+                print (tunnel.exitstatus)
+                print (tunnel.before)
+                print (tunnel.after)
+                raise RuntimeError("tunnel '%s' failed to start"%(cmd))
+            else:
+                return tunnel.pid
+        else:
+            if failed:
+                print("Password rejected, try again")
+                password=None
+            if password is None:
+                password = getpass("%s's password: "%(server))
+            tunnel.sendline(password)
+            failed = True
+    
+def _split_server(server):
+    if '@' in server:
+        username,server = server.split('@', 1)
+    else:
+        username = getuser()
+    if ':' in server:
+        server, port = server.split(':')
+        port = int(port)
+    else:
+        port = 22
+    return username, server, port
+
+def paramiko_tunnel(lport, rport, server, remoteip='127.0.0.1', keyfile=None, password=None, timeout=15):
+    """launch a tunner with paramiko in a subprocess. This should only be used
+    when shell ssh is unavailable (e.g. Windows).
+    
+    This creates a tunnel redirecting `localhost:lport` to `remoteip:rport`,
+    as seen from `server`.
+    
+    If you are familiar with ssh tunnels, this creates the tunnel:
+    
+    ssh server -L localhost:lport:remoteip:rport
+    
+    keyfile and password may be specified, but ssh config is checked for defaults.
+    
+    
+    Parameters
+    ----------
+    
+        lport : int
+            local port for connecting to the tunnel from this machine.
+        rport : int
+            port on the remote machine to connect to.
+        server : str
+            The ssh server to connect to. The full ssh server string will be parsed.
+            user@server:port
+        remoteip : str [Default: 127.0.0.1]
+            The remote ip, specifying the destination of the tunnel.
+            Default is localhost, which means that the tunnel would redirect
+            localhost:lport on this machine to localhost:rport on the *server*.
+        
+        keyfile : str; path to public key file
+            This specifies a key to be used in ssh login, default None.
+            Regular default ssh keys will be used without specifying this argument.
+        password : str; 
+            Your ssh password to the ssh server. Note that if this is left None,
+            you will be prompted for it if passwordless key based login is unavailable.
+    
+    """
+    if paramiko is None:
+        raise ImportError("Paramiko not available")
+    
+    if password is None:
+        if not _check_passwordless_paramiko(server, keyfile):
+            password = getpass("%s's password: "%(server))
+
+    p = Process(target=_paramiko_tunnel, 
+            args=(lport, rport, server, remoteip), 
+            kwargs=dict(keyfile=keyfile, password=password))
+    p.daemon=False
+    p.start()
+    atexit.register(_shutdown_process, p)
+    return p
+    
+def _shutdown_process(p):
+    if p.isalive():
+        p.terminate()
+
+def _paramiko_tunnel(lport, rport, server, remoteip, keyfile=None, password=None):
+    """Function for actually starting a paramiko tunnel, to be passed
+    to multiprocessing.Process(target=this), and not called directly.
+    """
+    username, server, port = _split_server(server)
+    client = paramiko.SSHClient()
+    client.load_system_host_keys()
+    client.set_missing_host_key_policy(paramiko.WarningPolicy())
+
+    try:
+        client.connect(server, port, username=username, key_filename=keyfile,
+                       look_for_keys=True, password=password)
+#    except paramiko.AuthenticationException:
+#        if password is None:
+#            password = getpass("%s@%s's password: "%(username, server))
+#            client.connect(server, port, username=username, password=password)
+#        else:
+#            raise
+    except Exception as e:
+        print ('*** Failed to connect to %s:%d: %r' % (server, port, e))
+        sys.exit(1)
+
+    # print ('Now forwarding port %d to %s:%d ...' % (lport, server, rport))
+
+    try:
+        forward_tunnel(lport, remoteip, rport, client.get_transport())
+    except KeyboardInterrupt:
+        print ('SIGINT: Port forwarding stopped cleanly')
+        sys.exit(0)
+    except Exception as e:
+        print ("Port forwarding stopped uncleanly: %s"%e)
+        sys.exit(255)
+
+if sys.platform == 'win32':
+    ssh_tunnel = paramiko_tunnel
+else:
+    ssh_tunnel = openssh_tunnel
+
+    
+__all__ = ['tunnel_connection', 'ssh_tunnel', 'openssh_tunnel', 'paramiko_tunnel', 'try_passwordless_ssh']
+
+

IPython/parallel/__init__.py

@@ -0,0 +1,26 b''
+"""The IPython ZMQ-based parallel computing interface."""
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2011 The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+import zmq
+
+if zmq.__version__ < '2.1.4':
+    raise ImportError("IPython.parallel requires pyzmq/0MQ >= 2.1.4, you appear to have %s"%zmq.__version__)
+
+from IPython.utils.pickleutil import Reference
+
+from .client.asyncresult import *
+from .client.client import Client
+from .client.remotefunction import *
+from .client.view import *
+from .controller.dependency import *
+
+

IPython/parallel/apps/clusterdir.py

@@ -0,0 +1,537 b''
+#!/usr/bin/env python
+# encoding: utf-8
+"""
+The IPython cluster directory
+"""
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2009  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+from __future__ import with_statement
+
+import os
+import logging
+import re
+import shutil
+import sys
+
+from IPython.config.loader import PyFileConfigLoader
+from IPython.config.configurable import Configurable
+from IPython.core.application import Application, BaseAppConfigLoader
+from IPython.core.crashhandler import CrashHandler
+from IPython.core import release
+from IPython.utils.path import (
+    get_ipython_package_dir,
+    expand_path
+)
+from IPython.utils.traitlets import Unicode
+
+#-----------------------------------------------------------------------------
+# Module errors
+#-----------------------------------------------------------------------------
+
+class ClusterDirError(Exception):
+    pass
+
+
+class PIDFileError(Exception):
+    pass
+
+
+#-----------------------------------------------------------------------------
+# Class for managing cluster directories
+#-----------------------------------------------------------------------------
+
+class ClusterDir(Configurable):
+    """An object to manage the cluster directory and its resources.
+
+    The cluster directory is used by :command:`ipengine`, 
+    :command:`ipcontroller` and :command:`ipclsuter` to manage the
+    configuration, logging and security of these applications.
+
+    This object knows how to find, create and manage these directories. This
+    should be used by any code that want's to handle cluster directories.
+    """
+
+    security_dir_name = Unicode('security')
+    log_dir_name = Unicode('log')
+    pid_dir_name = Unicode('pid')
+    security_dir = Unicode(u'')
+    log_dir = Unicode(u'')
+    pid_dir = Unicode(u'')
+    location = Unicode(u'')
+
+    def __init__(self, location=u''):
+        super(ClusterDir, self).__init__(location=location)
+
+    def _location_changed(self, name, old, new):
+        if not os.path.isdir(new):
+            os.makedirs(new)
+        self.security_dir = os.path.join(new, self.security_dir_name)
+        self.log_dir = os.path.join(new, self.log_dir_name)
+        self.pid_dir = os.path.join(new, self.pid_dir_name)
+        self.check_dirs()
+
+    def _log_dir_changed(self, name, old, new):
+        self.check_log_dir()
+
+    def check_log_dir(self):
+        if not os.path.isdir(self.log_dir):
+            os.mkdir(self.log_dir)
+
+    def _security_dir_changed(self, name, old, new):
+        self.check_security_dir()
+
+    def check_security_dir(self):
+        if not os.path.isdir(self.security_dir):
+            os.mkdir(self.security_dir, 0700)
+        os.chmod(self.security_dir, 0700)
+
+    def _pid_dir_changed(self, name, old, new):
+        self.check_pid_dir()
+
+    def check_pid_dir(self):
+        if not os.path.isdir(self.pid_dir):
+            os.mkdir(self.pid_dir, 0700)
+        os.chmod(self.pid_dir, 0700)
+
+    def check_dirs(self):
+        self.check_security_dir()
+        self.check_log_dir()
+        self.check_pid_dir()
+
+    def load_config_file(self, filename):
+        """Load a config file from the top level of the cluster dir.
+
+        Parameters
+        ----------
+        filename : unicode or str
+            The filename only of the config file that must be located in
+            the top-level of the cluster directory.
+        """
+        loader = PyFileConfigLoader(filename, self.location)
+        return loader.load_config()
+
+    def copy_config_file(self, config_file, path=None, overwrite=False):
+        """Copy a default config file into the active cluster directory.
+
+        Default configuration files are kept in :mod:`IPython.config.default`.
+        This function moves these from that location to the working cluster
+        directory.
+        """
+        if path is None:
+            import IPython.config.default
+            path = IPython.config.default.__file__.split(os.path.sep)[:-1]
+            path = os.path.sep.join(path)
+        src = os.path.join(path, config_file)
+        dst = os.path.join(self.location, config_file)
+        if not os.path.isfile(dst) or overwrite:
+            shutil.copy(src, dst)
+
+    def copy_all_config_files(self, path=None, overwrite=False):
+        """Copy all config files into the active cluster directory."""
+        for f in [u'ipcontroller_config.py', u'ipengine_config.py',
+                  u'ipcluster_config.py']:
+            self.copy_config_file(f, path=path, overwrite=overwrite)
+
+    @classmethod
+    def create_cluster_dir(csl, cluster_dir):
+        """Create a new cluster directory given a full path.
+
+        Parameters
+        ----------
+        cluster_dir : str
+            The full path to the cluster directory.  If it does exist, it will
+            be used.  If not, it will be created.
+        """
+        return ClusterDir(location=cluster_dir)
+
+    @classmethod
+    def create_cluster_dir_by_profile(cls, path, profile=u'default'):
+        """Create a cluster dir by profile name and path.
+
+        Parameters
+        ----------
+        path : str
+            The path (directory) to put the cluster directory in.
+        profile : str
+            The name of the profile.  The name of the cluster directory will
+            be "cluster_<profile>".
+        """
+        if not os.path.isdir(path):
+            raise ClusterDirError('Directory not found: %s' % path)
+        cluster_dir = os.path.join(path, u'cluster_' + profile)
+        return ClusterDir(location=cluster_dir)
+
+    @classmethod
+    def find_cluster_dir_by_profile(cls, ipython_dir, profile=u'default'):
+        """Find an existing cluster dir by profile name, return its ClusterDir.
+
+        This searches through a sequence of paths for a cluster dir.  If it
+        is not found, a :class:`ClusterDirError` exception will be raised.
+
+        The search path algorithm is:
+        1. ``os.getcwd()``
+        2. ``ipython_dir``
+        3. The directories found in the ":" separated 
+           :env:`IPCLUSTER_DIR_PATH` environment variable.
+
+        Parameters
+        ----------
+        ipython_dir : unicode or str
+            The IPython directory to use.
+        profile : unicode or str
+            The name of the profile.  The name of the cluster directory
+            will be "cluster_<profile>".
+        """
+        dirname = u'cluster_' + profile
+        cluster_dir_paths = os.environ.get('IPCLUSTER_DIR_PATH','')
+        if cluster_dir_paths:
+            cluster_dir_paths = cluster_dir_paths.split(':')
+        else:
+            cluster_dir_paths = []
+        paths = [os.getcwd(), ipython_dir] + cluster_dir_paths
+        for p in paths:
+            cluster_dir = os.path.join(p, dirname)
+            if os.path.isdir(cluster_dir):
+                return ClusterDir(location=cluster_dir)
+        else:
+            raise ClusterDirError('Cluster directory not found in paths: %s' % dirname)
+
+    @classmethod
+    def find_cluster_dir(cls, cluster_dir):
+        """Find/create a cluster dir and return its ClusterDir.
+
+        This will create the cluster directory if it doesn't exist.
+
+        Parameters
+        ----------
+        cluster_dir : unicode or str
+            The path of the cluster directory.  This is expanded using
+            :func:`IPython.utils.genutils.expand_path`.
+        """
+        cluster_dir = expand_path(cluster_dir)
+        if not os.path.isdir(cluster_dir):
+            raise ClusterDirError('Cluster directory not found: %s' % cluster_dir)
+        return ClusterDir(location=cluster_dir)
+
+
+#-----------------------------------------------------------------------------
+# Command line options
+#-----------------------------------------------------------------------------
+
+class ClusterDirConfigLoader(BaseAppConfigLoader):
+
+    def _add_cluster_profile(self, parser):
+        paa = parser.add_argument
+        paa('-p', '--profile',
+            dest='Global.profile',type=unicode,
+            help=
+            """The string name of the profile to be used. This determines the name
+            of the cluster dir as: cluster_<profile>. The default profile is named
+            'default'.  The cluster directory is resolve this way if the
+            --cluster-dir option is not used.""",
+            metavar='Global.profile')
+
+    def _add_cluster_dir(self, parser):
+        paa = parser.add_argument
+        paa('--cluster-dir',
+            dest='Global.cluster_dir',type=unicode,
+            help="""Set the cluster dir. This overrides the logic used by the
+            --profile option.""",
+            metavar='Global.cluster_dir')
+
+    def _add_work_dir(self, parser):
+        paa = parser.add_argument
+        paa('--work-dir',
+            dest='Global.work_dir',type=unicode,
+            help='Set the working dir for the process.',
+            metavar='Global.work_dir')
+
+    def _add_clean_logs(self, parser):
+        paa = parser.add_argument
+        paa('--clean-logs',
+            dest='Global.clean_logs', action='store_true',
+            help='Delete old log flies before starting.')
+
+    def _add_no_clean_logs(self, parser):
+        paa = parser.add_argument
+        paa('--no-clean-logs',
+            dest='Global.clean_logs', action='store_false',
+            help="Don't Delete old log flies before starting.")
+
+    def _add_arguments(self):
+        super(ClusterDirConfigLoader, self)._add_arguments()
+        self._add_cluster_profile(self.parser)
+        self._add_cluster_dir(self.parser)
+        self._add_work_dir(self.parser)
+        self._add_clean_logs(self.parser)
+        self._add_no_clean_logs(self.parser)
+
+
+#-----------------------------------------------------------------------------
+# Crash handler for this application
+#-----------------------------------------------------------------------------
+
+
+_message_template = """\
+Oops, $self.app_name crashed. We do our best to make it stable, but...
+
+A crash report was automatically generated with the following information:
+  - A verbatim copy of the crash traceback.
+  - Data on your current $self.app_name configuration.
+
+It was left in the file named:
+\t'$self.crash_report_fname'
+If you can email this file to the developers, the information in it will help
+them in understanding and correcting the problem.
+
+You can mail it to: $self.contact_name at $self.contact_email
+with the subject '$self.app_name Crash Report'.
+
+If you want to do it now, the following command will work (under Unix):
+mail -s '$self.app_name Crash Report' $self.contact_email < $self.crash_report_fname
+
+To ensure accurate tracking of this issue, please file a report about it at:
+$self.bug_tracker
+"""
+
+class ClusterDirCrashHandler(CrashHandler):
+    """sys.excepthook for IPython itself, leaves a detailed report on disk."""
+
+    message_template = _message_template
+
+    def __init__(self, app):
+        contact_name = release.authors['Brian'][0]
+        contact_email = release.authors['Brian'][1]
+        bug_tracker = 'http://github.com/ipython/ipython/issues'
+        super(ClusterDirCrashHandler,self).__init__(
+            app, contact_name, contact_email, bug_tracker
+        )
+
+
+#-----------------------------------------------------------------------------
+# Main application
+#-----------------------------------------------------------------------------
+
+class ApplicationWithClusterDir(Application):
+    """An application that puts everything into a cluster directory.
+
+    Instead of looking for things in the ipython_dir, this type of application
+    will use its own private directory called the "cluster directory"
+    for things like config files, log files, etc.
+
+    The cluster directory is resolved as follows:
+
+    * If the ``--cluster-dir`` option is given, it is used.
+    * If ``--cluster-dir`` is not given, the application directory is 
+      resolve using the profile name as ``cluster_<profile>``. The search 
+      path for this directory is then i) cwd if it is found there
+      and ii) in ipython_dir otherwise.
+
+    The config file for the application is to be put in the cluster
+    dir and named the value of the ``config_file_name`` class attribute.
+    """
+
+    command_line_loader = ClusterDirConfigLoader
+    crash_handler_class = ClusterDirCrashHandler
+    auto_create_cluster_dir = True
+    # temporarily override default_log_level to INFO
+    default_log_level = logging.INFO
+
+    def create_default_config(self):
+        super(ApplicationWithClusterDir, self).create_default_config()
+        self.default_config.Global.profile = u'default'
+        self.default_config.Global.cluster_dir = u''
+        self.default_config.Global.work_dir = os.getcwd()
+        self.default_config.Global.log_to_file = False
+        self.default_config.Global.log_url = None
+        self.default_config.Global.clean_logs = False
+
+    def find_resources(self):
+        """This resolves the cluster directory.
+
+        This tries to find the cluster directory and if successful, it will
+        have done:
+        * Sets ``self.cluster_dir_obj`` to the :class:`ClusterDir` object for 
+          the application.
+        * Sets ``self.cluster_dir`` attribute of the application and config
+          objects.
+
+        The algorithm used for this is as follows:
+        1. Try ``Global.cluster_dir``.
+        2. Try using ``Global.profile``.
+        3. If both of these fail and ``self.auto_create_cluster_dir`` is
+           ``True``, then create the new cluster dir in the IPython directory.
+        4. If all fails, then raise :class:`ClusterDirError`.
+        """
+
+        try:
+            cluster_dir = self.command_line_config.Global.cluster_dir
+        except AttributeError:
+            cluster_dir = self.default_config.Global.cluster_dir
+        cluster_dir = expand_path(cluster_dir)
+        try:
+            self.cluster_dir_obj = ClusterDir.find_cluster_dir(cluster_dir)
+        except ClusterDirError:
+            pass
+        else:
+            self.log.info('Using existing cluster dir: %s' % \
+                self.cluster_dir_obj.location
+            )
+            self.finish_cluster_dir()
+            return
+
+        try:
+            self.profile = self.command_line_config.Global.profile
+        except AttributeError:
+            self.profile = self.default_config.Global.profile
+        try:
+            self.cluster_dir_obj = ClusterDir.find_cluster_dir_by_profile(
+                self.ipython_dir, self.profile)
+        except ClusterDirError:
+            pass
+        else:
+            self.log.info('Using existing cluster dir: %s' % \
+                self.cluster_dir_obj.location
+            )
+            self.finish_cluster_dir()
+            return
+
+        if self.auto_create_cluster_dir:
+            self.cluster_dir_obj = ClusterDir.create_cluster_dir_by_profile(
+                self.ipython_dir, self.profile
+            )
+            self.log.info('Creating new cluster dir: %s' % \
+                self.cluster_dir_obj.location
+            )
+            self.finish_cluster_dir()
+        else:
+            raise ClusterDirError('Could not find a valid cluster directory.')
+
+    def finish_cluster_dir(self):
+        # Set the cluster directory
+        self.cluster_dir = self.cluster_dir_obj.location
+        
+        # These have to be set because they could be different from the one
+        # that we just computed.  Because command line has the highest
+        # priority, this will always end up in the master_config.
+        self.default_config.Global.cluster_dir = self.cluster_dir
+        self.command_line_config.Global.cluster_dir = self.cluster_dir
+
+    def find_config_file_name(self):
+        """Find the config file name for this application."""
+        # For this type of Application it should be set as a class attribute.
+        if not hasattr(self, 'default_config_file_name'):
+            self.log.critical("No config filename found")
+        else:
+            self.config_file_name = self.default_config_file_name
+
+    def find_config_file_paths(self):
+        # Set the search path to to the cluster directory. We should NOT
+        # include IPython.config.default here as the default config files
+        # are ALWAYS automatically moved to the cluster directory.
+        conf_dir = os.path.join(get_ipython_package_dir(), 'config', 'default')
+        self.config_file_paths = (self.cluster_dir,)
+
+    def pre_construct(self):
+        # The log and security dirs were set earlier, but here we put them
+        # into the config and log them.
+        config = self.master_config
+        sdir = self.cluster_dir_obj.security_dir
+        self.security_dir = config.Global.security_dir = sdir
+        ldir = self.cluster_dir_obj.log_dir
+        self.log_dir = config.Global.log_dir = ldir
+        pdir = self.cluster_dir_obj.pid_dir
+        self.pid_dir = config.Global.pid_dir = pdir
+        self.log.info("Cluster directory set to: %s" % self.cluster_dir)
+        config.Global.work_dir = unicode(expand_path(config.Global.work_dir))
+        # Change to the working directory. We do this just before construct
+        # is called so all the components there have the right working dir.
+        self.to_work_dir()
+
+    def to_work_dir(self):
+        wd = self.master_config.Global.work_dir
+        if unicode(wd) != unicode(os.getcwd()):
+            os.chdir(wd)
+            self.log.info("Changing to working dir: %s" % wd)
+
+    def start_logging(self):
+        # Remove old log files
+        if self.master_config.Global.clean_logs:
+            log_dir = self.master_config.Global.log_dir
+            for f in os.listdir(log_dir):
+                if re.match(r'%s-\d+\.(log|err|out)'%self.name,f):
+                # if f.startswith(self.name + u'-') and f.endswith('.log'):
+                    os.remove(os.path.join(log_dir, f))
+        # Start logging to the new log file
+        if self.master_config.Global.log_to_file:
+            log_filename = self.name + u'-' + str(os.getpid()) + u'.log'
+            logfile = os.path.join(self.log_dir, log_filename)
+            open_log_file = open(logfile, 'w')
+        elif self.master_config.Global.log_url:
+            open_log_file = None
+        else:
+            open_log_file = sys.stdout
+        if open_log_file is not None:
+            self.log.removeHandler(self._log_handler)
+            self._log_handler = logging.StreamHandler(open_log_file)
+            self._log_formatter = logging.Formatter("[%(name)s] %(message)s")
+            self._log_handler.setFormatter(self._log_formatter)
+            self.log.addHandler(self._log_handler)
+        # log.startLogging(open_log_file)
+
+    def write_pid_file(self, overwrite=False):
+        """Create a .pid file in the pid_dir with my pid.
+
+        This must be called after pre_construct, which sets `self.pid_dir`.
+        This raises :exc:`PIDFileError` if the pid file exists already.
+        """
+        pid_file = os.path.join(self.pid_dir, self.name + u'.pid')
+        if os.path.isfile(pid_file):
+            pid = self.get_pid_from_file()
+            if not overwrite:
+                raise PIDFileError(
+                    'The pid file [%s] already exists. \nThis could mean that this '
+                    'server is already running with [pid=%s].' % (pid_file, pid)
+                )
+        with open(pid_file, 'w') as f:
+            self.log.info("Creating pid file: %s" % pid_file)
+            f.write(repr(os.getpid())+'\n')
+
+    def remove_pid_file(self):
+        """Remove the pid file.
+
+        This should be called at shutdown by registering a callback with
+        :func:`reactor.addSystemEventTrigger`. This needs to return
+        ``None``.
+        """
+        pid_file = os.path.join(self.pid_dir, self.name + u'.pid')
+        if os.path.isfile(pid_file):
+            try:
+                self.log.info("Removing pid file: %s" % pid_file)
+                os.remove(pid_file)
+            except:
+                self.log.warn("Error removing the pid file: %s" % pid_file)
+
+    def get_pid_from_file(self):
+        """Get the pid from the pid file.
+
+        If the  pid file doesn't exist a :exc:`PIDFileError` is raised.
+        """
+        pid_file = os.path.join(self.pid_dir, self.name + u'.pid')
+        if os.path.isfile(pid_file):
+            with open(pid_file, 'r') as f:
+                pid = int(f.read().strip())
+                return pid
+        else:
+            raise PIDFileError('pid file not found: %s' % pid_file)
+

IPython/parallel/apps/ipclusterapp.py

@@ -0,0 +1,592 b''
+#!/usr/bin/env python
+# encoding: utf-8
+"""
+The ipcluster application.
+"""
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2009  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+import errno
+import logging
+import os
+import re
+import signal
+
+import zmq
+from zmq.eventloop import ioloop
+
+from IPython.external.argparse import ArgumentParser, SUPPRESS
+from IPython.utils.importstring import import_item
+from .clusterdir import (
+    ApplicationWithClusterDir, ClusterDirConfigLoader,
+    ClusterDirError, PIDFileError
+)
+
+
+#-----------------------------------------------------------------------------
+# Module level variables
+#-----------------------------------------------------------------------------
+
+
+default_config_file_name = u'ipcluster_config.py'
+
+
+_description = """\
+Start an IPython cluster for parallel computing.\n\n
+
+An IPython cluster consists of 1 controller and 1 or more engines.
+This command automates the startup of these processes using a wide
+range of startup methods (SSH, local processes, PBS, mpiexec,
+Windows HPC Server 2008). To start a cluster with 4 engines on your
+local host simply do 'ipcluster start -n 4'. For more complex usage 
+you will typically do 'ipcluster create -p mycluster', then edit
+configuration files, followed by 'ipcluster start -p mycluster -n 4'.
+"""
+
+
+# Exit codes for ipcluster
+
+# This will be the exit code if the ipcluster appears to be running because
+# a .pid file exists
+ALREADY_STARTED = 10
+
+
+# This will be the exit code if ipcluster stop is run, but there is not .pid
+# file to be found.
+ALREADY_STOPPED = 11
+
+# This will be the exit code if ipcluster engines is run, but there is not .pid
+# file to be found.
+NO_CLUSTER = 12
+
+
+#-----------------------------------------------------------------------------
+# Command line options
+#-----------------------------------------------------------------------------
+
+
+class IPClusterAppConfigLoader(ClusterDirConfigLoader):
+
+    def _add_arguments(self):
+        # Don't call ClusterDirConfigLoader._add_arguments as we don't want
+        # its defaults on self.parser. Instead, we will put those on
+        # default options on our subparsers.
+        
+        # This has all the common options that all subcommands use
+        parent_parser1 = ArgumentParser(
+            add_help=False,
+            argument_default=SUPPRESS
+        )
+        self._add_ipython_dir(parent_parser1)
+        self._add_log_level(parent_parser1)
+
+        # This has all the common options that other subcommands use
+        parent_parser2 = ArgumentParser(
+            add_help=False,
+            argument_default=SUPPRESS
+        )
+        self._add_cluster_profile(parent_parser2)
+        self._add_cluster_dir(parent_parser2)
+        self._add_work_dir(parent_parser2)
+        paa = parent_parser2.add_argument
+        paa('--log-to-file',
+            action='store_true', dest='Global.log_to_file', 
+            help='Log to a file in the log directory (default is stdout)')
+
+        # Create the object used to create the subparsers.
+        subparsers = self.parser.add_subparsers(
+            dest='Global.subcommand',
+            title='ipcluster subcommands',
+            description=
+            """ipcluster has a variety of subcommands. The general way of 
+            running ipcluster is 'ipcluster <cmd> [options]'. To get help
+            on a particular subcommand do 'ipcluster <cmd> -h'."""
+            # help="For more help, type 'ipcluster <cmd> -h'",
+        )
+
+        # The "list" subcommand parser
+        parser_list = subparsers.add_parser(
+            'list',
+            parents=[parent_parser1],
+            argument_default=SUPPRESS,
+            help="List all clusters in cwd and ipython_dir.",
+            description=
+            """List all available clusters, by cluster directory, that can
+            be found in the current working directly or in the ipython
+            directory. Cluster directories are named using the convention
+            'cluster_<profile>'."""
+        )
+
+        # The "create" subcommand parser
+        parser_create = subparsers.add_parser(
+            'create',
+            parents=[parent_parser1, parent_parser2],
+            argument_default=SUPPRESS,
+            help="Create a new cluster directory.",
+            description=
+            """Create an ipython cluster directory by its profile name or 
+            cluster directory path. Cluster directories contain 
+            configuration, log and security related files and are named 
+            using the convention 'cluster_<profile>'. By default they are
+            located in your ipython directory. Once created, you will
+            probably need to edit the configuration files in the cluster
+            directory to configure your cluster. Most users will create a
+            cluster directory by profile name, 
+            'ipcluster create -p mycluster', which will put the directory
+            in '<ipython_dir>/cluster_mycluster'. 
+            """
+        )
+        paa = parser_create.add_argument
+        paa('--reset-config',
+            dest='Global.reset_config', action='store_true',
+            help=
+            """Recopy the default config files to the cluster directory.
+            You will loose any modifications you have made to these files.""")
+
+        # The "start" subcommand parser
+        parser_start = subparsers.add_parser(
+            'start',
+            parents=[parent_parser1, parent_parser2],
+            argument_default=SUPPRESS,
+            help="Start a cluster.",
+            description=
+            """Start an ipython cluster by its profile name or cluster 
+            directory. Cluster directories contain configuration, log and
+            security related files and are named using the convention
+            'cluster_<profile>' and should be creating using the 'start'
+            subcommand of 'ipcluster'. If your cluster directory is in 
+            the cwd or the ipython directory, you can simply refer to it
+            using its profile name, 'ipcluster start -n 4 -p <profile>`,
+            otherwise use the '--cluster-dir' option.
+            """
+        )
+        
+        paa = parser_start.add_argument
+        paa('-n', '--number',
+            type=int, dest='Global.n',
+            help='The number of engines to start.',
+            metavar='Global.n')
+        paa('--clean-logs',
+            dest='Global.clean_logs', action='store_true',
+            help='Delete old log flies before starting.')
+        paa('--no-clean-logs',
+            dest='Global.clean_logs', action='store_false',
+            help="Don't delete old log flies before starting.")
+        paa('--daemon',
+            dest='Global.daemonize', action='store_true',
+            help='Daemonize the ipcluster program. This implies --log-to-file')
+        paa('--no-daemon',
+            dest='Global.daemonize', action='store_false',
+            help="Dont't daemonize the ipcluster program.")
+        paa('--delay',
+            type=float, dest='Global.delay',
+            help="Specify the delay (in seconds) between starting the controller and starting the engine(s).")
+
+        # The "stop" subcommand parser
+        parser_stop = subparsers.add_parser(
+            'stop',
+            parents=[parent_parser1, parent_parser2],
+            argument_default=SUPPRESS,
+            help="Stop a running cluster.",
+            description=
+            """Stop a running ipython cluster by its profile name or cluster 
+            directory. Cluster directories are named using the convention
+            'cluster_<profile>'. If your cluster directory is in 
+            the cwd or the ipython directory, you can simply refer to it
+            using its profile name, 'ipcluster stop -p <profile>`, otherwise
+            use the '--cluster-dir' option.
+            """
+        )
+        paa = parser_stop.add_argument
+        paa('--signal',
+            dest='Global.signal', type=int,
+            help="The signal number to use in stopping the cluster (default=2).",
+            metavar="Global.signal")
+        
+        # the "engines" subcommand parser
+        parser_engines = subparsers.add_parser(
+            'engines',
+            parents=[parent_parser1, parent_parser2],
+            argument_default=SUPPRESS,
+            help="Attach some engines to an existing controller or cluster.",
+            description=
+            """Start one or more engines to connect to an existing Cluster
+            by profile name or cluster directory.
+            Cluster directories contain configuration, log and
+            security related files and are named using the convention
+            'cluster_<profile>' and should be creating using the 'start'
+            subcommand of 'ipcluster'. If your cluster directory is in 
+            the cwd or the ipython directory, you can simply refer to it
+            using its profile name, 'ipcluster engines -n 4 -p <profile>`,
+            otherwise use the '--cluster-dir' option.
+            """
+        )
+        paa = parser_engines.add_argument
+        paa('-n', '--number',
+            type=int, dest='Global.n',
+            help='The number of engines to start.',
+            metavar='Global.n')
+        paa('--daemon',
+            dest='Global.daemonize', action='store_true',
+            help='Daemonize the ipcluster program. This implies --log-to-file')
+        paa('--no-daemon',
+            dest='Global.daemonize', action='store_false',
+            help="Dont't daemonize the ipcluster program.")
+
+#-----------------------------------------------------------------------------
+# Main application
+#-----------------------------------------------------------------------------
+
+
+class IPClusterApp(ApplicationWithClusterDir):
+
+    name = u'ipcluster'
+    description = _description
+    usage = None
+    command_line_loader = IPClusterAppConfigLoader
+    default_config_file_name = default_config_file_name
+    default_log_level = logging.INFO
+    auto_create_cluster_dir = False
+
+    def create_default_config(self):
+        super(IPClusterApp, self).create_default_config()
+        self.default_config.Global.controller_launcher = \
+            'IPython.parallel.launcher.LocalControllerLauncher'
+        self.default_config.Global.engine_launcher = \
+            'IPython.parallel.launcher.LocalEngineSetLauncher'
+        self.default_config.Global.n = 2
+        self.default_config.Global.delay = 2
+        self.default_config.Global.reset_config = False
+        self.default_config.Global.clean_logs = True
+        self.default_config.Global.signal = signal.SIGINT
+        self.default_config.Global.daemonize = False
+
+    def find_resources(self):
+        subcommand = self.command_line_config.Global.subcommand
+        if subcommand=='list':
+            self.list_cluster_dirs()
+            # Exit immediately because there is nothing left to do.
+            self.exit()
+        elif subcommand=='create':
+            self.auto_create_cluster_dir = True
+            super(IPClusterApp, self).find_resources()
+        elif subcommand=='start' or subcommand=='stop':
+            self.auto_create_cluster_dir = True
+            try:
+                super(IPClusterApp, self).find_resources()
+            except ClusterDirError:
+                raise ClusterDirError(
+                    "Could not find a cluster directory. A cluster dir must "
+                    "be created before running 'ipcluster start'.  Do "
+                    "'ipcluster create -h' or 'ipcluster list -h' for more "
+                    "information about creating and listing cluster dirs."
+                )
+        elif subcommand=='engines':
+            self.auto_create_cluster_dir = False
+            try:
+                super(IPClusterApp, self).find_resources()
+            except ClusterDirError:
+                raise ClusterDirError(
+                    "Could not find a cluster directory. A cluster dir must "
+                    "be created before running 'ipcluster start'.  Do "
+                    "'ipcluster create -h' or 'ipcluster list -h' for more "
+                    "information about creating and listing cluster dirs."
+                )
+
+    def list_cluster_dirs(self):
+        # Find the search paths
+        cluster_dir_paths = os.environ.get('IPCLUSTER_DIR_PATH','')
+        if cluster_dir_paths:
+            cluster_dir_paths = cluster_dir_paths.split(':')
+        else:
+            cluster_dir_paths = []
+        try:
+            ipython_dir = self.command_line_config.Global.ipython_dir
+        except AttributeError:
+            ipython_dir = self.default_config.Global.ipython_dir
+        paths = [os.getcwd(), ipython_dir] + \
+            cluster_dir_paths
+        paths = list(set(paths))
+
+        self.log.info('Searching for cluster dirs in paths: %r' % paths)
+        for path in paths:
+            files = os.listdir(path)
+            for f in files:
+                full_path = os.path.join(path, f)
+                if os.path.isdir(full_path) and f.startswith('cluster_'):
+                    profile = full_path.split('_')[-1]
+                    start_cmd = 'ipcluster start -p %s -n 4' % profile
+                    print start_cmd + " ==> " + full_path
+
+    def pre_construct(self):
+        # IPClusterApp.pre_construct() is where we cd to the working directory.
+        super(IPClusterApp, self).pre_construct()
+        config = self.master_config
+        try:
+            daemon = config.Global.daemonize
+            if daemon:
+                config.Global.log_to_file = True
+        except AttributeError:
+            pass
+
+    def construct(self):
+        config = self.master_config
+        subcmd = config.Global.subcommand
+        reset = config.Global.reset_config
+        if subcmd == 'list':
+            return
+        if subcmd == 'create':
+            self.log.info('Copying default config files to cluster directory '
+            '[overwrite=%r]' % (reset,))
+            self.cluster_dir_obj.copy_all_config_files(overwrite=reset)
+        if subcmd =='start':
+            self.cluster_dir_obj.copy_all_config_files(overwrite=False)
+            self.start_logging()
+            self.loop = ioloop.IOLoop.instance()
+            # reactor.callWhenRunning(self.start_launchers)
+            dc = ioloop.DelayedCallback(self.start_launchers, 0, self.loop)
+            dc.start()
+        if subcmd == 'engines':
+            self.start_logging()
+            self.loop = ioloop.IOLoop.instance()
+            # reactor.callWhenRunning(self.start_launchers)
+            engine_only = lambda : self.start_launchers(controller=False)
+            dc = ioloop.DelayedCallback(engine_only, 0, self.loop)
+            dc.start()
+
+    def start_launchers(self, controller=True):
+        config = self.master_config
+
+        # Create the launchers. In both bases, we set the work_dir of
+        # the launcher to the cluster_dir. This is where the launcher's
+        # subprocesses will be launched. It is not where the controller
+        # and engine will be launched.
+        if controller:
+            cl_class = import_item(config.Global.controller_launcher)
+            self.controller_launcher = cl_class(
+                work_dir=self.cluster_dir, config=config,
+                logname=self.log.name
+            )
+            # Setup the observing of stopping. If the controller dies, shut
+            # everything down as that will be completely fatal for the engines.
+            self.controller_launcher.on_stop(self.stop_launchers)
+            # But, we don't monitor the stopping of engines. An engine dying
+            # is just fine and in principle a user could start a new engine.
+            # Also, if we did monitor engine stopping, it is difficult to
+            # know what to do when only some engines die. Currently, the
+            # observing of engine stopping is inconsistent. Some launchers
+            # might trigger on a single engine stopping, other wait until
+            # all stop.  TODO: think more about how to handle this.
+        else:
+            self.controller_launcher = None
+        
+        el_class = import_item(config.Global.engine_launcher)
+        self.engine_launcher = el_class(
+            work_dir=self.cluster_dir, config=config, logname=self.log.name
+        )
+
+        # Setup signals
+        signal.signal(signal.SIGINT, self.sigint_handler)
+
+        # Start the controller and engines
+        self._stopping = False  # Make sure stop_launchers is not called 2x.
+        if controller:
+            self.start_controller()
+        dc = ioloop.DelayedCallback(self.start_engines, 1000*config.Global.delay*controller, self.loop)
+        dc.start()
+        self.startup_message()
+
+    def startup_message(self, r=None):
+        self.log.info("IPython cluster: started")
+        return r
+        
+    def start_controller(self, r=None):
+        # self.log.info("In start_controller")
+        config = self.master_config
+        d = self.controller_launcher.start(
+            cluster_dir=config.Global.cluster_dir
+        )
+        return d
+            
+    def start_engines(self, r=None):
+        # self.log.info("In start_engines")
+        config = self.master_config
+        
+        d = self.engine_launcher.start(
+            config.Global.n,
+            cluster_dir=config.Global.cluster_dir
+        )
+        return d
+
+    def stop_controller(self, r=None):
+        # self.log.info("In stop_controller")
+        if self.controller_launcher and self.controller_launcher.running:
+            return self.controller_launcher.stop()
+
+    def stop_engines(self, r=None):
+        # self.log.info("In stop_engines")
+        if self.engine_launcher.running:
+            d = self.engine_launcher.stop()
+            # d.addErrback(self.log_err)
+            return d
+        else:
+            return None
+
+    def log_err(self, f):
+        self.log.error(f.getTraceback())
+        return None
+        
+    def stop_launchers(self, r=None):
+        if not self._stopping:
+            self._stopping = True
+            # if isinstance(r, failure.Failure):
+            #     self.log.error('Unexpected error in ipcluster:')
+            #     self.log.info(r.getTraceback())
+            self.log.error("IPython cluster: stopping")
+            # These return deferreds. We are not doing anything with them
+            # but we are holding refs to them as a reminder that they 
+            # do return deferreds.
+            d1 = self.stop_engines()
+            d2 = self.stop_controller()
+            # Wait a few seconds to let things shut down.
+            dc = ioloop.DelayedCallback(self.loop.stop, 4000, self.loop)
+            dc.start()
+            # reactor.callLater(4.0, reactor.stop)
+
+    def sigint_handler(self, signum, frame):
+        self.stop_launchers()
+        
+    def start_logging(self):
+        # Remove old log files of the controller and engine
+        if self.master_config.Global.clean_logs:
+            log_dir = self.master_config.Global.log_dir
+            for f in os.listdir(log_dir):
+                if re.match(r'ip(engine|controller)z-\d+\.(log|err|out)',f):
+                    os.remove(os.path.join(log_dir, f))
+        # This will remove old log files for ipcluster itself
+        super(IPClusterApp, self).start_logging()
+
+    def start_app(self):
+        """Start the application, depending on what subcommand is used."""
+        subcmd = self.master_config.Global.subcommand
+        if subcmd=='create' or subcmd=='list':
+            return
+        elif subcmd=='start':
+            self.start_app_start()
+        elif subcmd=='stop':
+            self.start_app_stop()
+        elif subcmd=='engines':
+            self.start_app_engines()
+
+    def start_app_start(self):
+        """Start the app for the start subcommand."""
+        config = self.master_config
+        # First see if the cluster is already running
+        try:
+            pid = self.get_pid_from_file()
+        except PIDFileError:
+            pass
+        else:
+            self.log.critical(
+                'Cluster is already running with [pid=%s]. '
+                'use "ipcluster stop" to stop the cluster.' % pid
+            )
+            # Here I exit with a unusual exit status that other processes
+            # can watch for to learn how I existed.
+            self.exit(ALREADY_STARTED)
+
+        # Now log and daemonize
+        self.log.info(
+            'Starting ipcluster with [daemon=%r]' % config.Global.daemonize
+        )
+        # TODO: Get daemonize working on Windows or as a Windows Server.
+        if config.Global.daemonize:
+            if os.name=='posix':
+                from twisted.scripts._twistd_unix import daemonize
+                daemonize()
+
+        # Now write the new pid file AFTER our new forked pid is active.
+        self.write_pid_file()
+        try:
+            self.loop.start()
+        except KeyboardInterrupt:
+            pass
+        except zmq.ZMQError as e:
+            if e.errno == errno.EINTR:
+                pass
+            else:
+                raise
+        self.remove_pid_file()
+
+    def start_app_engines(self):
+        """Start the app for the start subcommand."""
+        config = self.master_config
+        # First see if the cluster is already running
+        
+        # Now log and daemonize
+        self.log.info(
+            'Starting engines with [daemon=%r]' % config.Global.daemonize
+        )
+        # TODO: Get daemonize working on Windows or as a Windows Server.
+        if config.Global.daemonize:
+            if os.name=='posix':
+                from twisted.scripts._twistd_unix import daemonize
+                daemonize()
+
+        # Now write the new pid file AFTER our new forked pid is active.
+        # self.write_pid_file()
+        try:
+            self.loop.start()
+        except KeyboardInterrupt:
+            pass
+        except zmq.ZMQError as e:
+            if e.errno == errno.EINTR:
+                pass
+            else:
+                raise
+        # self.remove_pid_file()
+    
+    def start_app_stop(self):
+        """Start the app for the stop subcommand."""
+        config = self.master_config
+        try:
+            pid = self.get_pid_from_file()
+        except PIDFileError:
+            self.log.critical(
+                'Problem reading pid file, cluster is probably not running.'
+            )
+            # Here I exit with a unusual exit status that other processes
+            # can watch for to learn how I existed.
+            self.exit(ALREADY_STOPPED)
+        else:
+            if os.name=='posix':
+                sig = config.Global.signal
+                self.log.info(
+                    "Stopping cluster [pid=%r] with [signal=%r]" % (pid, sig)
+                )
+                os.kill(pid, sig)
+            elif os.name=='nt':
+                # As of right now, we don't support daemonize on Windows, so
+                # stop will not do anything. Minimally, it should clean up the
+                # old .pid files.
+                self.remove_pid_file()
+
+
+def launch_new_instance():
+    """Create and run the IPython cluster."""
+    app = IPClusterApp()
+    app.start()
+
+
+if __name__ == '__main__':
+    launch_new_instance()
+

IPython/parallel/apps/ipcontrollerapp.py

@@ -0,0 +1,432 b''
+#!/usr/bin/env python
+# encoding: utf-8
+"""
+The IPython controller application.
+"""
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2009  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+from __future__ import with_statement
+
+import copy
+import os
+import logging
+import socket
+import stat
+import sys
+import uuid
+
+import zmq
+from zmq.log.handlers import PUBHandler
+from zmq.utils import jsonapi as json
+
+from IPython.config.loader import Config
+
+from IPython.parallel import factory
+from .clusterdir import (
+    ApplicationWithClusterDir,
+    ClusterDirConfigLoader
+)
+from IPython.parallel.util import disambiguate_ip_address, split_url
+# from IPython.kernel.fcutil import FCServiceFactory, FURLError
+from IPython.utils.traitlets import Instance, Unicode
+
+from IPython.parallel.controller.controller import ControllerFactory
+
+
+#-----------------------------------------------------------------------------
+# Module level variables
+#-----------------------------------------------------------------------------
+
+
+#: The default config file name for this application
+default_config_file_name = u'ipcontroller_config.py'
+
+
+_description = """Start the IPython controller for parallel computing.
+
+The IPython controller provides a gateway between the IPython engines and
+clients. The controller needs to be started before the engines and can be
+configured using command line options or using a cluster directory. Cluster
+directories contain config, log and security files and are usually located in
+your ipython directory and named as "cluster_<profile>". See the --profile
+and --cluster-dir options for details.
+"""
+
+#-----------------------------------------------------------------------------
+# Default interfaces
+#-----------------------------------------------------------------------------
+
+# The default client interfaces for FCClientServiceFactory.interfaces
+default_client_interfaces = Config()
+default_client_interfaces.Default.url_file = 'ipcontroller-client.url'
+
+# Make this a dict we can pass to Config.__init__ for the default
+default_client_interfaces = dict(copy.deepcopy(default_client_interfaces.items()))
+
+
+
+# The default engine interfaces for FCEngineServiceFactory.interfaces
+default_engine_interfaces = Config()
+default_engine_interfaces.Default.url_file = u'ipcontroller-engine.url'
+
+# Make this a dict we can pass to Config.__init__ for the default
+default_engine_interfaces = dict(copy.deepcopy(default_engine_interfaces.items()))
+
+
+#-----------------------------------------------------------------------------
+# Service factories
+#-----------------------------------------------------------------------------
+
+# 
+# class FCClientServiceFactory(FCServiceFactory):
+#     """A Foolscap implementation of the client services."""
+# 
+#     cert_file = Unicode(u'ipcontroller-client.pem', config=True)
+#     interfaces = Instance(klass=Config, kw=default_client_interfaces,
+#                           allow_none=False, config=True)
+# 
+# 
+# class FCEngineServiceFactory(FCServiceFactory):
+#     """A Foolscap implementation of the engine services."""
+# 
+#     cert_file = Unicode(u'ipcontroller-engine.pem', config=True)
+#     interfaces = Instance(klass=dict, kw=default_engine_interfaces,
+#                           allow_none=False, config=True)
+# 
+
+#-----------------------------------------------------------------------------
+# Command line options
+#-----------------------------------------------------------------------------
+
+
+class IPControllerAppConfigLoader(ClusterDirConfigLoader):
+
+    def _add_arguments(self):
+        super(IPControllerAppConfigLoader, self)._add_arguments()
+        paa = self.parser.add_argument
+        
+        ## Hub Config:
+        paa('--mongodb', 
+            dest='HubFactory.db_class', action='store_const',
+            const='IPython.parallel.controller.mongodb.MongoDB', 
+            help='Use MongoDB for task storage [default: in-memory]')
+        paa('--sqlite', 
+            dest='HubFactory.db_class', action='store_const',
+            const='IPython.parallel.controller.sqlitedb.SQLiteDB',
+            help='Use SQLite3 for DB task storage [default: in-memory]')
+        paa('--hb',
+            type=int, dest='HubFactory.hb', nargs=2,
+            help='The (2) ports the Hub\'s Heartmonitor will use for the heartbeat '
+            'connections [default: random]',
+            metavar='Hub.hb_ports')
+        paa('--ping',
+            type=int, dest='HubFactory.ping',
+            help='The frequency at which the Hub pings the engines for heartbeats '
+            ' (in ms) [default: 100]',
+            metavar='Hub.ping')
+        
+        # Client config
+        paa('--client-ip',
+            type=str, dest='HubFactory.client_ip', 
+            help='The IP address or hostname the Hub will listen on for '
+            'client connections. Both engine-ip and client-ip can be set simultaneously '
+            'via --ip [default: loopback]',
+            metavar='Hub.client_ip')
+        paa('--client-transport',
+            type=str, dest='HubFactory.client_transport', 
+            help='The ZeroMQ transport the Hub will use for '
+            'client connections. Both engine-transport and client-transport can be set simultaneously '
+            'via --transport [default: tcp]',
+            metavar='Hub.client_transport')
+        paa('--query',
+            type=int, dest='HubFactory.query_port', 
+            help='The port on which the Hub XREP socket will listen for result queries from clients [default: random]',
+            metavar='Hub.query_port')
+        paa('--notifier',
+            type=int, dest='HubFactory.notifier_port', 
+            help='The port on which the Hub PUB socket will listen for notification connections [default: random]',
+            metavar='Hub.notifier_port')
+        
+        # Engine config
+        paa('--engine-ip',
+            type=str, dest='HubFactory.engine_ip', 
+            help='The IP address or hostname the Hub will listen on for '
+            'engine connections. This applies to the Hub and its schedulers'
+            'engine-ip and client-ip can be set simultaneously '
+            'via --ip [default: loopback]',
+            metavar='Hub.engine_ip')
+        paa('--engine-transport',
+            type=str, dest='HubFactory.engine_transport', 
+            help='The ZeroMQ transport the Hub will use for '
+            'client connections. Both engine-transport and client-transport can be set simultaneously '
+            'via --transport [default: tcp]',
+            metavar='Hub.engine_transport')
+        
+        # Scheduler config
+        paa('--mux',
+            type=int, dest='ControllerFactory.mux', nargs=2,
+            help='The (2) ports the MUX scheduler will listen on for client,engine '
+            'connections, respectively [default: random]',
+            metavar='Scheduler.mux_ports')
+        paa('--task',
+            type=int, dest='ControllerFactory.task', nargs=2,
+            help='The (2) ports the Task scheduler will listen on for client,engine '
+            'connections, respectively [default: random]',
+            metavar='Scheduler.task_ports')
+        paa('--control',
+            type=int, dest='ControllerFactory.control', nargs=2,
+            help='The (2) ports the Control scheduler will listen on for client,engine '
+            'connections, respectively [default: random]',
+            metavar='Scheduler.control_ports')
+        paa('--iopub',
+            type=int, dest='ControllerFactory.iopub', nargs=2,
+            help='The (2) ports the IOPub scheduler will listen on for client,engine '
+            'connections, respectively [default: random]',
+            metavar='Scheduler.iopub_ports')
+            
+        paa('--scheme',
+            type=str, dest='HubFactory.scheme',
+            choices = ['pure', 'lru', 'plainrandom', 'weighted', 'twobin','leastload'],
+            help='select the task scheduler scheme  [default: Python LRU]',
+            metavar='Scheduler.scheme')
+        paa('--usethreads',
+            dest='ControllerFactory.usethreads', action="store_true",
+            help='Use threads instead of processes for the schedulers',
+            )
+        paa('--hwm',
+            dest='ControllerFactory.hwm', type=int,
+            help='specify the High Water Mark (HWM) for the downstream '
+            'socket in the pure ZMQ scheduler. This is the maximum number '
+            'of allowed outstanding tasks on each engine.',
+            )
+        
+        ## Global config
+        paa('--log-to-file',
+            action='store_true', dest='Global.log_to_file',
+            help='Log to a file in the log directory (default is stdout)')
+        paa('--log-url',
+            type=str, dest='Global.log_url',
+            help='Broadcast logs to an iploggerz process [default: disabled]')
+        paa('-r','--reuse-files', 
+            action='store_true', dest='Global.reuse_files',
+            help='Try to reuse existing json connection files.')
+        paa('--no-secure',
+            action='store_false', dest='Global.secure',
+            help='Turn off execution keys (default).')
+        paa('--secure',
+            action='store_true', dest='Global.secure',
+            help='Turn on execution keys.')
+        paa('--execkey',
+            type=str, dest='Global.exec_key',
+            help='path to a file containing an execution key.',
+            metavar='keyfile')
+        paa('--ssh',
+            type=str, dest='Global.sshserver',
+            help='ssh url for clients to use when connecting to the Controller '
+            'processes. It should be of the form: [user@]server[:port]. The '
+            'Controller\'s listening addresses must be accessible from the ssh server',
+            metavar='Global.sshserver')
+        paa('--location',
+            type=str, dest='Global.location',
+            help="The external IP or domain name of this machine, used for disambiguating "
+            "engine and client connections.",
+            metavar='Global.location')
+        factory.add_session_arguments(self.parser)
+        factory.add_registration_arguments(self.parser)
+
+
+#-----------------------------------------------------------------------------
+# The main application
+#-----------------------------------------------------------------------------
+
+
+class IPControllerApp(ApplicationWithClusterDir):
+
+    name = u'ipcontroller'
+    description = _description
+    command_line_loader = IPControllerAppConfigLoader
+    default_config_file_name = default_config_file_name
+    auto_create_cluster_dir = True
+    
+
+    def create_default_config(self):
+        super(IPControllerApp, self).create_default_config()
+        # Don't set defaults for Global.secure or Global.reuse_furls
+        # as those are set in a component.
+        self.default_config.Global.import_statements = []
+        self.default_config.Global.clean_logs = True
+        self.default_config.Global.secure = True
+        self.default_config.Global.reuse_files = False
+        self.default_config.Global.exec_key = "exec_key.key"
+        self.default_config.Global.sshserver = None
+        self.default_config.Global.location = None
+
+    def pre_construct(self):
+        super(IPControllerApp, self).pre_construct()
+        c = self.master_config
+        # The defaults for these are set in FCClientServiceFactory and
+        # FCEngineServiceFactory, so we only set them here if the global
+        # options have be set to override the class level defaults.
+        
+        # if hasattr(c.Global, 'reuse_furls'):
+        #     c.FCClientServiceFactory.reuse_furls = c.Global.reuse_furls
+        #     c.FCEngineServiceFactory.reuse_furls = c.Global.reuse_furls
+        #     del c.Global.reuse_furls
+        # if hasattr(c.Global, 'secure'):
+        #     c.FCClientServiceFactory.secure = c.Global.secure
+        #     c.FCEngineServiceFactory.secure = c.Global.secure
+        #     del c.Global.secure
+    
+    def save_connection_dict(self, fname, cdict):
+        """save a connection dict to json file."""
+        c = self.master_config
+        url = cdict['url']
+        location = cdict['location']
+        if not location:
+            try:
+                proto,ip,port = split_url(url)
+            except AssertionError:
+                pass
+            else:
+                location = socket.gethostbyname_ex(socket.gethostname())[2][-1]
+            cdict['location'] = location
+        fname = os.path.join(c.Global.security_dir, fname)
+        with open(fname, 'w') as f:
+            f.write(json.dumps(cdict, indent=2))
+        os.chmod(fname, stat.S_IRUSR|stat.S_IWUSR)
+    
+    def load_config_from_json(self):
+        """load config from existing json connector files."""
+        c = self.master_config
+        # load from engine config
+        with open(os.path.join(c.Global.security_dir, 'ipcontroller-engine.json')) as f:
+            cfg = json.loads(f.read())
+        key = c.SessionFactory.exec_key = cfg['exec_key']
+        xport,addr = cfg['url'].split('://')
+        c.HubFactory.engine_transport = xport
+        ip,ports = addr.split(':')
+        c.HubFactory.engine_ip = ip
+        c.HubFactory.regport = int(ports)
+        c.Global.location = cfg['location']
+        
+        # load client config
+        with open(os.path.join(c.Global.security_dir, 'ipcontroller-client.json')) as f:
+            cfg = json.loads(f.read())
+        assert key == cfg['exec_key'], "exec_key mismatch between engine and client keys"
+        xport,addr = cfg['url'].split('://')
+        c.HubFactory.client_transport = xport
+        ip,ports = addr.split(':')
+        c.HubFactory.client_ip = ip
+        c.Global.sshserver = cfg['ssh']
+        assert int(ports) == c.HubFactory.regport, "regport mismatch"
+    
+    def construct(self):
+        # This is the working dir by now.
+        sys.path.insert(0, '')
+        c = self.master_config
+        
+        self.import_statements()
+        reusing = c.Global.reuse_files
+        if reusing:
+            try:
+                self.load_config_from_json()
+            except (AssertionError,IOError):
+                reusing=False
+        # check again, because reusing may have failed:
+        if reusing:
+            pass
+        elif c.Global.secure:
+            keyfile = os.path.join(c.Global.security_dir, c.Global.exec_key)
+            key = str(uuid.uuid4())
+            with open(keyfile, 'w') as f:
+                f.write(key)
+            os.chmod(keyfile, stat.S_IRUSR|stat.S_IWUSR)
+            c.SessionFactory.exec_key = key
+        else:
+            c.SessionFactory.exec_key = ''
+            key = None
+        
+        try:
+            self.factory = ControllerFactory(config=c, logname=self.log.name)
+            self.start_logging()
+            self.factory.construct()
+        except:
+            self.log.error("Couldn't construct the Controller", exc_info=True)
+            self.exit(1)
+        
+        if not reusing:
+            # save to new json config files
+            f = self.factory
+            cdict = {'exec_key' : key,
+                    'ssh' : c.Global.sshserver,
+                    'url' : "%s://%s:%s"%(f.client_transport, f.client_ip, f.regport),
+                    'location' : c.Global.location
+                    }
+            self.save_connection_dict('ipcontroller-client.json', cdict)
+            edict = cdict
+            edict['url']="%s://%s:%s"%((f.client_transport, f.client_ip, f.regport))
+            self.save_connection_dict('ipcontroller-engine.json', edict)
+        
+    
+    def save_urls(self):
+        """save the registration urls to files."""
+        c = self.master_config
+        
+        sec_dir = c.Global.security_dir
+        cf = self.factory
+        
+        with open(os.path.join(sec_dir, 'ipcontroller-engine.url'), 'w') as f:
+            f.write("%s://%s:%s"%(cf.engine_transport, cf.engine_ip, cf.regport))
+        
+        with open(os.path.join(sec_dir, 'ipcontroller-client.url'), 'w') as f:
+            f.write("%s://%s:%s"%(cf.client_transport, cf.client_ip, cf.regport))
+        
+    
+    def import_statements(self):
+        statements = self.master_config.Global.import_statements
+        for s in statements:
+            try:
+                self.log.msg("Executing statement: '%s'" % s)
+                exec s in globals(), locals()
+            except:
+                self.log.msg("Error running statement: %s" % s)
+
+    def start_logging(self):
+        super(IPControllerApp, self).start_logging()
+        if self.master_config.Global.log_url:
+            context = self.factory.context
+            lsock = context.socket(zmq.PUB)
+            lsock.connect(self.master_config.Global.log_url)
+            handler = PUBHandler(lsock)
+            handler.root_topic = 'controller'
+            handler.setLevel(self.log_level)
+            self.log.addHandler(handler)
+    # 
+    def start_app(self):
+        # Start the subprocesses:
+        self.factory.start()
+        self.write_pid_file(overwrite=True)
+        try:
+            self.factory.loop.start()
+        except KeyboardInterrupt:
+            self.log.critical("Interrupted, Exiting...\n")
+
+
+def launch_new_instance():
+    """Create and run the IPython controller"""
+    app = IPControllerApp()
+    app.start()
+
+
+if __name__ == '__main__':
+    launch_new_instance()

IPython/parallel/apps/ipengineapp.py

@@ -0,0 +1,295 b''
+#!/usr/bin/env python
+# encoding: utf-8
+"""
+The IPython engine application
+"""
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2009  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+import json
+import os
+import sys
+
+import zmq
+from zmq.eventloop import ioloop
+
+from .clusterdir import (
+    ApplicationWithClusterDir,
+    ClusterDirConfigLoader
+)
+from IPython.zmq.log import EnginePUBHandler
+
+from IPython.parallel import factory
+from IPython.parallel.engine.engine import EngineFactory
+from IPython.parallel.engine.streamkernel import Kernel
+from IPython.parallel.util import disambiguate_url
+
+from IPython.utils.importstring import import_item
+
+
+#-----------------------------------------------------------------------------
+# Module level variables
+#-----------------------------------------------------------------------------
+
+#: The default config file name for this application
+default_config_file_name = u'ipengine_config.py'
+
+
+mpi4py_init = """from mpi4py import MPI as mpi
+mpi.size = mpi.COMM_WORLD.Get_size()
+mpi.rank = mpi.COMM_WORLD.Get_rank()
+"""
+
+
+pytrilinos_init = """from PyTrilinos import Epetra
+class SimpleStruct:
+pass
+mpi = SimpleStruct()
+mpi.rank = 0
+mpi.size = 0
+"""
+
+
+_description = """Start an IPython engine for parallel computing.\n\n
+
+IPython engines run in parallel and perform computations on behalf of a client
+and controller. A controller needs to be started before the engines. The
+engine can be configured using command line options or using a cluster
+directory. Cluster directories contain config, log and security files and are
+usually located in your ipython directory and named as "cluster_<profile>".
+See the --profile and --cluster-dir options for details.
+"""
+
+#-----------------------------------------------------------------------------
+# Command line options
+#-----------------------------------------------------------------------------
+
+
+class IPEngineAppConfigLoader(ClusterDirConfigLoader):
+
+    def _add_arguments(self):
+        super(IPEngineAppConfigLoader, self)._add_arguments()
+        paa = self.parser.add_argument
+        # Controller config
+        paa('--file', '-f',
+            type=unicode, dest='Global.url_file',
+            help='The full location of the file containing the connection information fo '
+            'controller. If this is not given, the file must be in the '
+            'security directory of the cluster directory.  This location is '
+            'resolved using the --profile and --app-dir options.',
+            metavar='Global.url_file')
+        # MPI
+        paa('--mpi',
+            type=str, dest='MPI.use',
+            help='How to enable MPI (mpi4py, pytrilinos, or empty string to disable).',
+            metavar='MPI.use')
+        # Global config
+        paa('--log-to-file',
+            action='store_true', dest='Global.log_to_file',
+            help='Log to a file in the log directory (default is stdout)')
+        paa('--log-url',
+            dest='Global.log_url',
+            help="url of ZMQ logger, as started with iploggerz")
+        # paa('--execkey',
+        #     type=str, dest='Global.exec_key',
+        #     help='path to a file containing an execution key.',
+        #     metavar='keyfile')
+        # paa('--no-secure',
+        #     action='store_false', dest='Global.secure',
+        #     help='Turn off execution keys.')
+        # paa('--secure',
+        #     action='store_true', dest='Global.secure',
+        #     help='Turn on execution keys (default).')
+        # init command
+        paa('-c',
+            type=str, dest='Global.extra_exec_lines',
+            help='specify a command to be run at startup')
+        
+        factory.add_session_arguments(self.parser)
+        factory.add_registration_arguments(self.parser)
+
+
+#-----------------------------------------------------------------------------
+# Main application
+#-----------------------------------------------------------------------------
+
+
+class IPEngineApp(ApplicationWithClusterDir):
+
+    name = u'ipengine'
+    description = _description
+    command_line_loader = IPEngineAppConfigLoader
+    default_config_file_name = default_config_file_name
+    auto_create_cluster_dir = True
+
+    def create_default_config(self):
+        super(IPEngineApp, self).create_default_config()
+
+        # The engine should not clean logs as we don't want to remove the
+        # active log files of other running engines.
+        self.default_config.Global.clean_logs = False
+        self.default_config.Global.secure = True
+
+        # Global config attributes
+        self.default_config.Global.exec_lines = []
+        self.default_config.Global.extra_exec_lines = ''
+
+        # Configuration related to the controller
+        # This must match the filename (path not included) that the controller
+        # used for the FURL file.
+        self.default_config.Global.url_file = u''
+        self.default_config.Global.url_file_name = u'ipcontroller-engine.json'
+        # If given, this is the actual location of the controller's FURL file.
+        # If not, this is computed using the profile, app_dir and furl_file_name
+        # self.default_config.Global.key_file_name = u'exec_key.key'
+        # self.default_config.Global.key_file = u''
+
+        # MPI related config attributes
+        self.default_config.MPI.use = ''
+        self.default_config.MPI.mpi4py = mpi4py_init
+        self.default_config.MPI.pytrilinos = pytrilinos_init
+
+    def post_load_command_line_config(self):
+        pass
+
+    def pre_construct(self):
+        super(IPEngineApp, self).pre_construct()
+        # self.find_cont_url_file()
+        self.find_url_file()
+        if self.master_config.Global.extra_exec_lines:
+            self.master_config.Global.exec_lines.append(self.master_config.Global.extra_exec_lines)
+
+    # def find_key_file(self):
+    #     """Set the key file.
+    # 
+    #     Here we don't try to actually see if it exists for is valid as that
+    #     is hadled by the connection logic.
+    #     """
+    #     config = self.master_config
+    #     # Find the actual controller key file
+    #     if not config.Global.key_file:
+    #         try_this = os.path.join(
+    #             config.Global.cluster_dir, 
+    #             config.Global.security_dir,
+    #             config.Global.key_file_name
+    #         )
+    #         config.Global.key_file = try_this
+        
+    def find_url_file(self):
+        """Set the key file.
+
+        Here we don't try to actually see if it exists for is valid as that
+        is hadled by the connection logic.
+        """
+        config = self.master_config
+        # Find the actual controller key file
+        if not config.Global.url_file:
+            try_this = os.path.join(
+                config.Global.cluster_dir, 
+                config.Global.security_dir,
+                config.Global.url_file_name
+            )
+            config.Global.url_file = try_this
+        
+    def construct(self):
+        # This is the working dir by now.
+        sys.path.insert(0, '')
+        config = self.master_config
+        # if os.path.exists(config.Global.key_file) and config.Global.secure:
+        #     config.SessionFactory.exec_key = config.Global.key_file
+        if os.path.exists(config.Global.url_file):
+            with open(config.Global.url_file) as f:
+                d = json.loads(f.read())
+                for k,v in d.iteritems():
+                    if isinstance(v, unicode):
+                        d[k] = v.encode()
+            if d['exec_key']:
+                config.SessionFactory.exec_key = d['exec_key']
+            d['url'] = disambiguate_url(d['url'], d['location'])
+            config.RegistrationFactory.url=d['url']
+            config.EngineFactory.location = d['location']
+        
+        
+        
+        config.Kernel.exec_lines = config.Global.exec_lines
+
+        self.start_mpi()
+
+        # Create the underlying shell class and EngineService
+        # shell_class = import_item(self.master_config.Global.shell_class)
+        try:
+            self.engine = EngineFactory(config=config, logname=self.log.name)
+        except:
+            self.log.error("Couldn't start the Engine", exc_info=True)
+            self.exit(1)
+        
+        self.start_logging()
+
+        # Create the service hierarchy
+        # self.main_service = service.MultiService()
+        # self.engine_service.setServiceParent(self.main_service)
+        # self.tub_service = Tub()
+        # self.tub_service.setServiceParent(self.main_service)
+        # # This needs to be called before the connection is initiated
+        # self.main_service.startService()
+
+        # This initiates the connection to the controller and calls
+        # register_engine to tell the controller we are ready to do work
+        # self.engine_connector = EngineConnector(self.tub_service)
+
+        # self.log.info("Using furl file: %s" % self.master_config.Global.furl_file)
+
+        # reactor.callWhenRunning(self.call_connect)
+
+
+    def start_logging(self):
+        super(IPEngineApp, self).start_logging()
+        if self.master_config.Global.log_url:
+            context = self.engine.context
+            lsock = context.socket(zmq.PUB)
+            lsock.connect(self.master_config.Global.log_url)
+            handler = EnginePUBHandler(self.engine, lsock)
+            handler.setLevel(self.log_level)
+            self.log.addHandler(handler)
+    
+    def start_mpi(self):
+        global mpi
+        mpikey = self.master_config.MPI.use
+        mpi_import_statement = self.master_config.MPI.get(mpikey, None)
+        if mpi_import_statement is not None:
+            try:
+                self.log.info("Initializing MPI:")
+                self.log.info(mpi_import_statement)
+                exec mpi_import_statement in globals()
+            except:
+                mpi = None
+        else:
+            mpi = None
+
+
+    def start_app(self):
+        self.engine.start()
+        try:
+            self.engine.loop.start()
+        except KeyboardInterrupt:
+            self.log.critical("Engine Interrupted, shutting down...\n")
+
+
+def launch_new_instance():
+    """Create and run the IPython controller"""
+    app = IPEngineApp()
+    app.start()
+
+
+if __name__ == '__main__':
+    launch_new_instance()
+

IPython/parallel/apps/iploggerapp.py

@@ -0,0 +1,132 b''
+#!/usr/bin/env python
+# encoding: utf-8
+"""
+A simple IPython logger application
+"""
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2011  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+import os
+import sys
+
+import zmq
+
+from .clusterdir import (
+    ApplicationWithClusterDir,
+    ClusterDirConfigLoader
+)
+from .logwatcher import LogWatcher
+
+#-----------------------------------------------------------------------------
+# Module level variables
+#-----------------------------------------------------------------------------
+
+#: The default config file name for this application
+default_config_file_name = u'iplogger_config.py'
+
+_description = """Start an IPython logger for parallel computing.\n\n
+
+IPython controllers and engines (and your own processes) can broadcast log messages
+by registering a `zmq.log.handlers.PUBHandler` with the `logging` module. The
+logger can be configured using command line options or using a cluster
+directory. Cluster directories contain config, log and security files and are
+usually located in your ipython directory and named as "cluster_<profile>".
+See the --profile and --cluster-dir options for details.
+"""
+
+#-----------------------------------------------------------------------------
+# Command line options
+#-----------------------------------------------------------------------------
+
+
+class IPLoggerAppConfigLoader(ClusterDirConfigLoader):
+
+    def _add_arguments(self):
+        super(IPLoggerAppConfigLoader, self)._add_arguments()
+        paa = self.parser.add_argument
+        # Controller config
+        paa('--url',
+            type=str, dest='LogWatcher.url',
+            help='The url the LogWatcher will listen on',
+            )
+        # MPI
+        paa('--topics',
+            type=str, dest='LogWatcher.topics', nargs='+',
+            help='What topics to subscribe to',
+            metavar='topics')
+        # Global config
+        paa('--log-to-file',
+            action='store_true', dest='Global.log_to_file',
+            help='Log to a file in the log directory (default is stdout)')
+        
+
+#-----------------------------------------------------------------------------
+# Main application
+#-----------------------------------------------------------------------------
+
+
+class IPLoggerApp(ApplicationWithClusterDir):
+
+    name = u'iploggerz'
+    description = _description
+    command_line_loader = IPLoggerAppConfigLoader
+    default_config_file_name = default_config_file_name
+    auto_create_cluster_dir = True
+
+    def create_default_config(self):
+        super(IPLoggerApp, self).create_default_config()
+
+        # The engine should not clean logs as we don't want to remove the
+        # active log files of other running engines.
+        self.default_config.Global.clean_logs = False
+
+        # If given, this is the actual location of the logger's URL file.
+        # If not, this is computed using the profile, app_dir and furl_file_name
+        self.default_config.Global.url_file_name = u'iplogger.url'
+        self.default_config.Global.url_file = u''
+
+    def post_load_command_line_config(self):
+        pass
+
+    def pre_construct(self):
+        super(IPLoggerApp, self).pre_construct()
+
+    def construct(self):
+        # This is the working dir by now.
+        sys.path.insert(0, '')
+
+        self.start_logging()
+
+        try:
+            self.watcher = LogWatcher(config=self.master_config, logname=self.log.name)
+        except:
+            self.log.error("Couldn't start the LogWatcher", exc_info=True)
+            self.exit(1)
+        
+
+    def start_app(self):
+        try:
+            self.watcher.start()
+            self.watcher.loop.start()
+        except KeyboardInterrupt:
+            self.log.critical("Logging Interrupted, shutting down...\n")
+
+
+def launch_new_instance():
+    """Create and run the IPython LogWatcher"""
+    app = IPLoggerApp()
+    app.start()
+
+
+if __name__ == '__main__':
+    launch_new_instance()
+

IPython/parallel/apps/launcher.py

@@ -0,0 +1,971 b''
+#!/usr/bin/env python
+# encoding: utf-8
+"""
+Facilities for launching IPython processes asynchronously.
+"""
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2009  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+import copy
+import logging
+import os
+import re
+import stat
+
+from signal import SIGINT, SIGTERM
+try:
+    from signal import SIGKILL
+except ImportError:
+    SIGKILL=SIGTERM
+
+from subprocess import Popen, PIPE, STDOUT
+try:
+    from subprocess import check_output
+except ImportError:
+    # pre-2.7, define check_output with Popen
+    def check_output(*args, **kwargs):
+        kwargs.update(dict(stdout=PIPE))
+        p = Popen(*args, **kwargs)
+        out,err = p.communicate()
+        return out
+
+from zmq.eventloop import ioloop
+
+from IPython.external import Itpl
+# from IPython.config.configurable import Configurable
+from IPython.utils.traitlets import Any, Str, Int, List, Unicode, Dict, Instance, CUnicode
+from IPython.utils.path import get_ipython_module_path
+from IPython.utils.process import find_cmd, pycmd2argv, FindCmdError
+
+from IPython.parallel.factory import LoggingFactory
+
+# load winhpcjob only on Windows
+try:
+    from .winhpcjob import (
+        IPControllerTask, IPEngineTask,
+        IPControllerJob, IPEngineSetJob
+    )
+except ImportError:
+    pass
+
+
+#-----------------------------------------------------------------------------
+# Paths to the kernel apps
+#-----------------------------------------------------------------------------
+
+
+ipcluster_cmd_argv = pycmd2argv(get_ipython_module_path(
+    'IPython.parallel.apps.ipclusterapp'
+))
+
+ipengine_cmd_argv = pycmd2argv(get_ipython_module_path(
+    'IPython.parallel.apps.ipengineapp'
+))
+
+ipcontroller_cmd_argv = pycmd2argv(get_ipython_module_path(
+    'IPython.parallel.apps.ipcontrollerapp'
+))
+
+#-----------------------------------------------------------------------------
+# Base launchers and errors
+#-----------------------------------------------------------------------------
+
+
+class LauncherError(Exception):
+    pass
+
+
+class ProcessStateError(LauncherError):
+    pass
+
+
+class UnknownStatus(LauncherError):
+    pass
+
+
+class BaseLauncher(LoggingFactory):
+    """An asbtraction for starting, stopping and signaling a process."""
+
+    # In all of the launchers, the work_dir is where child processes will be
+    # run. This will usually be the cluster_dir, but may not be. any work_dir
+    # passed into the __init__ method will override the config value.
+    # This should not be used to set the work_dir for the actual engine
+    # and controller. Instead, use their own config files or the
+    # controller_args, engine_args attributes of the launchers to add
+    # the --work-dir option.
+    work_dir = Unicode(u'.')
+    loop = Instance('zmq.eventloop.ioloop.IOLoop')
+    
+    start_data = Any()
+    stop_data = Any()
+    
+    def _loop_default(self):
+        return ioloop.IOLoop.instance()
+
+    def __init__(self, work_dir=u'.', config=None, **kwargs):
+        super(BaseLauncher, self).__init__(work_dir=work_dir, config=config, **kwargs)
+        self.state = 'before' # can be before, running, after
+        self.stop_callbacks = []
+        self.start_data = None
+        self.stop_data = None
+
+    @property
+    def args(self):
+        """A list of cmd and args that will be used to start the process.
+
+        This is what is passed to :func:`spawnProcess` and the first element
+        will be the process name.
+        """
+        return self.find_args()
+
+    def find_args(self):
+        """The ``.args`` property calls this to find the args list.
+
+        Subcommand should implement this to construct the cmd and args.
+        """
+        raise NotImplementedError('find_args must be implemented in a subclass')
+
+    @property
+    def arg_str(self):
+        """The string form of the program arguments."""
+        return ' '.join(self.args)
+
+    @property
+    def running(self):
+        """Am I running."""
+        if self.state == 'running':
+            return True
+        else:
+            return False
+
+    def start(self):
+        """Start the process.
+
+        This must return a deferred that fires with information about the
+        process starting (like a pid, job id, etc.).
+        """
+        raise NotImplementedError('start must be implemented in a subclass')
+
+    def stop(self):
+        """Stop the process and notify observers of stopping.
+
+        This must return a deferred that fires with information about the
+        processing stopping, like errors that occur while the process is
+        attempting to be shut down. This deferred won't fire when the process
+        actually stops. To observe the actual process stopping, see
+        :func:`observe_stop`.
+        """
+        raise NotImplementedError('stop must be implemented in a subclass')
+
+    def on_stop(self, f):
+        """Get a deferred that will fire when the process stops.
+
+        The deferred will fire with data that contains information about
+        the exit status of the process.
+        """
+        if self.state=='after':
+            return f(self.stop_data)
+        else:
+            self.stop_callbacks.append(f)
+
+    def notify_start(self, data):
+        """Call this to trigger startup actions.
+
+        This logs the process startup and sets the state to 'running'.  It is
+        a pass-through so it can be used as a callback.
+        """
+
+        self.log.info('Process %r started: %r' % (self.args[0], data))
+        self.start_data = data
+        self.state = 'running'
+        return data
+
+    def notify_stop(self, data):
+        """Call this to trigger process stop actions.
+
+        This logs the process stopping and sets the state to 'after'. Call
+        this to trigger all the deferreds from :func:`observe_stop`."""
+
+        self.log.info('Process %r stopped: %r' % (self.args[0], data))
+        self.stop_data = data
+        self.state = 'after'
+        for i in range(len(self.stop_callbacks)):
+            d = self.stop_callbacks.pop()
+            d(data)
+        return data
+
+    def signal(self, sig):
+        """Signal the process.
+
+        Return a semi-meaningless deferred after signaling the process.
+
+        Parameters
+        ----------
+        sig : str or int
+            'KILL', 'INT', etc., or any signal number
+        """
+        raise NotImplementedError('signal must be implemented in a subclass')
+
+
+#-----------------------------------------------------------------------------
+# Local process launchers
+#-----------------------------------------------------------------------------
+
+
+class LocalProcessLauncher(BaseLauncher):
+    """Start and stop an external process in an asynchronous manner.
+
+    This will launch the external process with a working directory of
+    ``self.work_dir``.
+    """
+
+    # This is used to to construct self.args, which is passed to 
+    # spawnProcess.
+    cmd_and_args = List([])
+    poll_frequency = Int(100) # in ms
+
+    def __init__(self, work_dir=u'.', config=None, **kwargs):
+        super(LocalProcessLauncher, self).__init__(
+            work_dir=work_dir, config=config, **kwargs
+        )
+        self.process = None
+        self.start_deferred = None
+        self.poller = None
+
+    def find_args(self):
+        return self.cmd_and_args
+
+    def start(self):
+        if self.state == 'before':
+            self.process = Popen(self.args,
+                stdout=PIPE,stderr=PIPE,stdin=PIPE,
+                env=os.environ,
+                cwd=self.work_dir
+            )
+            
+            self.loop.add_handler(self.process.stdout.fileno(), self.handle_stdout, self.loop.READ)
+            self.loop.add_handler(self.process.stderr.fileno(), self.handle_stderr, self.loop.READ)
+            self.poller = ioloop.PeriodicCallback(self.poll, self.poll_frequency, self.loop)
+            self.poller.start()
+            self.notify_start(self.process.pid)
+        else:
+            s = 'The process was already started and has state: %r' % self.state
+            raise ProcessStateError(s)
+
+    def stop(self):
+        return self.interrupt_then_kill()
+
+    def signal(self, sig):
+        if self.state == 'running':
+            self.process.send_signal(sig)
+
+    def interrupt_then_kill(self, delay=2.0):
+        """Send INT, wait a delay and then send KILL."""
+        self.signal(SIGINT)
+        self.killer  = ioloop.DelayedCallback(lambda : self.signal(SIGKILL), delay*1000, self.loop)
+        self.killer.start()
+
+    # callbacks, etc:
+    
+    def handle_stdout(self, fd, events):
+        line = self.process.stdout.readline()
+        # a stopped process will be readable but return empty strings
+        if line:
+            self.log.info(line[:-1])
+        else:
+            self.poll()
+    
+    def handle_stderr(self, fd, events):
+        line = self.process.stderr.readline()
+        # a stopped process will be readable but return empty strings
+        if line:
+            self.log.error(line[:-1])
+        else:
+            self.poll()
+    
+    def poll(self):
+        status = self.process.poll()
+        if status is not None:
+            self.poller.stop()
+            self.loop.remove_handler(self.process.stdout.fileno())
+            self.loop.remove_handler(self.process.stderr.fileno())
+            self.notify_stop(dict(exit_code=status, pid=self.process.pid))
+        return status
+
+class LocalControllerLauncher(LocalProcessLauncher):
+    """Launch a controller as a regular external process."""
+
+    controller_cmd = List(ipcontroller_cmd_argv, config=True)
+    # Command line arguments to ipcontroller.
+    controller_args = List(['--log-to-file','--log-level', str(logging.INFO)], config=True)
+
+    def find_args(self):
+        return self.controller_cmd + self.controller_args
+
+    def start(self, cluster_dir):
+        """Start the controller by cluster_dir."""
+        self.controller_args.extend(['--cluster-dir', cluster_dir])
+        self.cluster_dir = unicode(cluster_dir)
+        self.log.info("Starting LocalControllerLauncher: %r" % self.args)
+        return super(LocalControllerLauncher, self).start()
+
+
+class LocalEngineLauncher(LocalProcessLauncher):
+    """Launch a single engine as a regular externall process."""
+
+    engine_cmd = List(ipengine_cmd_argv, config=True)
+    # Command line arguments for ipengine.
+    engine_args = List(
+        ['--log-to-file','--log-level', str(logging.INFO)], config=True
+    )
+
+    def find_args(self):
+        return self.engine_cmd + self.engine_args
+
+    def start(self, cluster_dir):
+        """Start the engine by cluster_dir."""
+        self.engine_args.extend(['--cluster-dir', cluster_dir])
+        self.cluster_dir = unicode(cluster_dir)
+        return super(LocalEngineLauncher, self).start()
+
+
+class LocalEngineSetLauncher(BaseLauncher):
+    """Launch a set of engines as regular external processes."""
+
+    # Command line arguments for ipengine.
+    engine_args = List(
+        ['--log-to-file','--log-level', str(logging.INFO)], config=True
+    )
+    # launcher class
+    launcher_class = LocalEngineLauncher
+    
+    launchers = Dict()
+    stop_data = Dict()
+    
+    def __init__(self, work_dir=u'.', config=None, **kwargs):
+        super(LocalEngineSetLauncher, self).__init__(
+            work_dir=work_dir, config=config, **kwargs
+        )
+        self.stop_data = {}
+
+    def start(self, n, cluster_dir):
+        """Start n engines by profile or cluster_dir."""
+        self.cluster_dir = unicode(cluster_dir)
+        dlist = []
+        for i in range(n):
+            el = self.launcher_class(work_dir=self.work_dir, config=self.config, logname=self.log.name)
+            # Copy the engine args over to each engine launcher.
+            el.engine_args = copy.deepcopy(self.engine_args)
+            el.on_stop(self._notice_engine_stopped)
+            d = el.start(cluster_dir)
+            if i==0:
+                self.log.info("Starting LocalEngineSetLauncher: %r" % el.args)
+            self.launchers[i] = el
+            dlist.append(d)
+        self.notify_start(dlist)
+        # The consumeErrors here could be dangerous
+        # dfinal = gatherBoth(dlist, consumeErrors=True)
+        # dfinal.addCallback(self.notify_start)
+        return dlist
+
+    def find_args(self):
+        return ['engine set']
+
+    def signal(self, sig):
+        dlist = []
+        for el in self.launchers.itervalues():
+            d = el.signal(sig)
+            dlist.append(d)
+        # dfinal = gatherBoth(dlist, consumeErrors=True)
+        return dlist
+
+    def interrupt_then_kill(self, delay=1.0):
+        dlist = []
+        for el in self.launchers.itervalues():
+            d = el.interrupt_then_kill(delay)
+            dlist.append(d)
+        # dfinal = gatherBoth(dlist, consumeErrors=True)
+        return dlist
+
+    def stop(self):
+        return self.interrupt_then_kill()
+    
+    def _notice_engine_stopped(self, data):
+        pid = data['pid']
+        for idx,el in self.launchers.iteritems():
+            if el.process.pid == pid:
+                break
+        self.launchers.pop(idx)
+        self.stop_data[idx] = data
+        if not self.launchers:
+            self.notify_stop(self.stop_data)
+
+
+#-----------------------------------------------------------------------------
+# MPIExec launchers
+#-----------------------------------------------------------------------------
+
+
+class MPIExecLauncher(LocalProcessLauncher):
+    """Launch an external process using mpiexec."""
+
+    # The mpiexec command to use in starting the process.
+    mpi_cmd = List(['mpiexec'], config=True)
+    # The command line arguments to pass to mpiexec.
+    mpi_args = List([], config=True)
+    # The program to start using mpiexec.
+    program = List(['date'], config=True)
+    # The command line argument to the program.
+    program_args = List([], config=True)
+    # The number of instances of the program to start.
+    n = Int(1, config=True)
+
+    def find_args(self):
+        """Build self.args using all the fields."""
+        return self.mpi_cmd + ['-n', str(self.n)] + self.mpi_args + \
+               self.program + self.program_args
+
+    def start(self, n):
+        """Start n instances of the program using mpiexec."""
+        self.n = n
+        return super(MPIExecLauncher, self).start()
+
+
+class MPIExecControllerLauncher(MPIExecLauncher):
+    """Launch a controller using mpiexec."""
+
+    controller_cmd = List(ipcontroller_cmd_argv, config=True)
+    # Command line arguments to ipcontroller.
+    controller_args = List(['--log-to-file','--log-level', str(logging.INFO)], config=True)
+    n = Int(1, config=False)
+
+    def start(self, cluster_dir):
+        """Start the controller by cluster_dir."""
+        self.controller_args.extend(['--cluster-dir', cluster_dir])
+        self.cluster_dir = unicode(cluster_dir)
+        self.log.info("Starting MPIExecControllerLauncher: %r" % self.args)
+        return super(MPIExecControllerLauncher, self).start(1)
+
+    def find_args(self):
+        return self.mpi_cmd + ['-n', self.n] + self.mpi_args + \
+               self.controller_cmd + self.controller_args
+
+
+class MPIExecEngineSetLauncher(MPIExecLauncher):
+
+    program = List(ipengine_cmd_argv, config=True)
+    # Command line arguments for ipengine.
+    program_args = List(
+        ['--log-to-file','--log-level', str(logging.INFO)], config=True
+    )
+    n = Int(1, config=True)
+
+    def start(self, n, cluster_dir):
+        """Start n engines by profile or cluster_dir."""
+        self.program_args.extend(['--cluster-dir', cluster_dir])
+        self.cluster_dir = unicode(cluster_dir)
+        self.n = n
+        self.log.info('Starting MPIExecEngineSetLauncher: %r' % self.args)
+        return super(MPIExecEngineSetLauncher, self).start(n)
+
+#-----------------------------------------------------------------------------
+# SSH launchers
+#-----------------------------------------------------------------------------
+
+# TODO: Get SSH Launcher working again.
+
+class SSHLauncher(LocalProcessLauncher):
+    """A minimal launcher for ssh.
+
+    To be useful this will probably have to be extended to use the ``sshx``
+    idea for environment variables.  There could be other things this needs
+    as well.
+    """
+
+    ssh_cmd = List(['ssh'], config=True)
+    ssh_args = List(['-tt'], config=True)
+    program = List(['date'], config=True)
+    program_args = List([], config=True)
+    hostname = CUnicode('', config=True)
+    user = CUnicode('', config=True)
+    location = CUnicode('')
+
+    def _hostname_changed(self, name, old, new):
+        if self.user:
+            self.location = u'%s@%s' % (self.user, new)
+        else:
+            self.location = new
+
+    def _user_changed(self, name, old, new):
+        self.location = u'%s@%s' % (new, self.hostname)
+
+    def find_args(self):
+        return self.ssh_cmd + self.ssh_args + [self.location] + \
+               self.program + self.program_args
+
+    def start(self, cluster_dir, hostname=None, user=None):
+        self.cluster_dir = unicode(cluster_dir)
+        if hostname is not None:
+            self.hostname = hostname
+        if user is not None:
+            self.user = user
+        
+        return super(SSHLauncher, self).start()
+    
+    def signal(self, sig):
+        if self.state == 'running':
+            # send escaped ssh connection-closer
+            self.process.stdin.write('~.')
+            self.process.stdin.flush()
+        
+
+
+class SSHControllerLauncher(SSHLauncher):
+
+    program = List(ipcontroller_cmd_argv, config=True)
+    # Command line arguments to ipcontroller.
+    program_args = List(['-r', '--log-to-file','--log-level', str(logging.INFO)], config=True)
+
+
+class SSHEngineLauncher(SSHLauncher):
+    program = List(ipengine_cmd_argv, config=True)
+    # Command line arguments for ipengine.
+    program_args = List(
+        ['--log-to-file','--log-level', str(logging.INFO)], config=True
+    )
+    
+class SSHEngineSetLauncher(LocalEngineSetLauncher):
+    launcher_class = SSHEngineLauncher
+    engines = Dict(config=True)
+    
+    def start(self, n, cluster_dir):
+        """Start engines by profile or cluster_dir.
+        `n` is ignored, and the `engines` config property is used instead.
+        """
+        
+        self.cluster_dir = unicode(cluster_dir)
+        dlist = []
+        for host, n in self.engines.iteritems():
+            if isinstance(n, (tuple, list)):
+                n, args = n
+            else:
+                args = copy.deepcopy(self.engine_args)
+            
+            if '@' in host:
+                user,host = host.split('@',1)
+            else:
+                user=None
+            for i in range(n):
+                el = self.launcher_class(work_dir=self.work_dir, config=self.config, logname=self.log.name)
+                
+                # Copy the engine args over to each engine launcher.
+                i
+                el.program_args = args
+                el.on_stop(self._notice_engine_stopped)
+                d = el.start(cluster_dir, user=user, hostname=host)
+                if i==0:
+                    self.log.info("Starting SSHEngineSetLauncher: %r" % el.args)
+                self.launchers[host+str(i)] = el
+                dlist.append(d)
+        self.notify_start(dlist)
+        return dlist
+    
+
+
+#-----------------------------------------------------------------------------
+# Windows HPC Server 2008 scheduler launchers
+#-----------------------------------------------------------------------------
+
+
+# This is only used on Windows.
+def find_job_cmd():
+    if os.name=='nt':
+        try:
+            return find_cmd('job')
+        except FindCmdError:
+            return 'job'
+    else:
+        return 'job'
+
+
+class WindowsHPCLauncher(BaseLauncher):
+
+    # A regular expression used to get the job id from the output of the 
+    # submit_command.
+    job_id_regexp = Str(r'\d+', config=True)
+    # The filename of the instantiated job script.
+    job_file_name = CUnicode(u'ipython_job.xml', config=True)
+    # The full path to the instantiated job script. This gets made dynamically
+    # by combining the work_dir with the job_file_name.
+    job_file = CUnicode(u'')
+    # The hostname of the scheduler to submit the job to
+    scheduler = CUnicode('', config=True)
+    job_cmd = CUnicode(find_job_cmd(), config=True)
+
+    def __init__(self, work_dir=u'.', config=None, **kwargs):
+        super(WindowsHPCLauncher, self).__init__(
+            work_dir=work_dir, config=config, **kwargs
+        )
+
+    @property
+    def job_file(self):
+        return os.path.join(self.work_dir, self.job_file_name)
+
+    def write_job_file(self, n):
+        raise NotImplementedError("Implement write_job_file in a subclass.")
+
+    def find_args(self):
+        return [u'job.exe']
+        
+    def parse_job_id(self, output):
+        """Take the output of the submit command and return the job id."""
+        m = re.search(self.job_id_regexp, output)
+        if m is not None:
+            job_id = m.group()
+        else:
+            raise LauncherError("Job id couldn't be determined: %s" % output)
+        self.job_id = job_id
+        self.log.info('Job started with job id: %r' % job_id)
+        return job_id
+
+    def start(self, n):
+        """Start n copies of the process using the Win HPC job scheduler."""
+        self.write_job_file(n)
+        args = [
+            'submit',
+            '/jobfile:%s' % self.job_file,
+            '/scheduler:%s' % self.scheduler
+        ]
+        self.log.info("Starting Win HPC Job: %s" % (self.job_cmd + ' ' + ' '.join(args),))
+        # Twisted will raise DeprecationWarnings if we try to pass unicode to this
+        output = check_output([self.job_cmd]+args,
+            env=os.environ,
+            cwd=self.work_dir,
+            stderr=STDOUT
+        )
+        job_id = self.parse_job_id(output)
+        self.notify_start(job_id)
+        return job_id
+
+    def stop(self):
+        args = [
+            'cancel',
+            self.job_id,
+            '/scheduler:%s' % self.scheduler
+        ]
+        self.log.info("Stopping Win HPC Job: %s" % (self.job_cmd + ' ' + ' '.join(args),))
+        try:
+            output = check_output([self.job_cmd]+args,
+                env=os.environ,
+                cwd=self.work_dir,
+                stderr=STDOUT
+            )
+        except:
+            output = 'The job already appears to be stoppped: %r' % self.job_id
+        self.notify_stop(dict(job_id=self.job_id, output=output))  # Pass the output of the kill cmd
+        return output
+
+
+class WindowsHPCControllerLauncher(WindowsHPCLauncher):
+
+    job_file_name = CUnicode(u'ipcontroller_job.xml', config=True)
+    extra_args = List([], config=False)
+
+    def write_job_file(self, n):
+        job = IPControllerJob(config=self.config)
+
+        t = IPControllerTask(config=self.config)
+        # The tasks work directory is *not* the actual work directory of 
+        # the controller. It is used as the base path for the stdout/stderr
+        # files that the scheduler redirects to.
+        t.work_directory = self.cluster_dir
+        # Add the --cluster-dir and from self.start().
+        t.controller_args.extend(self.extra_args)
+        job.add_task(t)
+
+        self.log.info("Writing job description file: %s" % self.job_file)
+        job.write(self.job_file)
+
+    @property
+    def job_file(self):
+        return os.path.join(self.cluster_dir, self.job_file_name)
+
+    def start(self, cluster_dir):
+        """Start the controller by cluster_dir."""
+        self.extra_args = ['--cluster-dir', cluster_dir]
+        self.cluster_dir = unicode(cluster_dir)
+        return super(WindowsHPCControllerLauncher, self).start(1)
+
+
+class WindowsHPCEngineSetLauncher(WindowsHPCLauncher):
+
+    job_file_name = CUnicode(u'ipengineset_job.xml', config=True)
+    extra_args = List([], config=False)
+
+    def write_job_file(self, n):
+        job = IPEngineSetJob(config=self.config)
+
+        for i in range(n):
+            t = IPEngineTask(config=self.config)
+            # The tasks work directory is *not* the actual work directory of 
+            # the engine. It is used as the base path for the stdout/stderr
+            # files that the scheduler redirects to.
+            t.work_directory = self.cluster_dir
+            # Add the --cluster-dir and from self.start().
+            t.engine_args.extend(self.extra_args)
+            job.add_task(t)
+
+        self.log.info("Writing job description file: %s" % self.job_file)
+        job.write(self.job_file)
+
+    @property
+    def job_file(self):
+        return os.path.join(self.cluster_dir, self.job_file_name)
+
+    def start(self, n, cluster_dir):
+        """Start the controller by cluster_dir."""
+        self.extra_args = ['--cluster-dir', cluster_dir]
+        self.cluster_dir = unicode(cluster_dir)
+        return super(WindowsHPCEngineSetLauncher, self).start(n)
+
+
+#-----------------------------------------------------------------------------
+# Batch (PBS) system launchers
+#-----------------------------------------------------------------------------
+
+class BatchSystemLauncher(BaseLauncher):
+    """Launch an external process using a batch system.
+
+    This class is designed to work with UNIX batch systems like PBS, LSF,
+    GridEngine, etc.  The overall model is that there are different commands
+    like qsub, qdel, etc. that handle the starting and stopping of the process.
+
+    This class also has the notion of a batch script. The ``batch_template``
+    attribute can be set to a string that is a template for the batch script.
+    This template is instantiated using Itpl. Thus the template can use
+    ${n} fot the number of instances. Subclasses can add additional variables
+    to the template dict.
+    """
+
+    # Subclasses must fill these in.  See PBSEngineSet
+    # The name of the command line program used to submit jobs.
+    submit_command = List([''], config=True)
+    # The name of the command line program used to delete jobs.
+    delete_command = List([''], config=True)
+    # A regular expression used to get the job id from the output of the 
+    # submit_command.
+    job_id_regexp = CUnicode('', config=True)
+    # The string that is the batch script template itself.
+    batch_template = CUnicode('', config=True)
+    # The file that contains the batch template
+    batch_template_file = CUnicode(u'', config=True)
+    # The filename of the instantiated batch script.
+    batch_file_name = CUnicode(u'batch_script', config=True)
+    # The PBS Queue
+    queue = CUnicode(u'', config=True)
+    
+    # not configurable, override in subclasses
+    # PBS Job Array regex
+    job_array_regexp = CUnicode('')
+    job_array_template = CUnicode('')
+    # PBS Queue regex
+    queue_regexp = CUnicode('')
+    queue_template = CUnicode('')
+    # The default batch template, override in subclasses
+    default_template = CUnicode('')
+    # The full path to the instantiated batch script.
+    batch_file = CUnicode(u'')
+    # the format dict used with batch_template:
+    context = Dict()
+
+    
+    def find_args(self):
+        return self.submit_command + [self.batch_file]
+    
+    def __init__(self, work_dir=u'.', config=None, **kwargs):
+        super(BatchSystemLauncher, self).__init__(
+            work_dir=work_dir, config=config, **kwargs
+        )
+        self.batch_file = os.path.join(self.work_dir, self.batch_file_name)
+
+    def parse_job_id(self, output):
+        """Take the output of the submit command and return the job id."""
+        m = re.search(self.job_id_regexp, output)
+        if m is not None:
+            job_id = m.group()
+        else:
+            raise LauncherError("Job id couldn't be determined: %s" % output)
+        self.job_id = job_id
+        self.log.info('Job submitted with job id: %r' % job_id)
+        return job_id
+
+    def write_batch_script(self, n):
+        """Instantiate and write the batch script to the work_dir."""
+        self.context['n'] = n
+        self.context['queue'] = self.queue
+        print self.context
+        # first priority is batch_template if set
+        if self.batch_template_file and not self.batch_template:
+            # second priority is batch_template_file
+            with open(self.batch_template_file) as f:
+                self.batch_template = f.read()
+        if not self.batch_template:
+            # third (last) priority is default_template
+            self.batch_template = self.default_template
+        
+        regex = re.compile(self.job_array_regexp)
+        # print regex.search(self.batch_template)
+        if not regex.search(self.batch_template):
+            self.log.info("adding job array settings to batch script")
+            firstline, rest = self.batch_template.split('\n',1)
+            self.batch_template = u'\n'.join([firstline, self.job_array_template, rest])
+        
+        regex = re.compile(self.queue_regexp)
+        # print regex.search(self.batch_template)
+        if self.queue and not regex.search(self.batch_template):
+            self.log.info("adding PBS queue settings to batch script")
+            firstline, rest = self.batch_template.split('\n',1)
+            self.batch_template = u'\n'.join([firstline, self.queue_template, rest])
+            
+        script_as_string = Itpl.itplns(self.batch_template, self.context)
+        self.log.info('Writing instantiated batch script: %s' % self.batch_file)
+    
+        with open(self.batch_file, 'w') as f:
+            f.write(script_as_string)
+        os.chmod(self.batch_file, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)
+
+    def start(self, n, cluster_dir):
+        """Start n copies of the process using a batch system."""
+        # Here we save profile and cluster_dir in the context so they
+        # can be used in the batch script template as ${profile} and
+        # ${cluster_dir}
+        self.context['cluster_dir'] = cluster_dir
+        self.cluster_dir = unicode(cluster_dir)
+        self.write_batch_script(n)
+        output = check_output(self.args, env=os.environ)
+        
+        job_id = self.parse_job_id(output)
+        self.notify_start(job_id)
+        return job_id
+
+    def stop(self):
+        output = check_output(self.delete_command+[self.job_id], env=os.environ)
+        self.notify_stop(dict(job_id=self.job_id, output=output)) # Pass the output of the kill cmd
+        return output
+
+
+class PBSLauncher(BatchSystemLauncher):
+    """A BatchSystemLauncher subclass for PBS."""
+
+    submit_command = List(['qsub'], config=True)
+    delete_command = List(['qdel'], config=True)
+    job_id_regexp = CUnicode(r'\d+', config=True)
+    
+    batch_file = CUnicode(u'')
+    job_array_regexp = CUnicode('#PBS\W+-t\W+[\w\d\-\$]+')
+    job_array_template = CUnicode('#PBS -t 1-$n')
+    queue_regexp = CUnicode('#PBS\W+-q\W+\$?\w+')
+    queue_template = CUnicode('#PBS -q $queue')
+
+
+class PBSControllerLauncher(PBSLauncher):
+    """Launch a controller using PBS."""
+
+    batch_file_name = CUnicode(u'pbs_controller', config=True)
+    default_template= CUnicode("""#!/bin/sh
+#PBS -V
+#PBS -N ipcontroller
+%s --log-to-file --cluster-dir $cluster_dir
+"""%(' '.join(ipcontroller_cmd_argv)))
+
+    def start(self, cluster_dir):
+        """Start the controller by profile or cluster_dir."""
+        self.log.info("Starting PBSControllerLauncher: %r" % self.args)
+        return super(PBSControllerLauncher, self).start(1, cluster_dir)
+
+
+class PBSEngineSetLauncher(PBSLauncher):
+    """Launch Engines using PBS"""
+    batch_file_name = CUnicode(u'pbs_engines', config=True)
+    default_template= CUnicode(u"""#!/bin/sh
+#PBS -V
+#PBS -N ipengine
+%s --cluster-dir $cluster_dir
+"""%(' '.join(ipengine_cmd_argv)))
+
+    def start(self, n, cluster_dir):
+        """Start n engines by profile or cluster_dir."""
+        self.log.info('Starting %i engines with PBSEngineSetLauncher: %r' % (n, self.args))
+        return super(PBSEngineSetLauncher, self).start(n, cluster_dir)
+
+#SGE is very similar to PBS
+
+class SGELauncher(PBSLauncher):
+    """Sun GridEngine is a PBS clone with slightly different syntax"""
+    job_array_regexp = CUnicode('#$$\W+-t\W+[\w\d\-\$]+')
+    job_array_template = CUnicode('#$$ -t 1-$n')
+    queue_regexp = CUnicode('#$$\W+-q\W+\$?\w+')
+    queue_template = CUnicode('#$$ -q $queue')
+
+class SGEControllerLauncher(SGELauncher):
+    """Launch a controller using SGE."""
+
+    batch_file_name = CUnicode(u'sge_controller', config=True)
+    default_template= CUnicode(u"""#$$ -V
+#$$ -S /bin/sh
+#$$ -N ipcontroller
+%s --log-to-file --cluster-dir $cluster_dir
+"""%(' '.join(ipcontroller_cmd_argv)))
+
+    def start(self, cluster_dir):
+        """Start the controller by profile or cluster_dir."""
+        self.log.info("Starting PBSControllerLauncher: %r" % self.args)
+        return super(PBSControllerLauncher, self).start(1, cluster_dir)
+
+class SGEEngineSetLauncher(SGELauncher):
+    """Launch Engines with SGE"""
+    batch_file_name = CUnicode(u'sge_engines', config=True)
+    default_template = CUnicode("""#$$ -V
+#$$ -S /bin/sh
+#$$ -N ipengine
+%s --cluster-dir $cluster_dir
+"""%(' '.join(ipengine_cmd_argv)))
+
+    def start(self, n, cluster_dir):
+        """Start n engines by profile or cluster_dir."""
+        self.log.info('Starting %i engines with SGEEngineSetLauncher: %r' % (n, self.args))
+        return super(SGEEngineSetLauncher, self).start(n, cluster_dir)
+
+
+#-----------------------------------------------------------------------------
+# A launcher for ipcluster itself!
+#-----------------------------------------------------------------------------
+
+
+class IPClusterLauncher(LocalProcessLauncher):
+    """Launch the ipcluster program in an external process."""
+
+    ipcluster_cmd = List(ipcluster_cmd_argv, config=True)
+    # Command line arguments to pass to ipcluster.
+    ipcluster_args = List(
+        ['--clean-logs', '--log-to-file', '--log-level', str(logging.INFO)], config=True)
+    ipcluster_subcommand = Str('start')
+    ipcluster_n = Int(2)
+
+    def find_args(self):
+        return self.ipcluster_cmd + [self.ipcluster_subcommand] + \
+            ['-n', repr(self.ipcluster_n)] + self.ipcluster_args
+
+    def start(self):
+        self.log.info("Starting ipcluster: %r" % self.args)
+        return super(IPClusterLauncher, self).start()
+

IPython/parallel/apps/logwatcher.py

@@ -0,0 +1,98 b''
+#!/usr/bin/env python
+"""A simple logger object that consolidates messages incoming from ipcluster processes."""
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2011  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+
+import logging
+import sys
+
+import zmq
+from zmq.eventloop import ioloop, zmqstream
+
+from IPython.utils.traitlets import Int, Str, Instance, List
+
+from IPython.parallel.factory import LoggingFactory
+
+#-----------------------------------------------------------------------------
+# Classes
+#-----------------------------------------------------------------------------
+
+
+class LogWatcher(LoggingFactory):
+    """A simple class that receives messages on a SUB socket, as published
+    by subclasses of `zmq.log.handlers.PUBHandler`, and logs them itself.
+    
+    This can subscribe to multiple topics, but defaults to all topics.
+    """
+    # configurables
+    topics = List([''], config=True)
+    url = Str('tcp://127.0.0.1:20202', config=True)
+    
+    # internals
+    context = Instance(zmq.Context, (), {})
+    stream = Instance('zmq.eventloop.zmqstream.ZMQStream')
+    loop = Instance('zmq.eventloop.ioloop.IOLoop')
+    def _loop_default(self):
+        return ioloop.IOLoop.instance()
+    
+    def __init__(self, **kwargs):
+        super(LogWatcher, self).__init__(**kwargs)
+        s = self.context.socket(zmq.SUB)
+        s.bind(self.url)
+        self.stream = zmqstream.ZMQStream(s, self.loop)
+        self.subscribe()
+        self.on_trait_change(self.subscribe, 'topics')
+    
+    def start(self):
+        self.stream.on_recv(self.log_message)
+    
+    def stop(self):
+        self.stream.stop_on_recv()
+    
+    def subscribe(self):
+        """Update our SUB socket's subscriptions."""
+        self.stream.setsockopt(zmq.UNSUBSCRIBE, '')
+        for topic in self.topics:
+            self.log.debug("Subscribing to: %r"%topic)
+            self.stream.setsockopt(zmq.SUBSCRIBE, topic)
+    
+    def _extract_level(self, topic_str):
+        """Turn 'engine.0.INFO.extra' into (logging.INFO, 'engine.0.extra')"""
+        topics = topic_str.split('.')
+        for idx,t in enumerate(topics):
+            level = getattr(logging, t, None)
+            if level is not None:
+                break
+        
+        if level is None:
+            level = logging.INFO
+        else:
+            topics.pop(idx)
+        
+        return level, '.'.join(topics)
+            
+            
+    def log_message(self, raw):
+        """receive and parse a message, then log it."""
+        if len(raw) != 2 or '.' not in raw[0]:
+            self.log.error("Invalid log message: %s"%raw)
+            return
+        else:
+            topic, msg = raw
+            # don't newline, since log messages always newline:
+            topic,level_name = topic.rsplit('.',1)
+            level,topic = self._extract_level(topic)
+            if msg[-1] == '\n':
+                msg = msg[:-1]
+            logging.log(level, "[%s] %s" % (topic, msg))
+

IPython/parallel/apps/winhpcjob.py

@@ -0,0 +1,316 b''
+#!/usr/bin/env python
+# encoding: utf-8
+"""
+Job and task components for writing .xml files that the Windows HPC Server 
+2008 can use to start jobs.
+"""
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2009  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+from __future__ import with_statement
+
+import os
+import re
+import uuid
+
+from xml.etree import ElementTree as ET
+
+from IPython.config.configurable import Configurable
+from IPython.utils.traitlets import (
+    Str, Int, List, Instance,
+    Enum, Bool, CStr
+)
+
+#-----------------------------------------------------------------------------
+# Job and Task classes
+#-----------------------------------------------------------------------------
+
+
+def as_str(value):
+    if isinstance(value, str):
+        return value
+    elif isinstance(value, bool):
+        if value:
+            return 'true'
+        else:
+            return 'false'
+    elif isinstance(value, (int, float)):
+        return repr(value)
+    else:
+        return value
+
+
+def indent(elem, level=0):
+    i = "\n" + level*"  "
+    if len(elem):
+        if not elem.text or not elem.text.strip():
+            elem.text = i + "  "
+        if not elem.tail or not elem.tail.strip():
+            elem.tail = i
+        for elem in elem:
+            indent(elem, level+1)
+        if not elem.tail or not elem.tail.strip():
+            elem.tail = i
+    else:
+        if level and (not elem.tail or not elem.tail.strip()):
+            elem.tail = i
+
+
+def find_username():
+    domain = os.environ.get('USERDOMAIN')
+    username = os.environ.get('USERNAME','')
+    if domain is None:
+        return username
+    else:
+        return '%s\\%s' % (domain, username)
+
+
+class WinHPCJob(Configurable):
+
+    job_id = Str('')
+    job_name = Str('MyJob', config=True)
+    min_cores = Int(1, config=True)
+    max_cores = Int(1, config=True)
+    min_sockets = Int(1, config=True)
+    max_sockets = Int(1, config=True)
+    min_nodes = Int(1, config=True)
+    max_nodes = Int(1, config=True)
+    unit_type = Str("Core", config=True)
+    auto_calculate_min = Bool(True, config=True)
+    auto_calculate_max = Bool(True, config=True)
+    run_until_canceled = Bool(False, config=True)
+    is_exclusive = Bool(False, config=True)
+    username = Str(find_username(), config=True)
+    job_type = Str('Batch', config=True)
+    priority = Enum(('Lowest','BelowNormal','Normal','AboveNormal','Highest'),
+        default_value='Highest', config=True)
+    requested_nodes = Str('', config=True)
+    project = Str('IPython', config=True)
+    xmlns = Str('http://schemas.microsoft.com/HPCS2008/scheduler/')
+    version = Str("2.000")
+    tasks = List([])
+
+    @property
+    def owner(self):
+        return self.username
+
+    def _write_attr(self, root, attr, key):
+        s = as_str(getattr(self, attr, ''))
+        if s:
+            root.set(key, s)
+
+    def as_element(self):
+        # We have to add _A_ type things to get the right order than
+        # the MSFT XML parser expects.
+        root = ET.Element('Job')
+        self._write_attr(root, 'version', '_A_Version')
+        self._write_attr(root, 'job_name', '_B_Name')
+        self._write_attr(root, 'unit_type', '_C_UnitType')
+        self._write_attr(root, 'min_cores', '_D_MinCores')
+        self._write_attr(root, 'max_cores', '_E_MaxCores')
+        self._write_attr(root, 'min_sockets', '_F_MinSockets')
+        self._write_attr(root, 'max_sockets', '_G_MaxSockets')
+        self._write_attr(root, 'min_nodes', '_H_MinNodes')
+        self._write_attr(root, 'max_nodes', '_I_MaxNodes')
+        self._write_attr(root, 'run_until_canceled', '_J_RunUntilCanceled')
+        self._write_attr(root, 'is_exclusive', '_K_IsExclusive')
+        self._write_attr(root, 'username', '_L_UserName')
+        self._write_attr(root, 'job_type', '_M_JobType')
+        self._write_attr(root, 'priority', '_N_Priority')
+        self._write_attr(root, 'requested_nodes', '_O_RequestedNodes')
+        self._write_attr(root, 'auto_calculate_max', '_P_AutoCalculateMax')
+        self._write_attr(root, 'auto_calculate_min', '_Q_AutoCalculateMin')
+        self._write_attr(root, 'project', '_R_Project')
+        self._write_attr(root, 'owner', '_S_Owner')
+        self._write_attr(root, 'xmlns', '_T_xmlns')
+        dependencies = ET.SubElement(root, "Dependencies")
+        etasks = ET.SubElement(root, "Tasks")
+        for t in self.tasks:
+            etasks.append(t.as_element())
+        return root
+
+    def tostring(self):
+        """Return the string representation of the job description XML."""
+        root = self.as_element()
+        indent(root)
+        txt = ET.tostring(root, encoding="utf-8")
+        # Now remove the tokens used to order the attributes.
+        txt = re.sub(r'_[A-Z]_','',txt)
+        txt = '<?xml version="1.0" encoding="utf-8"?>\n' + txt
+        return txt
+
+    def write(self, filename):
+        """Write the XML job description to a file."""
+        txt = self.tostring()
+        with open(filename, 'w') as f:
+            f.write(txt)
+
+    def add_task(self, task):
+        """Add a task to the job.
+
+        Parameters
+        ----------
+        task : :class:`WinHPCTask`
+            The task object to add.
+        """
+        self.tasks.append(task)
+
+
+class WinHPCTask(Configurable):
+
+    task_id = Str('')
+    task_name = Str('')
+    version = Str("2.000")
+    min_cores = Int(1, config=True)
+    max_cores = Int(1, config=True)
+    min_sockets = Int(1, config=True)
+    max_sockets = Int(1, config=True)
+    min_nodes = Int(1, config=True)
+    max_nodes = Int(1, config=True)
+    unit_type = Str("Core", config=True)
+    command_line = CStr('', config=True)
+    work_directory = CStr('', config=True)
+    is_rerunnaable = Bool(True, config=True)
+    std_out_file_path = CStr('', config=True)
+    std_err_file_path = CStr('', config=True)
+    is_parametric = Bool(False, config=True)
+    environment_variables = Instance(dict, args=(), config=True)
+
+    def _write_attr(self, root, attr, key):
+        s = as_str(getattr(self, attr, ''))
+        if s:
+            root.set(key, s)
+
+    def as_element(self):
+        root = ET.Element('Task')
+        self._write_attr(root, 'version', '_A_Version')
+        self._write_attr(root, 'task_name', '_B_Name')
+        self._write_attr(root, 'min_cores', '_C_MinCores')
+        self._write_attr(root, 'max_cores', '_D_MaxCores')
+        self._write_attr(root, 'min_sockets', '_E_MinSockets')
+        self._write_attr(root, 'max_sockets', '_F_MaxSockets')
+        self._write_attr(root, 'min_nodes', '_G_MinNodes')
+        self._write_attr(root, 'max_nodes', '_H_MaxNodes')
+        self._write_attr(root, 'command_line', '_I_CommandLine')
+        self._write_attr(root, 'work_directory', '_J_WorkDirectory')
+        self._write_attr(root, 'is_rerunnaable', '_K_IsRerunnable')
+        self._write_attr(root, 'std_out_file_path', '_L_StdOutFilePath')
+        self._write_attr(root, 'std_err_file_path', '_M_StdErrFilePath')
+        self._write_attr(root, 'is_parametric', '_N_IsParametric')
+        self._write_attr(root, 'unit_type', '_O_UnitType')
+        root.append(self.get_env_vars())
+        return root
+
+    def get_env_vars(self):
+        env_vars = ET.Element('EnvironmentVariables')
+        for k, v in self.environment_variables.iteritems():
+            variable = ET.SubElement(env_vars, "Variable")
+            name = ET.SubElement(variable, "Name")
+            name.text = k
+            value = ET.SubElement(variable, "Value")
+            value.text = v
+        return env_vars
+
+
+
+# By declaring these, we can configure the controller and engine separately!
+
+class IPControllerJob(WinHPCJob):
+    job_name = Str('IPController', config=False)
+    is_exclusive = Bool(False, config=True)
+    username = Str(find_username(), config=True)
+    priority = Enum(('Lowest','BelowNormal','Normal','AboveNormal','Highest'),
+        default_value='Highest', config=True)
+    requested_nodes = Str('', config=True)
+    project = Str('IPython', config=True)
+
+
+class IPEngineSetJob(WinHPCJob):
+    job_name = Str('IPEngineSet', config=False)
+    is_exclusive = Bool(False, config=True)
+    username = Str(find_username(), config=True)
+    priority = Enum(('Lowest','BelowNormal','Normal','AboveNormal','Highest'),
+        default_value='Highest', config=True)
+    requested_nodes = Str('', config=True)
+    project = Str('IPython', config=True)
+
+
+class IPControllerTask(WinHPCTask):
+
+    task_name = Str('IPController', config=True)
+    controller_cmd = List(['ipcontroller.exe'], config=True)
+    controller_args = List(['--log-to-file', '--log-level', '40'], config=True)
+    # I don't want these to be configurable
+    std_out_file_path = CStr('', config=False)
+    std_err_file_path = CStr('', config=False)
+    min_cores = Int(1, config=False)
+    max_cores = Int(1, config=False)
+    min_sockets = Int(1, config=False)
+    max_sockets = Int(1, config=False)
+    min_nodes = Int(1, config=False)
+    max_nodes = Int(1, config=False)
+    unit_type = Str("Core", config=False)
+    work_directory = CStr('', config=False)
+
+    def __init__(self, config=None):
+        super(IPControllerTask, self).__init__(config=config)
+        the_uuid = uuid.uuid1()
+        self.std_out_file_path = os.path.join('log','ipcontroller-%s.out' % the_uuid)
+        self.std_err_file_path = os.path.join('log','ipcontroller-%s.err' % the_uuid)
+
+    @property
+    def command_line(self):
+        return ' '.join(self.controller_cmd + self.controller_args)
+
+
+class IPEngineTask(WinHPCTask):
+
+    task_name = Str('IPEngine', config=True)
+    engine_cmd = List(['ipengine.exe'], config=True)
+    engine_args = List(['--log-to-file', '--log-level', '40'], config=True)
+    # I don't want these to be configurable
+    std_out_file_path = CStr('', config=False)
+    std_err_file_path = CStr('', config=False)
+    min_cores = Int(1, config=False)
+    max_cores = Int(1, config=False)
+    min_sockets = Int(1, config=False)
+    max_sockets = Int(1, config=False)
+    min_nodes = Int(1, config=False)
+    max_nodes = Int(1, config=False)
+    unit_type = Str("Core", config=False)
+    work_directory = CStr('', config=False)
+
+    def __init__(self, config=None):
+        super(IPEngineTask,self).__init__(config=config)
+        the_uuid = uuid.uuid1()
+        self.std_out_file_path = os.path.join('log','ipengine-%s.out' % the_uuid)
+        self.std_err_file_path = os.path.join('log','ipengine-%s.err' % the_uuid)
+    
+    @property
+    def command_line(self):
+        return ' '.join(self.engine_cmd + self.engine_args)
+
+
+# j = WinHPCJob(None)
+# j.job_name = 'IPCluster'
+# j.username = 'GNET\\bgranger'
+# j.requested_nodes = 'GREEN'
+# 
+# t = WinHPCTask(None)
+# t.task_name = 'Controller'
+# t.command_line = r"\\blue\domainusers$\bgranger\Python\Python25\Scripts\ipcontroller.exe --log-to-file -p default --log-level 10"
+# t.work_directory = r"\\blue\domainusers$\bgranger\.ipython\cluster_default"
+# t.std_out_file_path = 'controller-out.txt'
+# t.std_err_file_path = 'controller-err.txt'
+# t.environment_variables['PYTHONPATH'] = r"\\blue\domainusers$\bgranger\Python\Python25\Lib\site-packages"
+# j.add_task(t)
+

IPython/parallel/client/asyncresult.py

@@ -0,0 +1,340 b''
+"""AsyncResult objects for the client"""
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2010-2011  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+import time
+
+from zmq import MessageTracker
+
+from IPython.external.decorator import decorator
+from IPython.parallel import error
+
+#-----------------------------------------------------------------------------
+# Classes
+#-----------------------------------------------------------------------------
+
+# global empty tracker that's always done:
+finished_tracker = MessageTracker()
+
+@decorator
+def check_ready(f, self, *args, **kwargs):
+    """Call spin() to sync state prior to calling the method."""
+    self.wait(0)
+    if not self._ready:
+        raise error.TimeoutError("result not ready")
+    return f(self, *args, **kwargs)
+
+class AsyncResult(object):
+    """Class for representing results of non-blocking calls.
+    
+    Provides the same interface as :py:class:`multiprocessing.pool.AsyncResult`.
+    """
+    
+    msg_ids = None
+    _targets = None
+    _tracker = None
+    _single_result = False
+    
+    def __init__(self, client, msg_ids, fname='unknown', targets=None, tracker=None):
+        if isinstance(msg_ids, basestring):
+            # always a list
+            msg_ids = [msg_ids]
+        if tracker is None:
+            # default to always done
+            tracker = finished_tracker
+        self._client = client
+        self.msg_ids = msg_ids
+        self._fname=fname
+        self._targets = targets
+        self._tracker = tracker
+        self._ready = False
+        self._success = None
+        if len(msg_ids) == 1:
+            self._single_result = not isinstance(targets, (list, tuple))
+        else:
+            self._single_result = False
+    
+    def __repr__(self):
+        if self._ready:
+            return "<%s: finished>"%(self.__class__.__name__)
+        else:
+            return "<%s: %s>"%(self.__class__.__name__,self._fname)
+    
+    
+    def _reconstruct_result(self, res):
+        """Reconstruct our result from actual result list (always a list)
+        
+        Override me in subclasses for turning a list of results
+        into the expected form.
+        """
+        if self._single_result:
+            return res[0]
+        else:
+            return res
+        
+    def get(self, timeout=-1):
+        """Return the result when it arrives. 
+        
+        If `timeout` is not ``None`` and the result does not arrive within
+        `timeout` seconds then ``TimeoutError`` is raised. If the
+        remote call raised an exception then that exception will be reraised
+        by get() inside a `RemoteError`.
+        """
+        if not self.ready():
+            self.wait(timeout)
+        
+        if self._ready:
+            if self._success:
+                return self._result
+            else:
+                raise self._exception
+        else:
+            raise error.TimeoutError("Result not ready.")
+    
+    def ready(self):
+        """Return whether the call has completed."""
+        if not self._ready:
+            self.wait(0)
+        return self._ready
+    
+    def wait(self, timeout=-1):
+        """Wait until the result is available or until `timeout` seconds pass.
+        
+        This method always returns None.
+        """
+        if self._ready:
+            return
+        self._ready = self._client.wait(self.msg_ids, timeout)
+        if self._ready:
+            try:
+                results = map(self._client.results.get, self.msg_ids)
+                self._result = results
+                if self._single_result:
+                    r = results[0]
+                    if isinstance(r, Exception):
+                        raise r
+                else:
+                    results = error.collect_exceptions(results, self._fname)
+                self._result = self._reconstruct_result(results)
+            except Exception, e:
+                self._exception = e
+                self._success = False
+            else:
+                self._success = True
+            finally:
+                self._metadata = map(self._client.metadata.get, self.msg_ids)
+            
+    
+    def successful(self):
+        """Return whether the call completed without raising an exception. 
+        
+        Will raise ``AssertionError`` if the result is not ready.
+        """
+        assert self.ready()
+        return self._success
+    
+    #----------------------------------------------------------------
+    # Extra methods not in mp.pool.AsyncResult
+    #----------------------------------------------------------------
+    
+    def get_dict(self, timeout=-1):
+        """Get the results as a dict, keyed by engine_id.
+        
+        timeout behavior is described in `get()`.
+        """
+        
+        results = self.get(timeout)
+        engine_ids = [ md['engine_id'] for md in self._metadata ]
+        bycount = sorted(engine_ids, key=lambda k: engine_ids.count(k))
+        maxcount = bycount.count(bycount[-1])
+        if maxcount > 1:
+            raise ValueError("Cannot build dict, %i jobs ran on engine #%i"%(
+                    maxcount, bycount[-1]))
+        
+        return dict(zip(engine_ids,results))
+    
+    @property
+    def result(self):
+        """result property wrapper for `get(timeout=0)`."""
+        return self.get()
+    
+    # abbreviated alias:
+    r = result
+    
+    @property
+    @check_ready
+    def metadata(self):
+        """property for accessing execution metadata."""
+        if self._single_result:
+            return self._metadata[0]
+        else:
+            return self._metadata
+    
+    @property
+    def result_dict(self):
+        """result property as a dict."""
+        return self.get_dict()
+    
+    def __dict__(self):
+        return self.get_dict(0)
+    
+    def abort(self):
+        """abort my tasks."""
+        assert not self.ready(), "Can't abort, I am already done!"
+        return self.client.abort(self.msg_ids, targets=self._targets, block=True)
+
+    @property
+    def sent(self):
+        """check whether my messages have been sent."""
+        return self._tracker.done
+    
+    def wait_for_send(self, timeout=-1):
+        """wait for pyzmq send to complete.
+        
+        This is necessary when sending arrays that you intend to edit in-place.
+        `timeout` is in seconds, and will raise TimeoutError if it is reached
+        before the send completes.
+        """
+        return self._tracker.wait(timeout)
+
+    #-------------------------------------
+    # dict-access
+    #-------------------------------------
+    
+    @check_ready
+    def __getitem__(self, key):
+        """getitem returns result value(s) if keyed by int/slice, or metadata if key is str.
+        """
+        if isinstance(key, int):
+            return error.collect_exceptions([self._result[key]], self._fname)[0]
+        elif isinstance(key, slice):
+            return error.collect_exceptions(self._result[key], self._fname)
+        elif isinstance(key, basestring):
+            values = [ md[key] for md in self._metadata ]
+            if self._single_result:
+                return values[0]
+            else:
+                return values
+        else:
+            raise TypeError("Invalid key type %r, must be 'int','slice', or 'str'"%type(key))
+    
+    @check_ready
+    def __getattr__(self, key):
+        """getattr maps to getitem for convenient attr access to metadata."""
+        if key not in self._metadata[0].keys():
+            raise AttributeError("%r object has no attribute %r"%(
+                    self.__class__.__name__, key))
+        return self.__getitem__(key)
+    
+    # asynchronous iterator:
+    def __iter__(self):
+        if self._single_result:
+            raise TypeError("AsyncResults with a single result are not iterable.")
+        try:
+            rlist = self.get(0)
+        except error.TimeoutError:
+            # wait for each result individually
+            for msg_id in self.msg_ids:
+                ar = AsyncResult(self._client, msg_id, self._fname)
+                yield ar.get()
+        else:
+            # already done
+            for r in rlist:
+                yield r
+
+
+    
+class AsyncMapResult(AsyncResult):
+    """Class for representing results of non-blocking gathers.
+    
+    This will properly reconstruct the gather.
+    """
+    
+    def __init__(self, client, msg_ids, mapObject, fname=''):
+        AsyncResult.__init__(self, client, msg_ids, fname=fname)
+        self._mapObject = mapObject
+        self._single_result = False
+    
+    def _reconstruct_result(self, res):
+        """Perform the gather on the actual results."""
+        return self._mapObject.joinPartitions(res)
+    
+    # asynchronous iterator:
+    def __iter__(self):
+        try:
+            rlist = self.get(0)
+        except error.TimeoutError:
+            # wait for each result individually
+            for msg_id in self.msg_ids:
+                ar = AsyncResult(self._client, msg_id, self._fname)
+                rlist = ar.get()
+                try:
+                    for r in rlist:
+                        yield r
+                except TypeError:
+                    # flattened, not a list
+                    # this could get broken by flattened data that returns iterables
+                    # but most calls to map do not expose the `flatten` argument
+                    yield rlist
+        else:
+            # already done
+            for r in rlist:
+                yield r
+
+
+class AsyncHubResult(AsyncResult):
+    """Class to wrap pending results that must be requested from the Hub.
+    
+    Note that waiting/polling on these objects requires polling the Hubover the network,
+    so use `AsyncHubResult.wait()` sparingly.
+    """
+    
+    def wait(self, timeout=-1):
+        """wait for result to complete."""
+        start = time.time()
+        if self._ready:
+            return
+        local_ids = filter(lambda msg_id: msg_id in self._client.outstanding, self.msg_ids)
+        local_ready = self._client.wait(local_ids, timeout)
+        if local_ready:
+            remote_ids = filter(lambda msg_id: msg_id not in self._client.results, self.msg_ids)
+            if not remote_ids:
+                self._ready = True
+            else:
+                rdict = self._client.result_status(remote_ids, status_only=False)
+                pending = rdict['pending']
+                while pending and (timeout < 0 or time.time() < start+timeout):
+                    rdict = self._client.result_status(remote_ids, status_only=False)
+                    pending = rdict['pending']
+                    if pending:
+                        time.sleep(0.1)
+                if not pending:
+                    self._ready = True
+        if self._ready:
+            try:
+                results = map(self._client.results.get, self.msg_ids)
+                self._result = results
+                if self._single_result:
+                    r = results[0]
+                    if isinstance(r, Exception):
+                        raise r
+                else:
+                    results = error.collect_exceptions(results, self._fname)
+                self._result = self._reconstruct_result(results)
+            except Exception, e:
+                self._exception = e
+                self._success = False
+            else:
+                self._success = True
+            finally:
+                self._metadata = map(self._client.metadata.get, self.msg_ids)
+        
+__all__ = ['AsyncResult', 'AsyncMapResult', 'AsyncHubResult']
  No newline at end of file

IPython/parallel/client/client.py

@@ -0,0 +1,1279 b''
+"""A semi-synchronous Client for the ZMQ cluster"""
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2010  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+import os
+import json
+import time
+import warnings
+from datetime import datetime
+from getpass import getpass
+from pprint import pprint
+
+pjoin = os.path.join
+
+import zmq
+# from zmq.eventloop import ioloop, zmqstream
+
+from IPython.utils.path import get_ipython_dir
+from IPython.utils.traitlets import (HasTraits, Int, Instance, CUnicode, 
+                                    Dict, List, Bool, Str, Set)
+from IPython.external.decorator import decorator
+from IPython.external.ssh import tunnel
+
+from IPython.parallel import error
+from IPython.parallel import streamsession as ss
+from IPython.parallel import util
+
+from .asyncresult import AsyncResult, AsyncHubResult
+from IPython.parallel.apps.clusterdir import ClusterDir, ClusterDirError
+from .view import DirectView, LoadBalancedView
+
+#--------------------------------------------------------------------------
+# Decorators for Client methods
+#--------------------------------------------------------------------------
+
+@decorator
+def spin_first(f, self, *args, **kwargs):
+    """Call spin() to sync state prior to calling the method."""
+    self.spin()
+    return f(self, *args, **kwargs)
+
+@decorator
+def default_block(f, self, *args, **kwargs):
+    """Default to self.block; preserve self.block."""
+    block = kwargs.get('block',None)
+    block = self.block if block is None else block
+    saveblock = self.block
+    self.block = block
+    try:
+        ret = f(self, *args, **kwargs)
+    finally:
+        self.block = saveblock
+    return ret
+
+
+#--------------------------------------------------------------------------
+# Classes
+#--------------------------------------------------------------------------
+
+class Metadata(dict):
+    """Subclass of dict for initializing metadata values.
+    
+    Attribute access works on keys.
+    
+    These objects have a strict set of keys - errors will raise if you try
+    to add new keys.
+    """
+    def __init__(self, *args, **kwargs):
+        dict.__init__(self)
+        md = {'msg_id' : None,
+              'submitted' : None,
+              'started' : None,
+              'completed' : None,
+              'received' : None,
+              'engine_uuid' : None,
+              'engine_id' : None,
+              'follow' : None,
+              'after' : None,
+              'status' : None,
+
+              'pyin' : None,
+              'pyout' : None,
+              'pyerr' : None,
+              'stdout' : '',
+              'stderr' : '',
+            }
+        self.update(md)
+        self.update(dict(*args, **kwargs))
+    
+    def __getattr__(self, key):
+        """getattr aliased to getitem"""
+        if key in self.iterkeys():
+            return self[key]
+        else:
+            raise AttributeError(key)
+
+    def __setattr__(self, key, value):
+        """setattr aliased to setitem, with strict"""
+        if key in self.iterkeys():
+            self[key] = value
+        else:
+            raise AttributeError(key)
+    
+    def __setitem__(self, key, value):
+        """strict static key enforcement"""
+        if key in self.iterkeys():
+            dict.__setitem__(self, key, value)
+        else:
+            raise KeyError(key)
+        
+
+class Client(HasTraits):
+    """A semi-synchronous client to the IPython ZMQ cluster
+    
+    Parameters
+    ----------
+    
+    url_or_file : bytes; zmq url or path to ipcontroller-client.json
+        Connection information for the Hub's registration.  If a json connector
+        file is given, then likely no further configuration is necessary.
+        [Default: use profile]
+    profile : bytes
+        The name of the Cluster profile to be used to find connector information.
+        [Default: 'default']
+    context : zmq.Context
+        Pass an existing zmq.Context instance, otherwise the client will create its own.
+    username : bytes
+        set username to be passed to the Session object
+    debug : bool
+        flag for lots of message printing for debug purposes
+ 
+    #-------------- ssh related args ----------------
+    # These are args for configuring the ssh tunnel to be used
+    # credentials are used to forward connections over ssh to the Controller
+    # Note that the ip given in `addr` needs to be relative to sshserver
+    # The most basic case is to leave addr as pointing to localhost (127.0.0.1),
+    # and set sshserver as the same machine the Controller is on. However, 
+    # the only requirement is that sshserver is able to see the Controller
+    # (i.e. is within the same trusted network).
+    
+    sshserver : str
+        A string of the form passed to ssh, i.e. 'server.tld' or '[e-mail]:port'
+        If keyfile or password is specified, and this is not, it will default to
+        the ip given in addr.
+    sshkey : str; path to public ssh key file
+        This specifies a key to be used in ssh login, default None.
+        Regular default ssh keys will be used without specifying this argument.
+    password : str 
+        Your ssh password to sshserver. Note that if this is left None,
+        you will be prompted for it if passwordless key based login is unavailable.
+    paramiko : bool
+        flag for whether to use paramiko instead of shell ssh for tunneling.
+        [default: True on win32, False else]
+    
+    ------- exec authentication args -------
+    If even localhost is untrusted, you can have some protection against
+    unauthorized execution by using a key.  Messages are still sent
+    as cleartext, so if someone can snoop your loopback traffic this will
+    not help against malicious attacks.
+    
+    exec_key : str
+        an authentication key or file containing a key
+        default: None
+    
+    
+    Attributes
+    ----------
+    
+    ids : list of int engine IDs
+        requesting the ids attribute always synchronizes
+        the registration state. To request ids without synchronization,
+        use semi-private _ids attributes.
+    
+    history : list of msg_ids
+        a list of msg_ids, keeping track of all the execution
+        messages you have submitted in order.
+    
+    outstanding : set of msg_ids
+        a set of msg_ids that have been submitted, but whose
+        results have not yet been received.
+    
+    results : dict
+        a dict of all our results, keyed by msg_id
+    
+    block : bool
+        determines default behavior when block not specified
+        in execution methods
+    
+    Methods
+    -------
+    
+    spin
+        flushes incoming results and registration state changes
+        control methods spin, and requesting `ids` also ensures up to date
+    
+    wait
+        wait on one or more msg_ids
+    
+    execution methods
+        apply
+        legacy: execute, run
+    
+    data movement
+        push, pull, scatter, gather
+    
+    query methods
+        queue_status, get_result, purge, result_status
+    
+    control methods
+        abort, shutdown
+    
+    """
+    
+    
+    block = Bool(False)
+    outstanding = Set()
+    results = Instance('collections.defaultdict', (dict,))
+    metadata = Instance('collections.defaultdict', (Metadata,))
+    history = List()
+    debug = Bool(False)
+    profile=CUnicode('default')
+    
+    _outstanding_dict = Instance('collections.defaultdict', (set,))
+    _ids = List()
+    _connected=Bool(False)
+    _ssh=Bool(False)
+    _context = Instance('zmq.Context')
+    _config = Dict()
+    _engines=Instance(util.ReverseDict, (), {})
+    # _hub_socket=Instance('zmq.Socket')
+    _query_socket=Instance('zmq.Socket')
+    _control_socket=Instance('zmq.Socket')
+    _iopub_socket=Instance('zmq.Socket')
+    _notification_socket=Instance('zmq.Socket')
+    _mux_socket=Instance('zmq.Socket')
+    _task_socket=Instance('zmq.Socket')
+    _task_scheme=Str()
+    _closed = False
+    _ignored_control_replies=Int(0)
+    _ignored_hub_replies=Int(0)
+    
+    def __init__(self, url_or_file=None, profile='default', cluster_dir=None, ipython_dir=None,
+            context=None, username=None, debug=False, exec_key=None,
+            sshserver=None, sshkey=None, password=None, paramiko=None,
+            timeout=10
+            ):
+        super(Client, self).__init__(debug=debug, profile=profile)
+        if context is None:
+            context = zmq.Context.instance()
+        self._context = context
+            
+        
+        self._setup_cluster_dir(profile, cluster_dir, ipython_dir)
+        if self._cd is not None:
+            if url_or_file is None:
+                url_or_file = pjoin(self._cd.security_dir, 'ipcontroller-client.json')
+        assert url_or_file is not None, "I can't find enough information to connect to a hub!"\
+            " Please specify at least one of url_or_file or profile."
+        
+        try:
+            util.validate_url(url_or_file)
+        except AssertionError:
+            if not os.path.exists(url_or_file):
+                if self._cd:
+                    url_or_file = os.path.join(self._cd.security_dir, url_or_file)
+                assert os.path.exists(url_or_file), "Not a valid connection file or url: %r"%url_or_file
+            with open(url_or_file) as f:
+                cfg = json.loads(f.read())
+        else:
+            cfg = {'url':url_or_file}
+        
+        # sync defaults from args, json:
+        if sshserver:
+            cfg['ssh'] = sshserver
+        if exec_key:
+            cfg['exec_key'] = exec_key
+        exec_key = cfg['exec_key']
+        sshserver=cfg['ssh']
+        url = cfg['url']
+        location = cfg.setdefault('location', None)
+        cfg['url'] = util.disambiguate_url(cfg['url'], location)
+        url = cfg['url']
+        
+        self._config = cfg
+        
+        self._ssh = bool(sshserver or sshkey or password)
+        if self._ssh and sshserver is None:
+            # default to ssh via localhost
+            sshserver = url.split('://')[1].split(':')[0]
+        if self._ssh and password is None:
+            if tunnel.try_passwordless_ssh(sshserver, sshkey, paramiko):
+                password=False
+            else:
+                password = getpass("SSH Password for %s: "%sshserver)
+        ssh_kwargs = dict(keyfile=sshkey, password=password, paramiko=paramiko)
+        if exec_key is not None and os.path.isfile(exec_key):
+            arg = 'keyfile'
+        else:
+            arg = 'key'
+        key_arg = {arg:exec_key}
+        if username is None:
+            self.session = ss.StreamSession(**key_arg)
+        else:
+            self.session = ss.StreamSession(username, **key_arg)
+        self._query_socket = self._context.socket(zmq.XREQ)
+        self._query_socket.setsockopt(zmq.IDENTITY, self.session.session)
+        if self._ssh:
+            tunnel.tunnel_connection(self._query_socket, url, sshserver, **ssh_kwargs)
+        else:
+            self._query_socket.connect(url)
+        
+        self.session.debug = self.debug
+        
+        self._notification_handlers = {'registration_notification' : self._register_engine,
+                                    'unregistration_notification' : self._unregister_engine,
+                                    'shutdown_notification' : lambda msg: self.close(),
+                                    }
+        self._queue_handlers = {'execute_reply' : self._handle_execute_reply,
+                                'apply_reply' : self._handle_apply_reply}
+        self._connect(sshserver, ssh_kwargs, timeout)
+        
+    def __del__(self):
+        """cleanup sockets, but _not_ context."""
+        self.close()
+    
+    def _setup_cluster_dir(self, profile, cluster_dir, ipython_dir):
+        if ipython_dir is None:
+            ipython_dir = get_ipython_dir()
+        if cluster_dir is not None:
+            try:
+                self._cd = ClusterDir.find_cluster_dir(cluster_dir)
+                return
+            except ClusterDirError:
+                pass
+        elif profile is not None:
+            try:
+                self._cd = ClusterDir.find_cluster_dir_by_profile(
+                    ipython_dir, profile)
+                return
+            except ClusterDirError:
+                pass
+        self._cd = None
+    
+    def _update_engines(self, engines):
+        """Update our engines dict and _ids from a dict of the form: {id:uuid}."""
+        for k,v in engines.iteritems():
+            eid = int(k)
+            self._engines[eid] = bytes(v) # force not unicode
+            self._ids.append(eid)
+        self._ids = sorted(self._ids)
+        if sorted(self._engines.keys()) != range(len(self._engines)) and \
+                        self._task_scheme == 'pure' and self._task_socket:
+            self._stop_scheduling_tasks()
+    
+    def _stop_scheduling_tasks(self):
+        """Stop scheduling tasks because an engine has been unregistered
+        from a pure ZMQ scheduler.
+        """
+        self._task_socket.close()
+        self._task_socket = None
+        msg = "An engine has been unregistered, and we are using pure " +\
+              "ZMQ task scheduling.  Task farming will be disabled."
+        if self.outstanding:
+            msg += " If you were running tasks when this happened, " +\
+                   "some `outstanding` msg_ids may never resolve."
+        warnings.warn(msg, RuntimeWarning)
+    
+    def _build_targets(self, targets):
+        """Turn valid target IDs or 'all' into two lists:
+        (int_ids, uuids).
+        """
+        if targets is None:
+            targets = self._ids
+        elif isinstance(targets, str):
+            if targets.lower() == 'all':
+                targets = self._ids
+            else:
+                raise TypeError("%r not valid str target, must be 'all'"%(targets))
+        elif isinstance(targets, int):
+            if targets < 0:
+                targets = self.ids[targets]
+            if targets not in self.ids:
+                raise IndexError("No such engine: %i"%targets)
+            targets = [targets]
+        
+        if isinstance(targets, slice):
+            indices = range(len(self._ids))[targets]
+            ids = self.ids
+            targets = [ ids[i] for i in indices ]
+        
+        if not isinstance(targets, (tuple, list, xrange)):
+            raise TypeError("targets by int/slice/collection of ints only, not %s"%(type(targets)))
+            
+        return [self._engines[t] for t in targets], list(targets)
+    
+    def _connect(self, sshserver, ssh_kwargs, timeout):
+        """setup all our socket connections to the cluster. This is called from
+        __init__."""
+        
+        # Maybe allow reconnecting?
+        if self._connected:
+            return
+        self._connected=True
+
+        def connect_socket(s, url):
+            url = util.disambiguate_url(url, self._config['location'])
+            if self._ssh:
+                return tunnel.tunnel_connection(s, url, sshserver, **ssh_kwargs)
+            else:
+                return s.connect(url)
+            
+        self.session.send(self._query_socket, 'connection_request')
+        r,w,x = zmq.select([self._query_socket],[],[], timeout)
+        if not r:
+            raise error.TimeoutError("Hub connection request timed out")
+        idents,msg = self.session.recv(self._query_socket,mode=0)
+        if self.debug:
+            pprint(msg)
+        msg = ss.Message(msg)
+        content = msg.content
+        self._config['registration'] = dict(content)
+        if content.status == 'ok':
+            if content.mux:
+                self._mux_socket = self._context.socket(zmq.XREQ)
+                self._mux_socket.setsockopt(zmq.IDENTITY, self.session.session)
+                connect_socket(self._mux_socket, content.mux)
+            if content.task:
+                self._task_scheme, task_addr = content.task
+                self._task_socket = self._context.socket(zmq.XREQ)
+                self._task_socket.setsockopt(zmq.IDENTITY, self.session.session)
+                connect_socket(self._task_socket, task_addr)
+            if content.notification:
+                self._notification_socket = self._context.socket(zmq.SUB)
+                connect_socket(self._notification_socket, content.notification)
+                self._notification_socket.setsockopt(zmq.SUBSCRIBE, b'')
+            # if content.query:
+            #     self._query_socket = self._context.socket(zmq.XREQ)
+            #     self._query_socket.setsockopt(zmq.IDENTITY, self.session.session)
+            #     connect_socket(self._query_socket, content.query)
+            if content.control:
+                self._control_socket = self._context.socket(zmq.XREQ)
+                self._control_socket.setsockopt(zmq.IDENTITY, self.session.session)
+                connect_socket(self._control_socket, content.control)
+            if content.iopub:
+                self._iopub_socket = self._context.socket(zmq.SUB)
+                self._iopub_socket.setsockopt(zmq.SUBSCRIBE, b'')
+                self._iopub_socket.setsockopt(zmq.IDENTITY, self.session.session)
+                connect_socket(self._iopub_socket, content.iopub)
+            self._update_engines(dict(content.engines))
+        else:
+            self._connected = False
+            raise Exception("Failed to connect!")
+    
+    #--------------------------------------------------------------------------
+    # handlers and callbacks for incoming messages
+    #--------------------------------------------------------------------------
+    
+    def _unwrap_exception(self, content):
+        """unwrap exception, and remap engine_id to int."""
+        e = error.unwrap_exception(content)
+        # print e.traceback
+        if e.engine_info:
+            e_uuid = e.engine_info['engine_uuid']
+            eid = self._engines[e_uuid]
+            e.engine_info['engine_id'] = eid
+        return e
+    
+    def _extract_metadata(self, header, parent, content):
+        md = {'msg_id' : parent['msg_id'],
+              'received' : datetime.now(),
+              'engine_uuid' : header.get('engine', None),
+              'follow' : parent.get('follow', []),
+              'after' : parent.get('after', []),
+              'status' : content['status'],
+            }
+        
+        if md['engine_uuid'] is not None:
+            md['engine_id'] = self._engines.get(md['engine_uuid'], None)
+        
+        if 'date' in parent:
+            md['submitted'] = datetime.strptime(parent['date'], util.ISO8601)
+        if 'started' in header:
+            md['started'] = datetime.strptime(header['started'], util.ISO8601)
+        if 'date' in header:
+            md['completed'] = datetime.strptime(header['date'], util.ISO8601)
+        return md
+    
+    def _register_engine(self, msg):
+        """Register a new engine, and update our connection info."""
+        content = msg['content']
+        eid = content['id']
+        d = {eid : content['queue']}
+        self._update_engines(d)
+
+    def _unregister_engine(self, msg):
+        """Unregister an engine that has died."""
+        content = msg['content']
+        eid = int(content['id'])
+        if eid in self._ids:
+            self._ids.remove(eid)
+            uuid = self._engines.pop(eid)
+            
+            self._handle_stranded_msgs(eid, uuid)
+                
+        if self._task_socket and self._task_scheme == 'pure':
+            self._stop_scheduling_tasks()
+    
+    def _handle_stranded_msgs(self, eid, uuid):
+        """Handle messages known to be on an engine when the engine unregisters.
+        
+        It is possible that this will fire prematurely - that is, an engine will
+        go down after completing a result, and the client will be notified
+        of the unregistration and later receive the successful result.
+        """
+        
+        outstanding = self._outstanding_dict[uuid]
+        
+        for msg_id in list(outstanding):
+            if msg_id in self.results:
+                # we already 
+                continue
+            try:
+                raise error.EngineError("Engine %r died while running task %r"%(eid, msg_id))
+            except:
+                content = error.wrap_exception()
+            # build a fake message:
+            parent = {}
+            header = {}
+            parent['msg_id'] = msg_id
+            header['engine'] = uuid
+            header['date'] = datetime.now().strftime(util.ISO8601)
+            msg = dict(parent_header=parent, header=header, content=content)
+            self._handle_apply_reply(msg)
+    
+    def _handle_execute_reply(self, msg):
+        """Save the reply to an execute_request into our results.
+        
+        execute messages are never actually used. apply is used instead.
+        """
+        
+        parent = msg['parent_header']
+        msg_id = parent['msg_id']
+        if msg_id not in self.outstanding:
+            if msg_id in self.history:
+                print ("got stale result: %s"%msg_id)
+            else:
+                print ("got unknown result: %s"%msg_id)
+        else:
+            self.outstanding.remove(msg_id)
+        self.results[msg_id] = self._unwrap_exception(msg['content'])
+    
+    def _handle_apply_reply(self, msg):
+        """Save the reply to an apply_request into our results."""
+        parent = msg['parent_header']
+        msg_id = parent['msg_id']
+        if msg_id not in self.outstanding:
+            if msg_id in self.history:
+                print ("got stale result: %s"%msg_id)
+                print self.results[msg_id]
+                print msg
+            else:
+                print ("got unknown result: %s"%msg_id)
+        else:
+            self.outstanding.remove(msg_id)
+        content = msg['content']
+        header = msg['header']
+        
+        # construct metadata:
+        md = self.metadata[msg_id]
+        md.update(self._extract_metadata(header, parent, content))
+        # is this redundant?
+        self.metadata[msg_id] = md
+        
+        e_outstanding = self._outstanding_dict[md['engine_uuid']]
+        if msg_id in e_outstanding:
+            e_outstanding.remove(msg_id)
+        
+        # construct result:
+        if content['status'] == 'ok':
+            self.results[msg_id] = util.unserialize_object(msg['buffers'])[0]
+        elif content['status'] == 'aborted':
+            self.results[msg_id] = error.TaskAborted(msg_id)
+        elif content['status'] == 'resubmitted':
+            # TODO: handle resubmission
+            pass
+        else:
+            self.results[msg_id] = self._unwrap_exception(content)
+    
+    def _flush_notifications(self):
+        """Flush notifications of engine registrations waiting
+        in ZMQ queue."""
+        msg = self.session.recv(self._notification_socket, mode=zmq.NOBLOCK)
+        while msg is not None:
+            if self.debug:
+                pprint(msg)
+            msg = msg[-1]
+            msg_type = msg['msg_type']
+            handler = self._notification_handlers.get(msg_type, None)
+            if handler is None:
+                raise Exception("Unhandled message type: %s"%msg.msg_type)
+            else:
+                handler(msg)
+            msg = self.session.recv(self._notification_socket, mode=zmq.NOBLOCK)
+    
+    def _flush_results(self, sock):
+        """Flush task or queue results waiting in ZMQ queue."""
+        msg = self.session.recv(sock, mode=zmq.NOBLOCK)
+        while msg is not None:
+            if self.debug:
+                pprint(msg)
+            msg = msg[-1]
+            msg_type = msg['msg_type']
+            handler = self._queue_handlers.get(msg_type, None)
+            if handler is None:
+                raise Exception("Unhandled message type: %s"%msg.msg_type)
+            else:
+                handler(msg)
+            msg = self.session.recv(sock, mode=zmq.NOBLOCK)
+    
+    def _flush_control(self, sock):
+        """Flush replies from the control channel waiting
+        in the ZMQ queue.
+        
+        Currently: ignore them."""
+        if self._ignored_control_replies <= 0:
+            return
+        msg = self.session.recv(sock, mode=zmq.NOBLOCK)
+        while msg is not None:
+            self._ignored_control_replies -= 1
+            if self.debug:
+                pprint(msg)
+            msg = self.session.recv(sock, mode=zmq.NOBLOCK)
+    
+    def _flush_ignored_control(self):
+        """flush ignored control replies"""
+        while self._ignored_control_replies > 0:
+            self.session.recv(self._control_socket)
+            self._ignored_control_replies -= 1
+    
+    def _flush_ignored_hub_replies(self):
+        msg = self.session.recv(self._query_socket, mode=zmq.NOBLOCK)
+        while msg is not None:
+            msg = self.session.recv(self._query_socket, mode=zmq.NOBLOCK)
+    
+    def _flush_iopub(self, sock):
+        """Flush replies from the iopub channel waiting
+        in the ZMQ queue.
+        """
+        msg = self.session.recv(sock, mode=zmq.NOBLOCK)
+        while msg is not None:
+            if self.debug:
+                pprint(msg)
+            msg = msg[-1]
+            parent = msg['parent_header']
+            msg_id = parent['msg_id']
+            content = msg['content']
+            header = msg['header']
+            msg_type = msg['msg_type']
+            
+            # init metadata:
+            md = self.metadata[msg_id]
+            
+            if msg_type == 'stream':
+                name = content['name']
+                s = md[name] or ''
+                md[name] = s + content['data']
+            elif msg_type == 'pyerr':
+                md.update({'pyerr' : self._unwrap_exception(content)})
+            else:
+                md.update({msg_type : content['data']})
+            
+            # reduntant?
+            self.metadata[msg_id] = md
+            
+            msg = self.session.recv(sock, mode=zmq.NOBLOCK)
+    
+    #--------------------------------------------------------------------------
+    # len, getitem
+    #--------------------------------------------------------------------------
+    
+    def __len__(self):
+        """len(client) returns # of engines."""
+        return len(self.ids)
+    
+    def __getitem__(self, key):
+        """index access returns DirectView multiplexer objects
+        
+        Must be int, slice, or list/tuple/xrange of ints"""
+        if not isinstance(key, (int, slice, tuple, list, xrange)):
+            raise TypeError("key by int/slice/iterable of ints only, not %s"%(type(key)))
+        else:
+            return self.direct_view(key)
+    
+    #--------------------------------------------------------------------------
+    # Begin public methods
+    #--------------------------------------------------------------------------
+    
+    @property
+    def ids(self):
+        """Always up-to-date ids property."""
+        self._flush_notifications()
+        # always copy:
+        return list(self._ids)
+        
+    def close(self):
+        if self._closed:
+            return
+        snames = filter(lambda n: n.endswith('socket'), dir(self))
+        for socket in map(lambda name: getattr(self, name), snames):
+            if isinstance(socket, zmq.Socket) and not socket.closed:
+                socket.close()
+        self._closed = True
+    
+    def spin(self):
+        """Flush any registration notifications and execution results
+        waiting in the ZMQ queue.
+        """
+        if self._notification_socket:
+            self._flush_notifications()
+        if self._mux_socket:
+            self._flush_results(self._mux_socket)
+        if self._task_socket:
+            self._flush_results(self._task_socket)
+        if self._control_socket:
+            self._flush_control(self._control_socket)
+        if self._iopub_socket:
+            self._flush_iopub(self._iopub_socket)
+        if self._query_socket:
+            self._flush_ignored_hub_replies()
+    
+    def wait(self, jobs=None, timeout=-1):
+        """waits on one or more `jobs`, for up to `timeout` seconds.
+        
+        Parameters
+        ----------
+        
+        jobs : int, str, or list of ints and/or strs, or one or more AsyncResult objects
+                ints are indices to self.history
+                strs are msg_ids
+                default: wait on all outstanding messages
+        timeout : float
+                a time in seconds, after which to give up.
+                default is -1, which means no timeout
+        
+        Returns
+        -------
+        
+        True : when all msg_ids are done
+        False : timeout reached, some msg_ids still outstanding
+        """
+        tic = time.time()
+        if jobs is None:
+            theids = self.outstanding
+        else:
+            if isinstance(jobs, (int, str, AsyncResult)):
+                jobs = [jobs]
+            theids = set()
+            for job in jobs:
+                if isinstance(job, int):
+                    # index access
+                    job = self.history[job]
+                elif isinstance(job, AsyncResult):
+                    map(theids.add, job.msg_ids)
+                    continue
+                theids.add(job)
+        if not theids.intersection(self.outstanding):
+            return True
+        self.spin()
+        while theids.intersection(self.outstanding):
+            if timeout >= 0 and ( time.time()-tic ) > timeout:
+                break
+            time.sleep(1e-3)
+            self.spin()
+        return len(theids.intersection(self.outstanding)) == 0
+    
+    #--------------------------------------------------------------------------
+    # Control methods
+    #--------------------------------------------------------------------------
+    
+    @spin_first
+    @default_block
+    def clear(self, targets=None, block=None):
+        """Clear the namespace in target(s)."""
+        targets = self._build_targets(targets)[0]
+        for t in targets:
+            self.session.send(self._control_socket, 'clear_request', content={}, ident=t)
+        error = False
+        if self.block:
+            self._flush_ignored_control()
+            for i in range(len(targets)):
+                idents,msg = self.session.recv(self._control_socket,0)
+                if self.debug:
+                    pprint(msg)
+                if msg['content']['status'] != 'ok':
+                    error = self._unwrap_exception(msg['content'])
+        else:
+            self._ignored_control_replies += len(targets)
+        if error:
+            raise error
+        
+    
+    @spin_first
+    @default_block
+    def abort(self, jobs=None, targets=None, block=None):
+        """Abort specific jobs from the execution queues of target(s).
+        
+        This is a mechanism to prevent jobs that have already been submitted
+        from executing.
+        
+        Parameters
+        ----------
+        
+        jobs : msg_id, list of msg_ids, or AsyncResult
+            The jobs to be aborted
+        
+        
+        """
+        targets = self._build_targets(targets)[0]
+        msg_ids = []
+        if isinstance(jobs, (basestring,AsyncResult)):
+            jobs = [jobs]
+        bad_ids = filter(lambda obj: not isinstance(obj, (basestring, AsyncResult)), jobs)
+        if bad_ids:
+            raise TypeError("Invalid msg_id type %r, expected str or AsyncResult"%bad_ids[0])
+        for j in jobs:
+            if isinstance(j, AsyncResult):
+                msg_ids.extend(j.msg_ids)
+            else:
+                msg_ids.append(j)
+        content = dict(msg_ids=msg_ids)
+        for t in targets:
+            self.session.send(self._control_socket, 'abort_request', 
+                    content=content, ident=t)
+        error = False
+        if self.block:
+            self._flush_ignored_control()
+            for i in range(len(targets)):
+                idents,msg = self.session.recv(self._control_socket,0)
+                if self.debug:
+                    pprint(msg)
+                if msg['content']['status'] != 'ok':
+                    error = self._unwrap_exception(msg['content'])
+        else:
+            self._ignored_control_replies += len(targets)
+        if error:
+            raise error
+    
+    @spin_first
+    @default_block
+    def shutdown(self, targets=None, restart=False, hub=False, block=None):
+        """Terminates one or more engine processes, optionally including the hub."""
+        if hub:
+            targets = 'all'
+        targets = self._build_targets(targets)[0]
+        for t in targets:
+            self.session.send(self._control_socket, 'shutdown_request', 
+                        content={'restart':restart},ident=t)
+        error = False
+        if block or hub:
+            self._flush_ignored_control()
+            for i in range(len(targets)):
+                idents,msg = self.session.recv(self._control_socket, 0)
+                if self.debug:
+                    pprint(msg)
+                if msg['content']['status'] != 'ok':
+                    error = self._unwrap_exception(msg['content'])
+        else:
+            self._ignored_control_replies += len(targets)
+        
+        if hub:
+            time.sleep(0.25)
+            self.session.send(self._query_socket, 'shutdown_request')
+            idents,msg = self.session.recv(self._query_socket, 0)
+            if self.debug:
+                pprint(msg)
+            if msg['content']['status'] != 'ok':
+                error = self._unwrap_exception(msg['content'])
+        
+        if error:
+            raise error
+    
+    #--------------------------------------------------------------------------
+    # Execution methods
+    #--------------------------------------------------------------------------
+    
+    @default_block
+    def _execute(self, code, targets='all', block=None):
+        """Executes `code` on `targets` in blocking or nonblocking manner.
+        
+        ``execute`` is always `bound` (affects engine namespace)
+        
+        Parameters
+        ----------
+        
+        code : str
+                the code string to be executed
+        targets : int/str/list of ints/strs
+                the engines on which to execute
+                default : all
+        block : bool
+                whether or not to wait until done to return
+                default: self.block
+        """
+        return self[targets].execute(code, block=block)
+    
+    def _maybe_raise(self, result):
+        """wrapper for maybe raising an exception if apply failed."""
+        if isinstance(result, error.RemoteError):
+            raise result
+        
+        return result
+    
+    def send_apply_message(self, socket, f, args=None, kwargs=None, subheader=None, track=False,
+                            ident=None):
+        """construct and send an apply message via a socket.
+        
+        This is the principal method with which all engine execution is performed by views.
+        """
+                            
+        assert not self._closed, "cannot use me anymore, I'm closed!"
+        # defaults:
+        args = args if args is not None else []
+        kwargs = kwargs if kwargs is not None else {}
+        subheader = subheader if subheader is not None else {}
+        
+        # validate arguments
+        if not callable(f):
+            raise TypeError("f must be callable, not %s"%type(f))
+        if not isinstance(args, (tuple, list)):
+            raise TypeError("args must be tuple or list, not %s"%type(args))
+        if not isinstance(kwargs, dict):
+            raise TypeError("kwargs must be dict, not %s"%type(kwargs))
+        if not isinstance(subheader, dict):
+            raise TypeError("subheader must be dict, not %s"%type(subheader))
+        
+        if not self._ids:
+            # flush notification socket if no engines yet
+            any_ids = self.ids
+            if not any_ids:
+                raise error.NoEnginesRegistered("Can't execute without any connected engines.")
+                # enforce types of f,args,kwargs
+        
+        bufs = util.pack_apply_message(f,args,kwargs)
+        
+        msg = self.session.send(socket, "apply_request", buffers=bufs, ident=ident,
+                            subheader=subheader, track=track)
+        
+        msg_id = msg['msg_id']
+        self.outstanding.add(msg_id)
+        if ident:
+            # possibly routed to a specific engine
+            if isinstance(ident, list):
+                ident = ident[-1]
+            if ident in self._engines.values():
+                # save for later, in case of engine death
+                self._outstanding_dict[ident].add(msg_id)
+        self.history.append(msg_id)
+        self.metadata[msg_id]['submitted'] = datetime.now()
+        
+        return msg
+
+    #--------------------------------------------------------------------------
+    # construct a View object
+    #--------------------------------------------------------------------------
+    
+    def load_balanced_view(self, targets=None):
+        """construct a DirectView object.
+        
+        If no arguments are specified, create a LoadBalancedView
+        using all engines.
+        
+        Parameters
+        ----------
+        
+        targets: list,slice,int,etc. [default: use all engines]
+            The subset of engines across which to load-balance
+        """
+        if targets is not None:
+            targets = self._build_targets(targets)[1]
+        return LoadBalancedView(client=self, socket=self._task_socket, targets=targets)
+    
+    def direct_view(self, targets='all'):
+        """construct a DirectView object.
+        
+        If no targets are specified, create a DirectView
+        using all engines.
+        
+        Parameters
+        ----------
+        
+        targets: list,slice,int,etc. [default: use all engines]
+            The engines to use for the View
+        """
+        single = isinstance(targets, int)
+        targets = self._build_targets(targets)[1]
+        if single:
+            targets = targets[0]
+        return DirectView(client=self, socket=self._mux_socket, targets=targets)
+    
+    #--------------------------------------------------------------------------
+    # Data movement (TO BE REMOVED)
+    #--------------------------------------------------------------------------
+    
+    @default_block
+    def _push(self, ns, targets='all', block=None, track=False):
+        """Push the contents of `ns` into the namespace on `target`"""
+        if not isinstance(ns, dict):
+            raise TypeError("Must be a dict, not %s"%type(ns))
+        result = self.apply(util._push, kwargs=ns, targets=targets, block=block, bound=True, balanced=False, track=track)
+        if not block:
+            return result
+    
+    @default_block
+    def _pull(self, keys, targets='all', block=None):
+        """Pull objects from `target`'s namespace by `keys`"""
+        if isinstance(keys, basestring):
+            pass
+        elif isinstance(keys, (list,tuple,set)):
+            for key in keys:
+                if not isinstance(key, basestring):
+                    raise TypeError("keys must be str, not type %r"%type(key))
+        else:
+            raise TypeError("keys must be strs, not %r"%keys)
+        result = self.apply(util._pull, (keys,), targets=targets, block=block, bound=True, balanced=False)
+        return result
+    
+    #--------------------------------------------------------------------------
+    # Query methods
+    #--------------------------------------------------------------------------
+    
+    @spin_first
+    @default_block
+    def get_result(self, indices_or_msg_ids=None, block=None):
+        """Retrieve a result by msg_id or history index, wrapped in an AsyncResult object.
+        
+        If the client already has the results, no request to the Hub will be made.
+        
+        This is a convenient way to construct AsyncResult objects, which are wrappers
+        that include metadata about execution, and allow for awaiting results that
+        were not submitted by this Client.
+        
+        It can also be a convenient way to retrieve the metadata associated with
+        blocking execution, since it always retrieves
+        
+        Examples
+        --------
+        ::
+        
+            In [10]: r = client.apply()
+        
+        Parameters
+        ----------
+        
+        indices_or_msg_ids : integer history index, str msg_id, or list of either
+            The indices or msg_ids of indices to be retrieved
+        
+        block : bool
+            Whether to wait for the result to be done
+        
+        Returns
+        -------
+        
+        AsyncResult
+            A single AsyncResult object will always be returned.
+        
+        AsyncHubResult
+            A subclass of AsyncResult that retrieves results from the Hub
+        
+        """
+        if indices_or_msg_ids is None:
+            indices_or_msg_ids = -1
+        
+        if not isinstance(indices_or_msg_ids, (list,tuple)):
+            indices_or_msg_ids = [indices_or_msg_ids]
+        
+        theids = []
+        for id in indices_or_msg_ids:
+            if isinstance(id, int):
+                id = self.history[id]
+            if not isinstance(id, str):
+                raise TypeError("indices must be str or int, not %r"%id)
+            theids.append(id)
+        
+        local_ids = filter(lambda msg_id: msg_id in self.history or msg_id in self.results, theids)
+        remote_ids = filter(lambda msg_id: msg_id not in local_ids, theids)
+        
+        if remote_ids:
+            ar = AsyncHubResult(self, msg_ids=theids)
+        else:
+            ar = AsyncResult(self, msg_ids=theids)
+        
+        if block:
+            ar.wait()
+        
+        return ar
+    
+    @spin_first
+    def result_status(self, msg_ids, status_only=True):
+        """Check on the status of the result(s) of the apply request with `msg_ids`.
+        
+        If status_only is False, then the actual results will be retrieved, else
+        only the status of the results will be checked.
+        
+        Parameters
+        ----------
+        
+        msg_ids : list of msg_ids
+            if int:
+                Passed as index to self.history for convenience.
+        status_only : bool (default: True)
+            if False:
+                Retrieve the actual results of completed tasks.
+        
+        Returns
+        -------
+        
+        results : dict
+            There will always be the keys 'pending' and 'completed', which will
+            be lists of msg_ids that are incomplete or complete. If `status_only`
+            is False, then completed results will be keyed by their `msg_id`.
+        """
+        if not isinstance(msg_ids, (list,tuple)):
+            msg_ids = [msg_ids]
+            
+        theids = []
+        for msg_id in msg_ids:
+            if isinstance(msg_id, int):
+                msg_id = self.history[msg_id]
+            if not isinstance(msg_id, basestring):
+                raise TypeError("msg_ids must be str, not %r"%msg_id)
+            theids.append(msg_id)
+        
+        completed = []
+        local_results = {}
+        
+        # comment this block out to temporarily disable local shortcut:
+        for msg_id in theids:
+            if msg_id in self.results:
+                completed.append(msg_id)
+                local_results[msg_id] = self.results[msg_id]
+                theids.remove(msg_id)
+        
+        if theids: # some not locally cached
+            content = dict(msg_ids=theids, status_only=status_only)
+            msg = self.session.send(self._query_socket, "result_request", content=content)
+            zmq.select([self._query_socket], [], [])
+            idents,msg = self.session.recv(self._query_socket, zmq.NOBLOCK)
+            if self.debug:
+                pprint(msg)
+            content = msg['content']
+            if content['status'] != 'ok':
+                raise self._unwrap_exception(content)
+            buffers = msg['buffers']
+        else:
+            content = dict(completed=[],pending=[])
+        
+        content['completed'].extend(completed)
+        
+        if status_only:
+            return content
+        
+        failures = []
+        # load cached results into result:
+        content.update(local_results)
+        # update cache with results:
+        for msg_id in sorted(theids):
+            if msg_id in content['completed']:
+                rec = content[msg_id]
+                parent = rec['header']
+                header = rec['result_header']
+                rcontent = rec['result_content']
+                iodict = rec['io']
+                if isinstance(rcontent, str):
+                    rcontent = self.session.unpack(rcontent)
+                
+                md = self.metadata[msg_id]
+                md.update(self._extract_metadata(header, parent, rcontent))
+                md.update(iodict)
+                
+                if rcontent['status'] == 'ok':
+                    res,buffers = util.unserialize_object(buffers)
+                else:
+                    print rcontent
+                    res = self._unwrap_exception(rcontent)
+                    failures.append(res)
+                
+                self.results[msg_id] = res
+                content[msg_id] = res
+        
+        if len(theids) == 1 and failures:
+                raise failures[0]
+        
+        error.collect_exceptions(failures, "result_status")
+        return content
+
+    @spin_first
+    def queue_status(self, targets='all', verbose=False):
+        """Fetch the status of engine queues.
+        
+        Parameters
+        ----------
+        
+        targets : int/str/list of ints/strs
+                the engines whose states are to be queried.
+                default : all
+        verbose : bool
+                Whether to return lengths only, or lists of ids for each element
+        """
+        engine_ids = self._build_targets(targets)[1]
+        content = dict(targets=engine_ids, verbose=verbose)
+        self.session.send(self._query_socket, "queue_request", content=content)
+        idents,msg = self.session.recv(self._query_socket, 0)
+        if self.debug:
+            pprint(msg)
+        content = msg['content']
+        status = content.pop('status')
+        if status != 'ok':
+            raise self._unwrap_exception(content)
+        content = util.rekey(content)
+        if isinstance(targets, int):
+            return content[targets]
+        else:
+            return content
+        
+    @spin_first
+    def purge_results(self, jobs=[], targets=[]):
+        """Tell the Hub to forget results.
+        
+        Individual results can be purged by msg_id, or the entire
+        history of specific targets can be purged.
+        
+        Parameters
+        ----------
+        
+        jobs : str or list of str or AsyncResult objects
+                the msg_ids whose results should be forgotten.
+        targets : int/str/list of ints/strs
+                The targets, by uuid or int_id, whose entire history is to be purged.
+                Use `targets='all'` to scrub everything from the Hub's memory.
+                
+                default : None
+        """
+        if not targets and not jobs:
+            raise ValueError("Must specify at least one of `targets` and `jobs`")
+        if targets:
+            targets = self._build_targets(targets)[1]
+        
+        # construct msg_ids from jobs
+        msg_ids = []
+        if isinstance(jobs, (basestring,AsyncResult)):
+            jobs = [jobs]
+        bad_ids = filter(lambda obj: not isinstance(obj, (basestring, AsyncResult)), jobs)
+        if bad_ids:
+            raise TypeError("Invalid msg_id type %r, expected str or AsyncResult"%bad_ids[0])
+        for j in jobs:
+            if isinstance(j, AsyncResult):
+                msg_ids.extend(j.msg_ids)
+            else:
+                msg_ids.append(j)
+        
+        content = dict(targets=targets, msg_ids=msg_ids)
+        self.session.send(self._query_socket, "purge_request", content=content)
+        idents, msg = self.session.recv(self._query_socket, 0)
+        if self.debug:
+            pprint(msg)
+        content = msg['content']
+        if content['status'] != 'ok':
+            raise self._unwrap_exception(content)
+
+
+__all__ = [ 'Client' ]

IPython/parallel/client/map.py

@@ -0,0 +1,158 b''
+# encoding: utf-8
+
+"""Classes used in scattering and gathering sequences.
+
+Scattering consists of partitioning a sequence and sending the various
+pieces to individual nodes in a cluster.
+"""
+
+__docformat__ = "restructuredtext en"
+
+#-------------------------------------------------------------------------------
+#  Copyright (C) 2008  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-------------------------------------------------------------------------------
+
+#-------------------------------------------------------------------------------
+# Imports
+#-------------------------------------------------------------------------------
+
+import types
+
+from IPython.utils.data import flatten as utils_flatten
+
+#-------------------------------------------------------------------------------
+# Figure out which array packages are present and their array types
+#-------------------------------------------------------------------------------
+
+arrayModules = []
+try:
+    import Numeric
+except ImportError:
+    pass
+else:
+    arrayModules.append({'module':Numeric, 'type':Numeric.arraytype})
+try:
+    import numpy
+except ImportError:
+    pass
+else:
+    arrayModules.append({'module':numpy, 'type':numpy.ndarray})
+try:
+    import numarray
+except ImportError:
+    pass
+else:
+    arrayModules.append({'module':numarray,
+        'type':numarray.numarraycore.NumArray})
+
+class Map:
+    """A class for partitioning a sequence using a map."""
+            
+    def getPartition(self, seq, p, q):
+        """Returns the pth partition of q partitions of seq."""
+        
+        # Test for error conditions here
+        if p<0 or p>=q:
+          print "No partition exists."
+          return
+          
+        remainder = len(seq)%q
+        basesize = len(seq)/q
+        hi = []
+        lo = []
+        for n in range(q):
+            if n < remainder:
+                lo.append(n * (basesize + 1))
+                hi.append(lo[-1] + basesize + 1)
+            else:
+                lo.append(n*basesize + remainder)
+                hi.append(lo[-1] + basesize)
+
+        
+        result = seq[lo[p]:hi[p]]
+        return result
+           
+    def joinPartitions(self, listOfPartitions):
+        return self.concatenate(listOfPartitions)
+                    
+    def concatenate(self, listOfPartitions):
+        testObject = listOfPartitions[0]
+        # First see if we have a known array type
+        for m in arrayModules:
+            #print m
+            if isinstance(testObject, m['type']):
+                return m['module'].concatenate(listOfPartitions)
+        # Next try for Python sequence types
+        if isinstance(testObject, (types.ListType, types.TupleType)):
+            return utils_flatten(listOfPartitions)
+        # If we have scalars, just return listOfPartitions
+        return listOfPartitions
+
+class RoundRobinMap(Map):
+    """Partitions a sequence in a roun robin fashion.
+    
+    This currently does not work!
+    """
+
+    def getPartition(self, seq, p, q):
+        # if not isinstance(seq,(list,tuple)):
+        #     raise NotImplementedError("cannot RR partition type %s"%type(seq))
+        return seq[p:len(seq):q]
+        #result = []
+        #for i in range(p,len(seq),q):
+        #    result.append(seq[i])
+        #return result
+
+    def joinPartitions(self, listOfPartitions):
+        testObject = listOfPartitions[0]
+        # First see if we have a known array type
+        for m in arrayModules:
+            #print m
+            if isinstance(testObject, m['type']):
+                return self.flatten_array(m['type'], listOfPartitions)
+        if isinstance(testObject, (types.ListType, types.TupleType)):
+            return self.flatten_list(listOfPartitions)
+        return listOfPartitions
+    
+    def flatten_array(self, klass, listOfPartitions):
+        test = listOfPartitions[0]
+        shape = list(test.shape)
+        shape[0] = sum([ p.shape[0] for p in listOfPartitions])
+        A = klass(shape)
+        N = shape[0]
+        q = len(listOfPartitions)
+        for p,part in enumerate(listOfPartitions):
+            A[p:N:q] = part
+        return A
+    
+    def flatten_list(self, listOfPartitions):
+        flat = []
+        for i in range(len(listOfPartitions[0])):
+            flat.extend([ part[i] for part in listOfPartitions if len(part) > i ])
+        return flat
+        #lengths = [len(x) for x in listOfPartitions]
+        #maxPartitionLength = len(listOfPartitions[0])
+        #numberOfPartitions = len(listOfPartitions)
+        #concat = self.concatenate(listOfPartitions)
+        #totalLength = len(concat)
+        #result = []
+        #for i in range(maxPartitionLength):
+        #    result.append(concat[i:totalLength:maxPartitionLength])
+        # return self.concatenate(listOfPartitions)
+
+def mappable(obj):
+    """return whether an object is mappable or not."""
+    if isinstance(obj, (tuple,list)):
+        return True
+    for m in arrayModules:
+        if isinstance(obj,m['type']):
+            return True
+    return False
+
+dists = {'b':Map,'r':RoundRobinMap}
+
+    
+    

IPython/parallel/client/remotefunction.py

@@ -0,0 +1,200 b''
+"""Remote Functions and decorators for Views."""
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2010  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+import warnings
+
+from IPython.testing import decorators as testdec
+
+from . import map as Map
+from .asyncresult import AsyncMapResult
+
+#-----------------------------------------------------------------------------
+# Decorators
+#-----------------------------------------------------------------------------
+
[e-mail]_doctest
+def remote(view, block=None, **flags):
+    """Turn a function into a remote function.
+    
+    This method can be used for map:
+    
+    In [1]: @remote(view,block=True)
+       ...: def func(a):
+       ...:    pass
+    """
+    
+    def remote_function(f):
+        return RemoteFunction(view, f, block=block, **flags)
+    return remote_function
+
[e-mail]_doctest
+def parallel(view, dist='b', block=None, **flags):
+    """Turn a function into a parallel remote function.
+    
+    This method can be used for map:
+    
+    In [1]: @parallel(view, block=True)
+       ...: def func(a):
+       ...:    pass
+    """
+    
+    def parallel_function(f):
+        return ParallelFunction(view, f, dist=dist, block=block, **flags)
+    return parallel_function
+
+#--------------------------------------------------------------------------
+# Classes
+#--------------------------------------------------------------------------
+
+class RemoteFunction(object):
+    """Turn an existing function into a remote function.
+    
+    Parameters
+    ----------
+    
+    view : View instance
+        The view to be used for execution
+    f : callable
+        The function to be wrapped into a remote function
+    block : bool [default: None]
+        Whether to wait for results or not.  The default behavior is
+        to use the current `block` attribute of `view`
+    
+    **flags : remaining kwargs are passed to View.temp_flags
+    """
+    
+    view = None # the remote connection
+    func = None # the wrapped function
+    block = None # whether to block
+    flags = None # dict of extra kwargs for temp_flags
+    
+    def __init__(self, view, f, block=None, **flags):
+        self.view = view
+        self.func = f
+        self.block=block
+        self.flags=flags
+    
+    def __call__(self, *args, **kwargs):
+        block = self.view.block if self.block is None else self.block
+        with self.view.temp_flags(block=block, **self.flags):
+            return self.view.apply(self.func, *args, **kwargs)
+    
+
+class ParallelFunction(RemoteFunction):
+    """Class for mapping a function to sequences.
+    
+    This will distribute the sequences according the a mapper, and call
+    the function on each sub-sequence.  If called via map, then the function
+    will be called once on each element, rather that each sub-sequence.
+    
+    Parameters
+    ----------
+    
+    view : View instance
+        The view to be used for execution
+    f : callable
+        The function to be wrapped into a remote function
+    dist : str [default: 'b']
+        The key for which mapObject to use to distribute sequences
+        options are:
+          * 'b' : use contiguous chunks in order
+          * 'r' : use round-robin striping
+    block : bool [default: None]
+        Whether to wait for results or not.  The default behavior is
+        to use the current `block` attribute of `view`
+    chunksize : int or None
+        The size of chunk to use when breaking up sequences in a load-balanced manner
+    **flags : remaining kwargs are passed to View.temp_flags
+    """
+    
+    chunksize=None
+    mapObject=None
+    
+    def __init__(self, view, f, dist='b', block=None, chunksize=None, **flags):
+        super(ParallelFunction, self).__init__(view, f, block=block, **flags)
+        self.chunksize = chunksize
+        
+        mapClass = Map.dists[dist]
+        self.mapObject = mapClass()
+    
+    def __call__(self, *sequences):
+        # check that the length of sequences match
+        len_0 = len(sequences[0])
+        for s in sequences:
+            if len(s)!=len_0:
+                msg = 'all sequences must have equal length, but %i!=%i'%(len_0,len(s))
+                raise ValueError(msg)
+        balanced = 'Balanced' in self.view.__class__.__name__
+        if balanced:
+            if self.chunksize:
+                nparts = len_0/self.chunksize + int(len_0%self.chunksize > 0)
+            else:
+                nparts = len_0
+            targets = [None]*nparts
+        else:
+            if self.chunksize:
+                warnings.warn("`chunksize` is ignored unless load balancing", UserWarning)
+            # multiplexed:
+            targets = self.view.targets
+            nparts = len(targets)
+        
+        msg_ids = []
+        # my_f = lambda *a: map(self.func, *a)
+        client = self.view.client
+        for index, t in enumerate(targets):
+            args = []
+            for seq in sequences:
+                part = self.mapObject.getPartition(seq, index, nparts)
+                if len(part) == 0:
+                    continue
+                else:
+                    args.append(part)
+            if not args:
+                continue
+            
+            # print (args)
+            if hasattr(self, '_map'):
+                f = map
+                args = [self.func]+args
+            else:
+                f=self.func
+            
+            view = self.view if balanced else client[t]
+            with view.temp_flags(block=False, **self.flags):
+                ar = view.apply(f, *args)
+            
+            msg_ids.append(ar.msg_ids[0])
+        
+        r = AsyncMapResult(self.view.client, msg_ids, self.mapObject, fname=self.func.__name__)
+        
+        if self.block:
+            try:
+                return r.get()
+            except KeyboardInterrupt:
+                return r
+        else:
+            return r
+    
+    def map(self, *sequences):
+        """call a function on each element of a sequence remotely. 
+        This should behave very much like the builtin map, but return an AsyncMapResult
+        if self.block is False.
+        """
+        # set _map as a flag for use inside self.__call__
+        self._map = True
+        try:
+            ret = self.__call__(*sequences)
+        finally:
+            del self._map
+        return ret
+
+__all__ = ['remote', 'parallel', 'RemoteFunction', 'ParallelFunction']
  No newline at end of file

IPython/parallel/client/view.py

@@ -0,0 +1,1033 b''
+"""Views of remote engines."""
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2010  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+import imp
+import sys
+import warnings
+from contextlib import contextmanager
+from types import ModuleType
+
+import zmq
+
+from IPython.testing import decorators as testdec
+from IPython.utils.traitlets import HasTraits, Any, Bool, List, Dict, Set, Int, Instance, CFloat
+
+from IPython.external.decorator import decorator
+
+from IPython.parallel import util
+from IPython.parallel.controller.dependency import Dependency, dependent
+
+from . import map as Map
+from .asyncresult import AsyncResult, AsyncMapResult
+from .remotefunction import ParallelFunction, parallel, remote
+
+#-----------------------------------------------------------------------------
+# Decorators
+#-----------------------------------------------------------------------------
+
+@decorator
+def save_ids(f, self, *args, **kwargs):
+    """Keep our history and outstanding attributes up to date after a method call."""
+    n_previous = len(self.client.history)
+    try:
+        ret = f(self, *args, **kwargs)
+    finally:
+        nmsgs = len(self.client.history) - n_previous
+        msg_ids = self.client.history[-nmsgs:]
+        self.history.extend(msg_ids)
+        map(self.outstanding.add, msg_ids)
+    return ret
+
+@decorator
+def sync_results(f, self, *args, **kwargs):
+    """sync relevant results from self.client to our results attribute."""
+    ret = f(self, *args, **kwargs)
+    delta = self.outstanding.difference(self.client.outstanding)
+    completed = self.outstanding.intersection(delta)
+    self.outstanding = self.outstanding.difference(completed)
+    for msg_id in completed:
+        self.results[msg_id] = self.client.results[msg_id]
+    return ret
+
+@decorator
+def spin_after(f, self, *args, **kwargs):
+    """call spin after the method."""
+    ret = f(self, *args, **kwargs)
+    self.spin()
+    return ret
+
+#-----------------------------------------------------------------------------
+# Classes
+#-----------------------------------------------------------------------------
+
[e-mail]_doctest
+class View(HasTraits):
+    """Base View class for more convenint apply(f,*args,**kwargs) syntax via attributes.
+    
+    Don't use this class, use subclasses.
+    
+    Methods
+    -------
+    
+    spin
+        flushes incoming results and registration state changes
+        control methods spin, and requesting `ids` also ensures up to date
+    
+    wait
+        wait on one or more msg_ids
+    
+    execution methods
+        apply
+        legacy: execute, run
+    
+    data movement
+        push, pull, scatter, gather
+    
+    query methods
+        get_result, queue_status, purge_results, result_status
+    
+    control methods
+        abort, shutdown
+    
+    """
+    # flags
+    block=Bool(False)
+    track=Bool(True)
+    targets = Any()
+    
+    history=List()
+    outstanding = Set()
+    results = Dict()
+    client = Instance('IPython.parallel.Client')
+    
+    _socket = Instance('zmq.Socket')
+    _flag_names = List(['targets', 'block', 'track'])
+    _targets = Any()
+    _idents = Any()
+    
+    def __init__(self, client=None, socket=None, **flags):
+        super(View, self).__init__(client=client, _socket=socket)
+        self.block = client.block
+        
+        self.set_flags(**flags)
+        
+        assert not self.__class__ is View, "Don't use base View objects, use subclasses"
+        
+
+    def __repr__(self):
+        strtargets = str(self.targets)
+        if len(strtargets) > 16:
+            strtargets = strtargets[:12]+'...]'
+        return "<%s %s>"%(self.__class__.__name__, strtargets)
+
+    def set_flags(self, **kwargs):
+        """set my attribute flags by keyword.
+        
+        Views determine behavior with a few attributes (`block`, `track`, etc.).
+        These attributes can be set all at once by name with this method.
+        
+        Parameters
+        ----------
+        
+        block : bool
+            whether to wait for results
+        track : bool
+            whether to create a MessageTracker to allow the user to 
+            safely edit after arrays and buffers during non-copying
+            sends.
+        """
+        for name, value in kwargs.iteritems():
+            if name not in self._flag_names:
+                raise KeyError("Invalid name: %r"%name)
+            else:
+                setattr(self, name, value)
+    
+    @contextmanager
+    def temp_flags(self, **kwargs):
+        """temporarily set flags, for use in `with` statements.
+        
+        See set_flags for permanent setting of flags
+        
+        Examples
+        --------
+        
+        >>> view.track=False
+        ...
+        >>> with view.temp_flags(track=True):
+        ...    ar = view.apply(dostuff, my_big_array)
+        ...    ar.tracker.wait() # wait for send to finish
+        >>> view.track
+        False
+        
+        """
+        # preflight: save flags, and set temporaries
+        saved_flags = {}
+        for f in self._flag_names:
+            saved_flags[f] = getattr(self, f)
+        self.set_flags(**kwargs)
+        # yield to the with-statement block
+        try:
+            yield
+        finally:
+            # postflight: restore saved flags
+            self.set_flags(**saved_flags)
+        
+        
+    #----------------------------------------------------------------
+    # apply
+    #----------------------------------------------------------------
+    
+    @sync_results
+    @save_ids
+    def _really_apply(self, f, args, kwargs, block=None, **options):
+        """wrapper for client.send_apply_message"""
+        raise NotImplementedError("Implement in subclasses")
+    
+    def apply(self, f, *args, **kwargs):
+        """calls f(*args, **kwargs) on remote engines, returning the result.
+        
+        This method sets all apply flags via this View's attributes.
+        
+        if self.block is False:
+            returns AsyncResult
+        else:
+            returns actual result of f(*args, **kwargs)
+        """
+        return self._really_apply(f, args, kwargs)
+
+    def apply_async(self, f, *args, **kwargs):
+        """calls f(*args, **kwargs) on remote engines in a nonblocking manner.
+        
+        returns AsyncResult
+        """
+        return self._really_apply(f, args, kwargs, block=False)
+
+    @spin_after
+    def apply_sync(self, f, *args, **kwargs):
+        """calls f(*args, **kwargs) on remote engines in a blocking manner,
+         returning the result.
+        
+        returns: actual result of f(*args, **kwargs)
+        """
+        return self._really_apply(f, args, kwargs, block=True)
+
+    #----------------------------------------------------------------
+    # wrappers for client and control methods
+    #----------------------------------------------------------------
+    @sync_results
+    def spin(self):
+        """spin the client, and sync"""
+        self.client.spin()
+    
+    @sync_results
+    def wait(self, jobs=None, timeout=-1):
+        """waits on one or more `jobs`, for up to `timeout` seconds.
+        
+        Parameters
+        ----------
+        
+        jobs : int, str, or list of ints and/or strs, or one or more AsyncResult objects
+                ints are indices to self.history
+                strs are msg_ids
+                default: wait on all outstanding messages
+        timeout : float
+                a time in seconds, after which to give up.
+                default is -1, which means no timeout
+        
+        Returns
+        -------
+        
+        True : when all msg_ids are done
+        False : timeout reached, some msg_ids still outstanding
+        """
+        if jobs is None:
+            jobs = self.history
+        return self.client.wait(jobs, timeout)
+    
+    def abort(self, jobs=None, targets=None, block=None):
+        """Abort jobs on my engines.
+        
+        Parameters
+        ----------
+        
+        jobs : None, str, list of strs, optional
+            if None: abort all jobs.
+            else: abort specific msg_id(s).
+        """
+        block = block if block is not None else self.block
+        targets = targets if targets is not None else self.targets
+        return self.client.abort(jobs=jobs, targets=targets, block=block)
+
+    def queue_status(self, targets=None, verbose=False):
+        """Fetch the Queue status of my engines"""
+        targets = targets if targets is not None else self.targets
+        return self.client.queue_status(targets=targets, verbose=verbose)
+    
+    def purge_results(self, jobs=[], targets=[]):
+        """Instruct the controller to forget specific results."""
+        if targets is None or targets == 'all':
+            targets = self.targets
+        return self.client.purge_results(jobs=jobs, targets=targets)
+    
+    def shutdown(self, targets=None, restart=False, hub=False, block=None):
+        """Terminates one or more engine processes, optionally including the hub.
+        """
+        block = self.block if block is None else block
+        if targets is None or targets == 'all':
+            targets = self.targets
+        return self.client.shutdown(targets=targets, restart=restart, hub=hub, block=block)
+    
+    @spin_after
+    def get_result(self, indices_or_msg_ids=None):
+        """return one or more results, specified by history index or msg_id.
+        
+        See client.get_result for details.
+        
+        """
+        
+        if indices_or_msg_ids is None:
+            indices_or_msg_ids = -1
+        if isinstance(indices_or_msg_ids, int):
+            indices_or_msg_ids = self.history[indices_or_msg_ids]
+        elif isinstance(indices_or_msg_ids, (list,tuple,set)):
+            indices_or_msg_ids = list(indices_or_msg_ids)
+            for i,index in enumerate(indices_or_msg_ids):
+                if isinstance(index, int):
+                    indices_or_msg_ids[i] = self.history[index]
+        return self.client.get_result(indices_or_msg_ids)
+    
+    #-------------------------------------------------------------------
+    # Map
+    #-------------------------------------------------------------------
+    
+    def map(self, f, *sequences, **kwargs):
+        """override in subclasses"""
+        raise NotImplementedError
+    
+    def map_async(self, f, *sequences, **kwargs):
+        """Parallel version of builtin `map`, using this view's engines.
+        
+        This is equivalent to map(...block=False)
+        
+        See `self.map` for details.
+        """
+        if 'block' in kwargs:
+            raise TypeError("map_async doesn't take a `block` keyword argument.")
+        kwargs['block'] = False
+        return self.map(f,*sequences,**kwargs)
+    
+    def map_sync(self, f, *sequences, **kwargs):
+        """Parallel version of builtin `map`, using this view's engines.
+        
+        This is equivalent to map(...block=True)
+        
+        See `self.map` for details.
+        """
+        if 'block' in kwargs:
+            raise TypeError("map_sync doesn't take a `block` keyword argument.")
+        kwargs['block'] = True
+        return self.map(f,*sequences,**kwargs)
+    
+    def imap(self, f, *sequences, **kwargs):
+        """Parallel version of `itertools.imap`.
+        
+        See `self.map` for details.
+        
+        """
+        
+        return iter(self.map_async(f,*sequences, **kwargs))
+    
+    #-------------------------------------------------------------------
+    # Decorators
+    #-------------------------------------------------------------------
+    
+    def remote(self, block=True, **flags):
+        """Decorator for making a RemoteFunction"""
+        block = self.block if block is None else block
+        return remote(self, block=block, **flags)
+    
+    def parallel(self, dist='b', block=None, **flags):
+        """Decorator for making a ParallelFunction"""
+        block = self.block if block is None else block
+        return parallel(self, dist=dist, block=block, **flags)
+
[e-mail]_doctest
+class DirectView(View):
+    """Direct Multiplexer View of one or more engines.
+    
+    These are created via indexed access to a client:
+    
+    >>> dv_1 = client[1]
+    >>> dv_all = client[:]
+    >>> dv_even = client[::2]
+    >>> dv_some = client[1:3]
+    
+    This object provides dictionary access to engine namespaces:
+    
+    # push a=5:
+    >>> dv['a'] = 5 
+    # pull 'foo':
+    >>> db['foo']
+    
+    """
+    
+    def __init__(self, client=None, socket=None, targets=None):
+        super(DirectView, self).__init__(client=client, socket=socket, targets=targets)
+    
+    @property
+    def importer(self):
+        """sync_imports(local=True) as a property.
+        
+        See sync_imports for details.
+        
+        """
+        return self.sync_imports(True)
+    
+    @contextmanager
+    def sync_imports(self, local=True):
+        """Context Manager for performing simultaneous local and remote imports.
+        
+        'import x as y' will *not* work.  The 'as y' part will simply be ignored.
+        
+        >>> with view.sync_imports():
+        ...    from numpy import recarray
+        importing recarray from numpy on engine(s)
+        
+        """
+        import __builtin__
+        local_import = __builtin__.__import__
+        modules = set()
+        results = []
+        @util.interactive
+        def remote_import(name, fromlist, level):
+            """the function to be passed to apply, that actually performs the import
+            on the engine, and loads up the user namespace.
+            """
+            import sys
+            user_ns = globals()
+            mod = __import__(name, fromlist=fromlist, level=level)
+            if fromlist:
+                for key in fromlist:
+                    user_ns[key] = getattr(mod, key)
+            else:
+                user_ns[name] = sys.modules[name]
+        
+        def view_import(name, globals={}, locals={}, fromlist=[], level=-1):
+            """the drop-in replacement for __import__, that optionally imports
+            locally as well.
+            """
+            # don't override nested imports
+            save_import = __builtin__.__import__
+            __builtin__.__import__ = local_import
+            
+            if imp.lock_held():
+                # this is a side-effect import, don't do it remotely, or even
+                # ignore the local effects
+                return local_import(name, globals, locals, fromlist, level)
+            
+            imp.acquire_lock()
+            if local:
+                mod = local_import(name, globals, locals, fromlist, level)
+            else:
+                raise NotImplementedError("remote-only imports not yet implemented")
+            imp.release_lock()
+            
+            key = name+':'+','.join(fromlist or [])
+            if level == -1 and key not in modules:
+                modules.add(key)
+                if fromlist:
+                    print "importing %s from %s on engine(s)"%(','.join(fromlist), name)
+                else:
+                    print "importing %s on engine(s)"%name
+                results.append(self.apply_async(remote_import, name, fromlist, level))
+            # restore override
+            __builtin__.__import__ = save_import
+                
+            return mod
+        
+        # override __import__
+        __builtin__.__import__ = view_import
+        try:
+            # enter the block
+            yield
+        except ImportError:
+            if not local:
+                # ignore import errors if not doing local imports
+                pass
+        finally:
+            # always restore __import__
+            __builtin__.__import__ = local_import
+        
+        for r in results:
+            # raise possible remote ImportErrors here
+            r.get()
+            
+    
+    @sync_results
+    @save_ids
+    def _really_apply(self, f, args=None, kwargs=None, targets=None, block=None, track=None):
+        """calls f(*args, **kwargs) on remote engines, returning the result.
+        
+        This method sets all of `apply`'s flags via this View's attributes.
+        
+        Parameters
+        ----------
+        
+        f : callable
+        
+        args : list [default: empty]
+        
+        kwargs : dict [default: empty]
+        
+        targets : target list [default: self.targets]
+            where to run
+        block : bool [default: self.block]
+            whether to block 
+        track : bool [default: self.track]
+            whether to ask zmq to track the message, for safe non-copying sends
+        
+        Returns
+        -------
+        
+        if self.block is False:
+            returns AsyncResult
+        else:
+            returns actual result of f(*args, **kwargs) on the engine(s)
+            This will be a list of self.targets is also a list (even length 1), or
+            the single result if self.targets is an integer engine id
+        """
+        args = [] if args is None else args
+        kwargs = {} if kwargs is None else kwargs
+        block = self.block if block is None else block
+        track = self.track if track is None else track
+        targets = self.targets if targets is None else targets
+        
+        _idents = self.client._build_targets(targets)[0]
+        msg_ids = []
+        trackers = []
+        for ident in _idents:
+            msg = self.client.send_apply_message(self._socket, f, args, kwargs, track=track,
+                                    ident=ident)
+            if track:
+                trackers.append(msg['tracker'])
+            msg_ids.append(msg['msg_id'])
+        tracker = None if track is False else zmq.MessageTracker(*trackers)
+        ar = AsyncResult(self.client, msg_ids, fname=f.__name__, targets=targets, tracker=tracker)
+        if block:
+            try:
+                return ar.get()
+            except KeyboardInterrupt:
+                pass
+        return ar
+    
+    @spin_after
+    def map(self, f, *sequences, **kwargs):
+        """view.map(f, *sequences, block=self.block) => list|AsyncMapResult
+        
+        Parallel version of builtin `map`, using this View's `targets`.
+        
+        There will be one task per target, so work will be chunked
+        if the sequences are longer than `targets`.  
+        
+        Results can be iterated as they are ready, but will become available in chunks.
+        
+        Parameters
+        ----------
+        
+        f : callable
+            function to be mapped
+        *sequences: one or more sequences of matching length
+            the sequences to be distributed and passed to `f`
+        block : bool
+            whether to wait for the result or not [default self.block]
+        
+        Returns
+        -------
+        
+        if block=False:
+            AsyncMapResult
+                An object like AsyncResult, but which reassembles the sequence of results
+                into a single list. AsyncMapResults can be iterated through before all
+                results are complete.
+        else:
+            list
+                the result of map(f,*sequences)
+        """
+        
+        block = kwargs.pop('block', self.block)
+        for k in kwargs.keys():
+            if k not in ['block', 'track']:
+                raise TypeError("invalid keyword arg, %r"%k)
+        
+        assert len(sequences) > 0, "must have some sequences to map onto!"
+        pf = ParallelFunction(self, f, block=block, **kwargs)
+        return pf.map(*sequences)
+    
+    def execute(self, code, targets=None, block=None):
+        """Executes `code` on `targets` in blocking or nonblocking manner.
+        
+        ``execute`` is always `bound` (affects engine namespace)
+        
+        Parameters
+        ----------
+        
+        code : str
+                the code string to be executed
+        block : bool
+                whether or not to wait until done to return
+                default: self.block
+        """
+        return self._really_apply(util._execute, args=(code,), block=block, targets=targets)
+    
+    def run(self, filename, targets=None, block=None):
+        """Execute contents of `filename` on my engine(s). 
+        
+        This simply reads the contents of the file and calls `execute`.
+        
+        Parameters
+        ----------
+        
+        filename : str
+                The path to the file
+        targets : int/str/list of ints/strs
+                the engines on which to execute
+                default : all
+        block : bool
+                whether or not to wait until done
+                default: self.block
+        
+        """
+        with open(filename, 'r') as f:
+            # add newline in case of trailing indented whitespace
+            # which will cause SyntaxError
+            code = f.read()+'\n'
+        return self.execute(code, block=block, targets=targets)
+    
+    def update(self, ns):
+        """update remote namespace with dict `ns`
+        
+        See `push` for details.
+        """
+        return self.push(ns, block=self.block, track=self.track)
+    
+    def push(self, ns, targets=None, block=None, track=None):
+        """update remote namespace with dict `ns`
+        
+        Parameters
+        ----------
+        
+        ns : dict
+            dict of keys with which to update engine namespace(s)
+        block : bool [default : self.block]
+            whether to wait to be notified of engine receipt
+        
+        """
+        
+        block = block if block is not None else self.block
+        track = track if track is not None else self.track
+        targets = targets if targets is not None else self.targets
+        # applier = self.apply_sync if block else self.apply_async
+        if not isinstance(ns, dict):
+            raise TypeError("Must be a dict, not %s"%type(ns))
+        return self._really_apply(util._push, (ns,), block=block, track=track, targets=targets)
+
+    def get(self, key_s):
+        """get object(s) by `key_s` from remote namespace
+        
+        see `pull` for details.
+        """
+        # block = block if block is not None else self.block
+        return self.pull(key_s, block=True)
+    
+    def pull(self, names, targets=None, block=True):
+        """get object(s) by `name` from remote namespace
+        
+        will return one object if it is a key.
+        can also take a list of keys, in which case it will return a list of objects.
+        """
+        block = block if block is not None else self.block
+        targets = targets if targets is not None else self.targets
+        applier = self.apply_sync if block else self.apply_async
+        if isinstance(names, basestring):
+            pass
+        elif isinstance(names, (list,tuple,set)):
+            for key in names:
+                if not isinstance(key, basestring):
+                    raise TypeError("keys must be str, not type %r"%type(key))
+        else:
+            raise TypeError("names must be strs, not %r"%names)
+        return self._really_apply(util._pull, (names,), block=block, targets=targets)
+    
+    def scatter(self, key, seq, dist='b', flatten=False, targets=None, block=None, track=None):
+        """
+        Partition a Python sequence and send the partitions to a set of engines.
+        """
+        block = block if block is not None else self.block
+        track = track if track is not None else self.track
+        targets = targets if targets is not None else self.targets
+        
+        mapObject = Map.dists[dist]()
+        nparts = len(targets)
+        msg_ids = []
+        trackers = []
+        for index, engineid in enumerate(targets):
+            partition = mapObject.getPartition(seq, index, nparts)
+            if flatten and len(partition) == 1:
+                ns = {key: partition[0]}
+            else:
+                ns = {key: partition}
+            r = self.push(ns, block=False, track=track, targets=engineid)
+            msg_ids.extend(r.msg_ids)
+            if track:
+                trackers.append(r._tracker)
+        
+        if track:
+            tracker = zmq.MessageTracker(*trackers)
+        else:
+            tracker = None
+        
+        r = AsyncResult(self.client, msg_ids, fname='scatter', targets=targets, tracker=tracker)
+        if block:
+            r.wait()
+        else:
+            return r
+    
+    @sync_results
+    @save_ids
+    def gather(self, key, dist='b', targets=None, block=None):
+        """
+        Gather a partitioned sequence on a set of engines as a single local seq.
+        """
+        block = block if block is not None else self.block
+        targets = targets if targets is not None else self.targets
+        mapObject = Map.dists[dist]()
+        msg_ids = []
+        
+        for index, engineid in enumerate(targets):
+            msg_ids.extend(self.pull(key, block=False, targets=engineid).msg_ids)
+        
+        r = AsyncMapResult(self.client, msg_ids, mapObject, fname='gather')
+        
+        if block:
+            try:
+                return r.get()
+            except KeyboardInterrupt:
+                pass
+        return r
+    
+    def __getitem__(self, key):
+        return self.get(key)
+    
+    def __setitem__(self,key, value):
+        self.update({key:value})
+    
+    def clear(self, targets=None, block=False):
+        """Clear the remote namespaces on my engines."""
+        block = block if block is not None else self.block
+        targets = targets if targets is not None else self.targets
+        return self.client.clear(targets=targets, block=block)
+    
+    def kill(self, targets=None, block=True):
+        """Kill my engines."""
+        block = block if block is not None else self.block
+        targets = targets if targets is not None else self.targets
+        return self.client.kill(targets=targets, block=block)
+    
+    #----------------------------------------
+    # activate for %px,%autopx magics
+    #----------------------------------------
+    def activate(self):
+        """Make this `View` active for parallel magic commands.
+        
+        IPython has a magic command syntax to work with `MultiEngineClient` objects.
+        In a given IPython session there is a single active one.  While
+        there can be many `Views` created and used by the user, 
+        there is only one active one.  The active `View` is used whenever 
+        the magic commands %px and %autopx are used.
+        
+        The activate() method is called on a given `View` to make it 
+        active.  Once this has been done, the magic commands can be used.
+        """
+        
+        try:
+            # This is injected into __builtins__.
+            ip = get_ipython()
+        except NameError:
+            print "The IPython parallel magics (%result, %px, %autopx) only work within IPython."
+        else:
+            pmagic = ip.plugin_manager.get_plugin('parallelmagic')
+            if pmagic is not None:
+                pmagic.active_multiengine_client = self
+            else:
+                print "You must first load the parallelmagic extension " \
+                      "by doing '%load_ext parallelmagic'"
+
+
[e-mail]_doctest
+class LoadBalancedView(View):
+    """An load-balancing View that only executes via the Task scheduler.
+    
+    Load-balanced views can be created with the client's `view` method:
+    
+    >>> v = client.load_balanced_view()
+    
+    or targets can be specified, to restrict the potential destinations:
+    
+    >>> v = client.client.load_balanced_view(([1,3])
+    
+    which would restrict loadbalancing to between engines 1 and 3.
+    
+    """
+    
+    follow=Any()
+    after=Any()
+    timeout=CFloat()
+    
+    _task_scheme = Any()
+    _flag_names = List(['targets', 'block', 'track', 'follow', 'after', 'timeout'])
+    
+    def __init__(self, client=None, socket=None, **flags):
+        super(LoadBalancedView, self).__init__(client=client, socket=socket, **flags)
+        self._task_scheme=client._task_scheme
+    
+    def _validate_dependency(self, dep):
+        """validate a dependency.
+        
+        For use in `set_flags`.
+        """
+        if dep is None or isinstance(dep, (str, AsyncResult, Dependency)):
+            return True
+        elif isinstance(dep, (list,set, tuple)):
+            for d in dep:
+                if not isinstance(d, (str, AsyncResult)):
+                    return False
+        elif isinstance(dep, dict):
+            if set(dep.keys()) != set(Dependency().as_dict().keys()):
+                return False
+            if not isinstance(dep['msg_ids'], list):
+                return False
+            for d in dep['msg_ids']:
+                if not isinstance(d, str):
+                    return False
+        else:
+            return False
+        
+        return True
+                
+    def _render_dependency(self, dep):
+        """helper for building jsonable dependencies from various input forms."""
+        if isinstance(dep, Dependency):
+            return dep.as_dict()
+        elif isinstance(dep, AsyncResult):
+            return dep.msg_ids
+        elif dep is None:
+            return []
+        else:
+            # pass to Dependency constructor
+            return list(Dependency(dep))
+
+    def set_flags(self, **kwargs):
+        """set my attribute flags by keyword.
+        
+        A View is a wrapper for the Client's apply method, but with attributes
+        that specify keyword arguments, those attributes can be set by keyword
+        argument with this method.
+        
+        Parameters
+        ----------
+        
+        block : bool
+            whether to wait for results
+        track : bool
+            whether to create a MessageTracker to allow the user to 
+            safely edit after arrays and buffers during non-copying
+            sends.
+        #
+        after : Dependency or collection of msg_ids
+            Only for load-balanced execution (targets=None)
+            Specify a list of msg_ids as a time-based dependency.
+            This job will only be run *after* the dependencies
+            have been met.
+
+        follow : Dependency or collection of msg_ids
+            Only for load-balanced execution (targets=None)
+            Specify a list of msg_ids as a location-based dependency.
+            This job will only be run on an engine where this dependency
+            is met.
+
+        timeout : float/int or None
+            Only for load-balanced execution (targets=None)
+            Specify an amount of time (in seconds) for the scheduler to
+            wait for dependencies to be met before failing with a
+            DependencyTimeout.
+        """
+        
+        super(LoadBalancedView, self).set_flags(**kwargs)
+        for name in ('follow', 'after'):
+            if name in kwargs:
+                value = kwargs[name]
+                if self._validate_dependency(value):
+                    setattr(self, name, value)
+                else:
+                    raise ValueError("Invalid dependency: %r"%value)
+        if 'timeout' in kwargs:
+            t = kwargs['timeout']
+            if not isinstance(t, (int, long, float, type(None))):
+                raise TypeError("Invalid type for timeout: %r"%type(t))
+            if t is not None:
+                if t < 0:
+                    raise ValueError("Invalid timeout: %s"%t)
+            self.timeout = t
+    
+    @sync_results
+    @save_ids
+    def _really_apply(self, f, args=None, kwargs=None, block=None, track=None,
+                                        after=None, follow=None, timeout=None,
+                                        targets=None):
+        """calls f(*args, **kwargs) on a remote engine, returning the result.
+        
+        This method temporarily sets all of `apply`'s flags for a single call.
+        
+        Parameters
+        ----------
+        
+        f : callable
+        
+        args : list [default: empty]
+        
+        kwargs : dict [default: empty]
+        
+        block : bool [default: self.block]
+            whether to block 
+        track : bool [default: self.track]
+            whether to ask zmq to track the message, for safe non-copying sends
+            
+        !!!!!! TODO: THE REST HERE  !!!!
+        
+        Returns
+        -------
+        
+        if self.block is False:
+            returns AsyncResult
+        else:
+            returns actual result of f(*args, **kwargs) on the engine(s)
+            This will be a list of self.targets is also a list (even length 1), or
+            the single result if self.targets is an integer engine id
+        """
+
+        # validate whether we can run
+        if self._socket.closed:
+            msg = "Task farming is disabled"
+            if self._task_scheme == 'pure':
+                msg += " because the pure ZMQ scheduler cannot handle"
+                msg += " disappearing engines."
+            raise RuntimeError(msg)
+        
+        if self._task_scheme == 'pure':
+            # pure zmq scheme doesn't support dependencies
+            msg = "Pure ZMQ scheduler doesn't support dependencies"
+            if (follow or after):
+                # hard fail on DAG dependencies
+                raise RuntimeError(msg)
+            if isinstance(f, dependent):
+                # soft warn on functional dependencies
+                warnings.warn(msg, RuntimeWarning)
+        
+        # build args
+        args = [] if args is None else args
+        kwargs = {} if kwargs is None else kwargs
+        block = self.block if block is None else block
+        track = self.track if track is None else track
+        after = self.after if after is None else after
+        follow = self.follow if follow is None else follow
+        timeout = self.timeout if timeout is None else timeout
+        targets = self.targets if targets is None else targets
+        
+        if targets is None:
+            idents = []
+        else:
+            idents = self.client._build_targets(targets)[0]
+        
+        after = self._render_dependency(after)
+        follow = self._render_dependency(follow)
+        subheader = dict(after=after, follow=follow, timeout=timeout, targets=idents)
+        
+        msg = self.client.send_apply_message(self._socket, f, args, kwargs, track=track,
+                                subheader=subheader)
+        tracker = None if track is False else msg['tracker']
+        
+        ar = AsyncResult(self.client, msg['msg_id'], fname=f.__name__, targets=None, tracker=tracker)
+        
+        if block:
+            try:
+                return ar.get()
+            except KeyboardInterrupt:
+                pass
+        return ar
+        
+    @spin_after
+    @save_ids
+    def map(self, f, *sequences, **kwargs):
+        """view.map(f, *sequences, block=self.block, chunksize=1) => list|AsyncMapResult
+        
+        Parallel version of builtin `map`, load-balanced by this View.
+        
+        `block`, and `chunksize` can be specified by keyword only.
+        
+        Each `chunksize` elements will be a separate task, and will be
+        load-balanced. This lets individual elements be available for iteration
+        as soon as they arrive.
+        
+        Parameters
+        ----------
+        
+        f : callable
+            function to be mapped
+        *sequences: one or more sequences of matching length
+            the sequences to be distributed and passed to `f`
+        block : bool
+            whether to wait for the result or not [default self.block]
+        track : bool
+            whether to create a MessageTracker to allow the user to 
+            safely edit after arrays and buffers during non-copying
+            sends.
+        chunksize : int
+            how many elements should be in each task [default 1]
+        
+        Returns
+        -------
+        
+        if block=False:
+            AsyncMapResult
+                An object like AsyncResult, but which reassembles the sequence of results
+                into a single list. AsyncMapResults can be iterated through before all
+                results are complete.
+            else:
+                the result of map(f,*sequences)
+        
+        """
+        
+        # default
+        block = kwargs.get('block', self.block)
+        chunksize = kwargs.get('chunksize', 1)
+        
+        keyset = set(kwargs.keys())
+        extra_keys = keyset.difference_update(set(['block', 'chunksize']))
+        if extra_keys:
+            raise TypeError("Invalid kwargs: %s"%list(extra_keys))
+        
+        assert len(sequences) > 0, "must have some sequences to map onto!"
+        
+        pf = ParallelFunction(self, f, block=block,  chunksize=chunksize)
+        return pf.map(*sequences)
+
+__all__ = ['LoadBalancedView', 'DirectView']
  No newline at end of file

IPython/parallel/controller/controller.py

@@ -0,0 +1,117 b''
+#!/usr/bin/env python
+"""The IPython Controller with 0MQ
+This is a collection of one Hub and several Schedulers.
+"""
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2010  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+from __future__ import print_function
+
+from multiprocessing import Process
+
+import zmq
+from zmq.devices import ProcessMonitoredQueue
+# internal:
+from IPython.utils.importstring import import_item
+from IPython.utils.traitlets import Int, CStr, Instance, List, Bool
+
+from IPython.parallel.util import signal_children
+from .hub import Hub, HubFactory
+from .scheduler import launch_scheduler
+
+#-----------------------------------------------------------------------------
+# Configurable
+#-----------------------------------------------------------------------------
+
+
+class ControllerFactory(HubFactory):
+    """Configurable for setting up a Hub and Schedulers."""
+    
+    usethreads = Bool(False, config=True)
+    # pure-zmq downstream HWM
+    hwm = Int(0, config=True)
+    
+    # internal
+    children = List()
+    mq_class = CStr('zmq.devices.ProcessMonitoredQueue')
+    
+    def _usethreads_changed(self, name, old, new):
+        self.mq_class = 'zmq.devices.%sMonitoredQueue'%('Thread' if new else 'Process')
+        
+    def __init__(self, **kwargs):
+        super(ControllerFactory, self).__init__(**kwargs)
+        self.subconstructors.append(self.construct_schedulers)
+    
+    def start(self):
+        super(ControllerFactory, self).start()
+        child_procs = []
+        for child in self.children:
+            child.start()
+            if isinstance(child, ProcessMonitoredQueue):
+                child_procs.append(child.launcher)
+            elif isinstance(child, Process):
+                child_procs.append(child)
+        if child_procs:
+            signal_children(child_procs)
+        
+    
+    def construct_schedulers(self):
+        children = self.children
+        mq = import_item(self.mq_class)
+        
+        maybe_inproc = 'inproc://monitor' if self.usethreads else self.monitor_url
+        # IOPub relay (in a Process)
+        q = mq(zmq.PUB, zmq.SUB, zmq.PUB, 'N/A','iopub')
+        q.bind_in(self.client_info['iopub'])
+        q.bind_out(self.engine_info['iopub'])
+        q.setsockopt_out(zmq.SUBSCRIBE, '')
+        q.connect_mon(maybe_inproc)
+        q.daemon=True
+        children.append(q)
+
+        # Multiplexer Queue (in a Process)
+        q = mq(zmq.XREP, zmq.XREP, zmq.PUB, 'in', 'out')
+        q.bind_in(self.client_info['mux'])
+        q.setsockopt_in(zmq.IDENTITY, 'mux')
+        q.bind_out(self.engine_info['mux'])
+        q.connect_mon(maybe_inproc)
+        q.daemon=True
+        children.append(q)
+
+        # Control Queue (in a Process)
+        q = mq(zmq.XREP, zmq.XREP, zmq.PUB, 'incontrol', 'outcontrol')
+        q.bind_in(self.client_info['control'])
+        q.setsockopt_in(zmq.IDENTITY, 'control')
+        q.bind_out(self.engine_info['control'])
+        q.connect_mon(maybe_inproc)
+        q.daemon=True
+        children.append(q)
+        # Task Queue (in a Process)
+        if self.scheme == 'pure':
+            self.log.warn("task::using pure XREQ Task scheduler")
+            q = mq(zmq.XREP, zmq.XREQ, zmq.PUB, 'intask', 'outtask')
+            q.setsockopt_out(zmq.HWM, self.hwm)
+            q.bind_in(self.client_info['task'][1])
+            q.setsockopt_in(zmq.IDENTITY, 'task')
+            q.bind_out(self.engine_info['task'])
+            q.connect_mon(maybe_inproc)
+            q.daemon=True
+            children.append(q)
+        elif self.scheme == 'none':
+            self.log.warn("task::using no Task scheduler")
+            
+        else:
+            self.log.info("task::using Python %s Task scheduler"%self.scheme)
+            sargs = (self.client_info['task'][1], self.engine_info['task'], self.monitor_url, self.client_info['notification'])
+            kwargs = dict(scheme=self.scheme,logname=self.log.name, loglevel=self.log.level, config=self.config)
+            q = Process(target=launch_scheduler, args=sargs, kwargs=kwargs)
+            q.daemon=True
+            children.append(q)
+

IPython/parallel/controller/dependency.py

@@ -0,0 +1,196 b''
+"""Dependency utilities"""
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2010-2011  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+from types import ModuleType
+
+from IPython.parallel.client.asyncresult import AsyncResult
+from IPython.parallel.error import UnmetDependency
+from IPython.parallel.util import interactive
+
+class depend(object):
+    """Dependency decorator, for use with tasks.
+    
+    `@depend` lets you define a function for engine dependencies
+    just like you use `apply` for tasks.
+    
+    
+    Examples
+    --------
+    ::
+    
+        @depend(df, a,b, c=5)
+        def f(m,n,p)
+        
+        view.apply(f, 1,2,3)
+    
+    will call df(a,b,c=5) on the engine, and if it returns False or
+    raises an UnmetDependency error, then the task will not be run
+    and another engine will be tried.
+    """
+    def __init__(self, f, *args, **kwargs):
+        self.f = f
+        self.args = args
+        self.kwargs = kwargs
+    
+    def __call__(self, f):
+        return dependent(f, self.f, *self.args, **self.kwargs)
+
+class dependent(object):
+    """A function that depends on another function.
+    This is an object to prevent the closure used
+    in traditional decorators, which are not picklable.
+    """
+
+    def __init__(self, f, df, *dargs, **dkwargs):
+        self.f = f
+        self.func_name = getattr(f, '__name__', 'f')
+        self.df = df
+        self.dargs = dargs
+        self.dkwargs = dkwargs
+
+    def __call__(self, *args, **kwargs):
+        # if hasattr(self.f, 'func_globals') and hasattr(self.df, 'func_globals'):
+        #     self.df.func_globals = self.f.func_globals
+        if self.df(*self.dargs, **self.dkwargs) is False:
+            raise UnmetDependency()
+        return self.f(*args, **kwargs)
+    
+    @property
+    def __name__(self):
+        return self.func_name
+
+@interactive
+def _require(*names):
+    """Helper for @require decorator."""
+    from IPython.parallel.error import UnmetDependency
+    user_ns = globals()
+    for name in names:
+        if name in user_ns:
+            continue
+        try:
+            exec 'import %s'%name in user_ns
+        except ImportError:
+            raise UnmetDependency(name)
+    return True
+
+def require(*mods):
+    """Simple decorator for requiring names to be importable.
+    
+    Examples
+    --------
+    
+    In [1]: @require('numpy')
+       ...: def norm(a):
+       ...:     import numpy
+       ...:     return numpy.linalg.norm(a,2)
+    """
+    names = []
+    for mod in mods:
+        if isinstance(mod, ModuleType):
+            mod = mod.__name__
+        
+        if isinstance(mod, basestring):
+            names.append(mod)
+        else:
+            raise TypeError("names must be modules or module names, not %s"%type(mod))
+    
+    return depend(_require, *names)
+
+class Dependency(set):
+    """An object for representing a set of msg_id dependencies.
+    
+    Subclassed from set().
+    
+    Parameters
+    ----------
+    dependencies: list/set of msg_ids or AsyncResult objects or output of Dependency.as_dict()
+        The msg_ids to depend on
+    all : bool [default True]
+        Whether the dependency should be considered met when *all* depending tasks have completed
+        or only when *any* have been completed.
+    success : bool [default True]
+        Whether to consider successes as fulfilling dependencies.
+    failure : bool [default False]
+        Whether to consider failures as fulfilling dependencies.
+    
+    If `all=success=True` and `failure=False`, then the task will fail with an ImpossibleDependency
+        as soon as the first depended-upon task fails.
+    """
+    
+    all=True
+    success=True
+    failure=True
+    
+    def __init__(self, dependencies=[], all=True, success=True, failure=False):
+        if isinstance(dependencies, dict):
+            # load from dict
+            all = dependencies.get('all', True)
+            success = dependencies.get('success', success)
+            failure = dependencies.get('failure', failure)
+            dependencies = dependencies.get('dependencies', [])
+        ids = []
+        
+        # extract ids from various sources:
+        if isinstance(dependencies, (basestring, AsyncResult)):
+            dependencies = [dependencies]
+        for d in dependencies:
+            if isinstance(d, basestring):
+                ids.append(d)
+            elif isinstance(d, AsyncResult):
+                ids.extend(d.msg_ids)
+            else:
+                raise TypeError("invalid dependency type: %r"%type(d))
+        
+        set.__init__(self, ids)
+        self.all = all
+        if not (success or failure):
+            raise ValueError("Must depend on at least one of successes or failures!")
+        self.success=success
+        self.failure = failure
+    
+    def check(self, completed, failed=None):
+        """check whether our dependencies have been met."""
+        if len(self) == 0:
+            return True
+        against = set()
+        if self.success:
+            against = completed
+        if failed is not None and self.failure:
+            against = against.union(failed)
+        if self.all:
+            return self.issubset(against)
+        else:
+            return not self.isdisjoint(against)
+    
+    def unreachable(self, completed, failed=None):
+        """return whether this dependency has become impossible."""
+        if len(self) == 0:
+            return False
+        against = set()
+        if not self.success:
+            against = completed
+        if failed is not None and not self.failure:
+            against = against.union(failed)
+        if self.all:
+            return not self.isdisjoint(against)
+        else:
+            return self.issubset(against)
+        
+    
+    def as_dict(self):
+        """Represent this dependency as a dict. For json compatibility."""
+        return dict(
+            dependencies=list(self),
+            all=self.all,
+            success=self.success,
+            failure=self.failure
+        )
+
+
+__all__ = ['depend', 'require', 'dependent', 'Dependency']
+

IPython/parallel/controller/dictdb.py

@@ -0,0 +1,155 b''
+"""A Task logger that presents our DB interface, 
+but exists entirely in memory and implemented with dicts.
+
+TaskRecords are dicts of the form:
+{
+    'msg_id' : str(uuid),
+    'client_uuid' : str(uuid),
+    'engine_uuid' : str(uuid) or None,
+    'header' : dict(header),
+    'content': dict(content),
+    'buffers': list(buffers),
+    'submitted': datetime,
+    'started': datetime or None,
+    'completed': datetime or None,
+    'resubmitted': datetime or None,
+    'result_header' : dict(header) or None,
+    'result_content' : dict(content) or None,
+    'result_buffers' : list(buffers) or None,
+}
+With this info, many of the special categories of tasks can be defined by query:
+
+pending:  completed is None
+client's outstanding: client_uuid = uuid && completed is None
+MIA: arrived is None (and completed is None)
+etc.
+
+EngineRecords are dicts of the form:
+{
+    'eid' : int(id),
+    'uuid': str(uuid)
+}
+This may be extended, but is currently.
+
+We support a subset of mongodb operators:
+    $lt,$gt,$lte,$gte,$ne,$in,$nin,$all,$mod,$exists
+"""
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2010  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+
+from datetime import datetime
+
+from IPython.config.configurable import Configurable
+
+from IPython.utils.traitlets import Dict, CUnicode
+
+filters = {
+ '$lt' : lambda a,b: a < b,
+ '$gt' : lambda a,b: b > a,
+ '$eq' : lambda a,b: a == b,
+ '$ne' : lambda a,b: a != b,
+ '$lte': lambda a,b: a <= b,
+ '$gte': lambda a,b: a >= b,
+ '$in' : lambda a,b: a in b,
+ '$nin': lambda a,b: a not in b,
+ '$all': lambda a,b: all([ a in bb for bb in b ]),
+ '$mod': lambda a,b: a%b[0] == b[1],
+ '$exists' : lambda a,b: (b and a is not None) or (a is None and not b)
+}
+
+
+class CompositeFilter(object):
+    """Composite filter for matching multiple properties."""
+    
+    def __init__(self, dikt):
+        self.tests = []
+        self.values = []
+        for key, value in dikt.iteritems():
+            self.tests.append(filters[key])
+            self.values.append(value)
+
+    def __call__(self, value):
+        for test,check in zip(self.tests, self.values):
+            if not test(value, check):
+                return False
+        return True
+
+class BaseDB(Configurable):
+    """Empty Parent class so traitlets work on DB."""
+    # base configurable traits:
+    session = CUnicode("")
+
+class DictDB(BaseDB):
+    """Basic in-memory dict-based object for saving Task Records.
+    
+    This is the first object to present the DB interface
+    for logging tasks out of memory.
+    
+    The interface is based on MongoDB, so adding a MongoDB
+    backend should be straightforward.
+    """
+    
+    _records = Dict()
+    
+    def _match_one(self, rec, tests):
+        """Check if a specific record matches tests."""
+        for key,test in tests.iteritems():
+            if not test(rec.get(key, None)):
+                return False
+        return True
+        
+    def _match(self, check, id_only=True):
+        """Find all the matches for a check dict."""
+        matches = {}
+        tests = {}
+        for k,v in check.iteritems():
+            if isinstance(v, dict):
+                tests[k] = CompositeFilter(v)
+            else:
+                tests[k] = lambda o: o==v
+        
+        for msg_id, rec in self._records.iteritems():
+            if self._match_one(rec, tests):
+                matches[msg_id] = rec
+        if id_only:
+            return matches.keys()
+        else:
+            return matches
+            
+    
+    def add_record(self, msg_id, rec):
+        """Add a new Task Record, by msg_id."""
+        if self._records.has_key(msg_id):
+            raise KeyError("Already have msg_id %r"%(msg_id))
+        self._records[msg_id] = rec
+    
+    def get_record(self, msg_id):
+        """Get a specific Task Record, by msg_id."""
+        if not self._records.has_key(msg_id):
+            raise KeyError("No such msg_id %r"%(msg_id))
+        return self._records[msg_id]
+    
+    def update_record(self, msg_id, rec):
+        """Update the data in an existing record."""
+        self._records[msg_id].update(rec)
+    
+    def drop_matching_records(self, check):
+        """Remove a record from the DB."""
+        matches = self._match(check, id_only=True)
+        for m in matches:
+            del self._records[m]
+        
+    def drop_record(self, msg_id):
+        """Remove a record from the DB."""
+        del self._records[msg_id]
+        
+    
+    def find_records(self, check, id_only=False):
+        """Find records matching a query dict."""
+        matches = self._match(check, id_only)
+        return matches
  No newline at end of file

IPython/parallel/controller/heartmonitor.py

@@ -0,0 +1,163 b''
+#!/usr/bin/env python
+"""
+A multi-heart Heartbeat system using PUB and XREP sockets. pings are sent out on the PUB,
+and hearts are tracked based on their XREQ identities.
+"""
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2010-2011  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+from __future__ import print_function
+import time
+import uuid
+
+import zmq
+from zmq.devices import ProcessDevice, ThreadDevice
+from zmq.eventloop import ioloop, zmqstream
+
+from IPython.utils.traitlets import Set, Instance, CFloat, Bool
+from IPython.parallel.factory import LoggingFactory
+
+class Heart(object):
+    """A basic heart object for responding to a HeartMonitor.
+    This is a simple wrapper with defaults for the most common
+    Device model for responding to heartbeats.
+    
+    It simply builds a threadsafe zmq.FORWARDER Device, defaulting to using 
+    SUB/XREQ for in/out.
+    
+    You can specify the XREQ's IDENTITY via the optional heart_id argument."""
+    device=None
+    id=None
+    def __init__(self, in_addr, out_addr, in_type=zmq.SUB, out_type=zmq.XREQ, heart_id=None):
+        self.device = ThreadDevice(zmq.FORWARDER, in_type, out_type)
+        self.device.daemon=True
+        self.device.connect_in(in_addr)
+        self.device.connect_out(out_addr)
+        if in_type == zmq.SUB:
+            self.device.setsockopt_in(zmq.SUBSCRIBE, "")
+        if heart_id is None:
+            heart_id = str(uuid.uuid4())
+        self.device.setsockopt_out(zmq.IDENTITY, heart_id)
+        self.id = heart_id
+    
+    def start(self):
+        return self.device.start()
+        
+class HeartMonitor(LoggingFactory):
+    """A basic HeartMonitor class
+    pingstream: a PUB stream
+    pongstream: an XREP stream
+    period: the period of the heartbeat in milliseconds"""
+    
+    period=CFloat(1000, config=True) # in milliseconds
+    
+    pingstream=Instance('zmq.eventloop.zmqstream.ZMQStream')
+    pongstream=Instance('zmq.eventloop.zmqstream.ZMQStream')
+    loop = Instance('zmq.eventloop.ioloop.IOLoop')
+    def _loop_default(self):
+        return ioloop.IOLoop.instance()
+    debug=Bool(False)
+    
+    # not settable:
+    hearts=Set()
+    responses=Set()
+    on_probation=Set()
+    last_ping=CFloat(0)
+    _new_handlers = Set()
+    _failure_handlers = Set()
+    lifetime = CFloat(0)
+    tic = CFloat(0)
+    
+    def __init__(self, **kwargs):
+        super(HeartMonitor, self).__init__(**kwargs)
+        
+        self.pongstream.on_recv(self.handle_pong)
+    
+    def start(self):
+        self.caller = ioloop.PeriodicCallback(self.beat, self.period, self.loop)
+        self.caller.start()
+    
+    def add_new_heart_handler(self, handler):
+        """add a new handler for new hearts"""
+        self.log.debug("heartbeat::new_heart_handler: %s"%handler)
+        self._new_handlers.add(handler)
+        
+    def add_heart_failure_handler(self, handler):
+        """add a new handler for heart failure"""
+        self.log.debug("heartbeat::new heart failure handler: %s"%handler)
+        self._failure_handlers.add(handler)
+            
+    def beat(self):
+        self.pongstream.flush() 
+        self.last_ping = self.lifetime
+        
+        toc = time.time()
+        self.lifetime += toc-self.tic
+        self.tic = toc
+        # self.log.debug("heartbeat::%s"%self.lifetime)
+        goodhearts = self.hearts.intersection(self.responses)
+        missed_beats = self.hearts.difference(goodhearts)
+        heartfailures = self.on_probation.intersection(missed_beats)
+        newhearts = self.responses.difference(goodhearts)
+        map(self.handle_new_heart, newhearts)
+        map(self.handle_heart_failure, heartfailures)
+        self.on_probation = missed_beats.intersection(self.hearts)
+        self.responses = set()
+        # print self.on_probation, self.hearts
+        # self.log.debug("heartbeat::beat %.3f, %i beating hearts"%(self.lifetime, len(self.hearts)))
+        self.pingstream.send(str(self.lifetime))
+    
+    def handle_new_heart(self, heart):
+        if self._new_handlers:
+            for handler in self._new_handlers:
+                handler(heart)
+        else:
+            self.log.info("heartbeat::yay, got new heart %s!"%heart)
+        self.hearts.add(heart)
+    
+    def handle_heart_failure(self, heart):
+        if self._failure_handlers:
+            for handler in self._failure_handlers:
+                try:
+                    handler(heart)
+                except Exception as e:
+                    self.log.error("heartbeat::Bad Handler! %s"%handler, exc_info=True)
+                    pass
+        else:
+            self.log.info("heartbeat::Heart %s failed :("%heart)
+        self.hearts.remove(heart)
+        
+    
+    def handle_pong(self, msg):
+        "a heart just beat"
+        if msg[1] == str(self.lifetime):
+            delta = time.time()-self.tic
+            # self.log.debug("heartbeat::heart %r took %.2f ms to respond"%(msg[0], 1000*delta))
+            self.responses.add(msg[0])
+        elif msg[1] == str(self.last_ping):
+            delta = time.time()-self.tic + (self.lifetime-self.last_ping)
+            self.log.warn("heartbeat::heart %r missed a beat, and took %.2f ms to respond"%(msg[0], 1000*delta))
+            self.responses.add(msg[0])
+        else:
+            self.log.warn("heartbeat::got bad heartbeat (possibly old?): %s (current=%.3f)"%
+            (msg[1],self.lifetime))
+
+
+if __name__ == '__main__':
+    loop = ioloop.IOLoop.instance()
+    context = zmq.Context()
+    pub = context.socket(zmq.PUB)
+    pub.bind('tcp://127.0.0.1:5555')
+    xrep = context.socket(zmq.XREP)
+    xrep.bind('tcp://127.0.0.1:5556')
+    
+    outstream = zmqstream.ZMQStream(pub, loop)
+    instream = zmqstream.ZMQStream(xrep, loop)
+    
+    hb = HeartMonitor(loop, outstream, instream)
+    
+    loop.start()

IPython/parallel/controller/hub.py

@@ -0,0 +1,1089 b''
+#!/usr/bin/env python
+"""The IPython Controller Hub with 0MQ
+This is the master object that handles connections from engines and clients,
+and monitors traffic through the various queues.
+"""
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2010  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+from __future__ import print_function
+
+import sys
+import time
+from datetime import datetime
+
+import zmq
+from zmq.eventloop import ioloop
+from zmq.eventloop.zmqstream import ZMQStream
+
+# internal:
+from IPython.utils.importstring import import_item
+from IPython.utils.traitlets import HasTraits, Instance, Int, CStr, Str, Dict, Set, List, Bool
+
+from IPython.parallel import error
+from IPython.parallel.factory import RegistrationFactory, LoggingFactory
+from IPython.parallel.util import select_random_ports, validate_url_container, ISO8601
+
+from .heartmonitor import HeartMonitor
+
+#-----------------------------------------------------------------------------
+# Code
+#-----------------------------------------------------------------------------
+
+def _passer(*args, **kwargs):
+    return
+
+def _printer(*args, **kwargs):
+    print (args)
+    print (kwargs)
+
+def empty_record():
+    """Return an empty dict with all record keys."""
+    return {
+        'msg_id' : None,
+        'header' : None,
+        'content': None,
+        'buffers': None,
+        'submitted': None,
+        'client_uuid' : None,
+        'engine_uuid' : None,
+        'started': None,
+        'completed': None,
+        'resubmitted': None,
+        'result_header' : None,
+        'result_content' : None,
+        'result_buffers' : None,
+        'queue' : None,
+        'pyin' : None,
+        'pyout': None,
+        'pyerr': None,
+        'stdout': '',
+        'stderr': '',
+    }
+    
+def init_record(msg):
+    """Initialize a TaskRecord based on a request."""
+    header = msg['header']
+    return {
+        'msg_id' : header['msg_id'],
+        'header' : header,
+        'content': msg['content'],
+        'buffers': msg['buffers'],
+        'submitted': datetime.strptime(header['date'], ISO8601),
+        'client_uuid' : None,
+        'engine_uuid' : None,
+        'started': None,
+        'completed': None,
+        'resubmitted': None,
+        'result_header' : None,
+        'result_content' : None,
+        'result_buffers' : None,
+        'queue' : None,
+        'pyin' : None,
+        'pyout': None,
+        'pyerr': None,
+        'stdout': '',
+        'stderr': '',
+    }
+
+
+class EngineConnector(HasTraits):
+    """A simple object for accessing the various zmq connections of an object.
+    Attributes are:
+    id (int): engine ID
+    uuid (str): uuid (unused?)
+    queue (str): identity of queue's XREQ socket
+    registration (str): identity of registration XREQ socket
+    heartbeat (str): identity of heartbeat XREQ socket
+    """
+    id=Int(0)
+    queue=Str()
+    control=Str()
+    registration=Str()
+    heartbeat=Str()
+    pending=Set()
+
+class HubFactory(RegistrationFactory):
+    """The Configurable for setting up a Hub."""
+    
+    # name of a scheduler scheme
+    scheme = Str('leastload', config=True)
+    
+    # port-pairs for monitoredqueues:
+    hb = Instance(list, config=True)
+    def _hb_default(self):
+        return select_random_ports(2)
+    
+    mux = Instance(list, config=True)
+    def _mux_default(self):
+        return select_random_ports(2)
+    
+    task = Instance(list, config=True)
+    def _task_default(self):
+        return select_random_ports(2)
+    
+    control = Instance(list, config=True)
+    def _control_default(self):
+        return select_random_ports(2)
+    
+    iopub = Instance(list, config=True)
+    def _iopub_default(self):
+        return select_random_ports(2)
+    
+    # single ports:
+    mon_port = Instance(int, config=True)
+    def _mon_port_default(self):
+        return select_random_ports(1)[0]
+    
+    notifier_port = Instance(int, config=True)
+    def _notifier_port_default(self):
+        return select_random_ports(1)[0]
+    
+    ping = Int(1000, config=True) # ping frequency
+    
+    engine_ip = CStr('127.0.0.1', config=True)
+    engine_transport = CStr('tcp', config=True)
+    
+    client_ip = CStr('127.0.0.1', config=True)
+    client_transport = CStr('tcp', config=True)
+    
+    monitor_ip = CStr('127.0.0.1', config=True)
+    monitor_transport = CStr('tcp', config=True)
+    
+    monitor_url = CStr('')
+    
+    db_class = CStr('IPython.parallel.controller.dictdb.DictDB', config=True)
+    
+    # not configurable
+    db = Instance('IPython.parallel.controller.dictdb.BaseDB')
+    heartmonitor = Instance('IPython.parallel.controller.heartmonitor.HeartMonitor')
+    subconstructors = List()
+    _constructed = Bool(False)
+    
+    def _ip_changed(self, name, old, new):
+        self.engine_ip = new
+        self.client_ip = new
+        self.monitor_ip = new
+        self._update_monitor_url()
+    
+    def _update_monitor_url(self):
+        self.monitor_url = "%s://%s:%i"%(self.monitor_transport, self.monitor_ip, self.mon_port)
+    
+    def _transport_changed(self, name, old, new):
+        self.engine_transport = new
+        self.client_transport = new
+        self.monitor_transport = new
+        self._update_monitor_url()
+        
+    def __init__(self, **kwargs):
+        super(HubFactory, self).__init__(**kwargs)
+        self._update_monitor_url()
+        # self.on_trait_change(self._sync_ips, 'ip')
+        # self.on_trait_change(self._sync_transports, 'transport')
+        self.subconstructors.append(self.construct_hub)
+    
+    
+    def construct(self):
+        assert not self._constructed, "already constructed!"
+        
+        for subc in self.subconstructors:
+            subc()
+        
+        self._constructed = True
+        
+    
+    def start(self):
+        assert self._constructed, "must be constructed by self.construct() first!"
+        self.heartmonitor.start()
+        self.log.info("Heartmonitor started")
+    
+    def construct_hub(self):
+        """construct"""
+        client_iface = "%s://%s:"%(self.client_transport, self.client_ip) + "%i"
+        engine_iface = "%s://%s:"%(self.engine_transport, self.engine_ip) + "%i"
+        
+        ctx = self.context
+        loop = self.loop
+        
+        # Registrar socket
+        q = ZMQStream(ctx.socket(zmq.XREP), loop)
+        q.bind(client_iface % self.regport)
+        self.log.info("Hub listening on %s for registration."%(client_iface%self.regport))
+        if self.client_ip != self.engine_ip:
+            q.bind(engine_iface % self.regport)
+            self.log.info("Hub listening on %s for registration."%(engine_iface%self.regport))
+        
+        ### Engine connections ###
+
+        # heartbeat
+        hpub = ctx.socket(zmq.PUB)
+        hpub.bind(engine_iface % self.hb[0])
+        hrep = ctx.socket(zmq.XREP)
+        hrep.bind(engine_iface % self.hb[1])
+        self.heartmonitor = HeartMonitor(loop=loop, pingstream=ZMQStream(hpub,loop), pongstream=ZMQStream(hrep,loop), 
+                                period=self.ping, logname=self.log.name)
+
+        ### Client connections ###
+        # Notifier socket
+        n = ZMQStream(ctx.socket(zmq.PUB), loop)
+        n.bind(client_iface%self.notifier_port)
+
+        ### build and launch the queues ###
+
+        # monitor socket
+        sub = ctx.socket(zmq.SUB)
+        sub.setsockopt(zmq.SUBSCRIBE, "")
+        sub.bind(self.monitor_url)
+        sub.bind('inproc://monitor')
+        sub = ZMQStream(sub, loop)
+        
+        # connect the db
+        self.log.info('Hub using DB backend: %r'%(self.db_class.split()[-1]))
+        # cdir = self.config.Global.cluster_dir
+        self.db = import_item(self.db_class)(session=self.session.session, config=self.config)
+        time.sleep(.25)
+
+        # build connection dicts
+        self.engine_info = {
+            'control' : engine_iface%self.control[1],
+            'mux': engine_iface%self.mux[1],
+            'heartbeat': (engine_iface%self.hb[0], engine_iface%self.hb[1]),
+            'task' : engine_iface%self.task[1],
+            'iopub' : engine_iface%self.iopub[1],
+            # 'monitor' : engine_iface%self.mon_port,
+            }
+
+        self.client_info = {
+            'control' : client_iface%self.control[0],
+            'mux': client_iface%self.mux[0],
+            'task' : (self.scheme, client_iface%self.task[0]),
+            'iopub' : client_iface%self.iopub[0],
+            'notification': client_iface%self.notifier_port
+            }
+        self.log.debug("Hub engine addrs: %s"%self.engine_info)
+        self.log.debug("Hub client addrs: %s"%self.client_info)
+        self.hub = Hub(loop=loop, session=self.session, monitor=sub, heartmonitor=self.heartmonitor,
+                query=q, notifier=n, db=self.db,
+                engine_info=self.engine_info, client_info=self.client_info,
+                logname=self.log.name)
+    
+
+class Hub(LoggingFactory):
+    """The IPython Controller Hub with 0MQ connections
+    
+    Parameters
+    ==========
+    loop: zmq IOLoop instance
+    session: StreamSession object
+    <removed> context: zmq context for creating new connections (?)
+    queue: ZMQStream for monitoring the command queue (SUB)
+    query: ZMQStream for engine registration and client queries requests (XREP)
+    heartbeat: HeartMonitor object checking the pulse of the engines
+    notifier: ZMQStream for broadcasting engine registration changes (PUB)
+    db: connection to db for out of memory logging of commands
+                NotImplemented
+    engine_info: dict of zmq connection information for engines to connect
+                to the queues.
+    client_info: dict of zmq connection information for engines to connect
+                to the queues.
+    """
+    # internal data structures:
+    ids=Set() # engine IDs
+    keytable=Dict()
+    by_ident=Dict()
+    engines=Dict()
+    clients=Dict()
+    hearts=Dict()
+    pending=Set()
+    queues=Dict()  # pending msg_ids keyed by engine_id
+    tasks=Dict() # pending msg_ids submitted as tasks, keyed by client_id
+    completed=Dict() # completed msg_ids keyed by engine_id
+    all_completed=Set() # completed msg_ids keyed by engine_id
+    dead_engines=Set() # completed msg_ids keyed by engine_id
+    # mia=None
+    incoming_registrations=Dict()
+    registration_timeout=Int()
+    _idcounter=Int(0)
+    
+    # objects from constructor:
+    loop=Instance(ioloop.IOLoop)
+    query=Instance(ZMQStream)
+    monitor=Instance(ZMQStream)
+    heartmonitor=Instance(HeartMonitor)
+    notifier=Instance(ZMQStream)
+    db=Instance(object)
+    client_info=Dict()
+    engine_info=Dict()
+    
+    
+    def __init__(self, **kwargs):
+        """
+        # universal:
+        loop: IOLoop for creating future connections
+        session: streamsession for sending serialized data
+        # engine:
+        queue: ZMQStream for monitoring queue messages
+        query: ZMQStream for engine+client registration and client requests
+        heartbeat: HeartMonitor object for tracking engines
+        # extra:
+        db: ZMQStream for db connection (NotImplemented)
+        engine_info: zmq address/protocol dict for engine connections
+        client_info: zmq address/protocol dict for client connections
+        """
+        
+        super(Hub, self).__init__(**kwargs)
+        self.registration_timeout = max(5000, 2*self.heartmonitor.period)
+        
+        # validate connection dicts:
+        for k,v in self.client_info.iteritems():
+            if k == 'task':
+                validate_url_container(v[1])
+            else:
+                validate_url_container(v)
+        # validate_url_container(self.client_info)
+        validate_url_container(self.engine_info)
+        
+        # register our callbacks
+        self.query.on_recv(self.dispatch_query)
+        self.monitor.on_recv(self.dispatch_monitor_traffic)
+        
+        self.heartmonitor.add_heart_failure_handler(self.handle_heart_failure)
+        self.heartmonitor.add_new_heart_handler(self.handle_new_heart)
+        
+        self.monitor_handlers = { 'in' : self.save_queue_request,
+                                'out': self.save_queue_result,
+                                'intask': self.save_task_request,
+                                'outtask': self.save_task_result,
+                                'tracktask': self.save_task_destination,
+                                'incontrol': _passer,
+                                'outcontrol': _passer,
+                                'iopub': self.save_iopub_message,
+        }
+        
+        self.query_handlers = {'queue_request': self.queue_status,
+                                'result_request': self.get_results,
+                                'purge_request': self.purge_results,
+                                'load_request': self.check_load,
+                                'resubmit_request': self.resubmit_task,
+                                'shutdown_request': self.shutdown_request,
+                                'registration_request' : self.register_engine,
+                                'unregistration_request' : self.unregister_engine,
+                                'connection_request': self.connection_request,
+        }
+        
+        self.log.info("hub::created hub")
+    
+    @property
+    def _next_id(self):
+        """gemerate a new ID.
+        
+        No longer reuse old ids, just count from 0."""
+        newid = self._idcounter
+        self._idcounter += 1
+        return newid
+        # newid = 0
+        # incoming = [id[0] for id in self.incoming_registrations.itervalues()]
+        # # print newid, self.ids, self.incoming_registrations
+        # while newid in self.ids or newid in incoming:
+        #     newid += 1
+        # return newid
+    
+    #-----------------------------------------------------------------------------
+    # message validation
+    #-----------------------------------------------------------------------------
+    
+    def _validate_targets(self, targets):
+        """turn any valid targets argument into a list of integer ids"""
+        if targets is None:
+            # default to all
+            targets = self.ids
+            
+        if isinstance(targets, (int,str,unicode)):
+            # only one target specified
+            targets = [targets]
+        _targets = []
+        for t in targets:
+            # map raw identities to ids
+            if isinstance(t, (str,unicode)):
+                t = self.by_ident.get(t, t)
+            _targets.append(t)
+        targets = _targets
+        bad_targets = [ t for t in targets if t not in self.ids ]
+        if bad_targets:
+            raise IndexError("No Such Engine: %r"%bad_targets)
+        if not targets:
+            raise IndexError("No Engines Registered")
+        return targets
+    
+    #-----------------------------------------------------------------------------
+    # dispatch methods (1 per stream)
+    #-----------------------------------------------------------------------------
+    
+    # def dispatch_registration_request(self, msg):
+    #     """"""
+    #     self.log.debug("registration::dispatch_register_request(%s)"%msg)
+    #     idents,msg = self.session.feed_identities(msg)
+    #     if not idents:
+    #         self.log.error("Bad Query Message: %s"%msg, exc_info=True)
+    #         return
+    #     try:
+    #         msg = self.session.unpack_message(msg,content=True)
+    #     except:
+    #         self.log.error("registration::got bad registration message: %s"%msg, exc_info=True)
+    #         return
+    #     
+    #     msg_type = msg['msg_type']
+    #     content = msg['content']
+    #     
+    #     handler = self.query_handlers.get(msg_type, None)
+    #     if handler is None:
+    #         self.log.error("registration::got bad registration message: %s"%msg)
+    #     else:
+    #         handler(idents, msg)
+    
+    def dispatch_monitor_traffic(self, msg):
+        """all ME and Task queue messages come through here, as well as
+        IOPub traffic."""
+        self.log.debug("monitor traffic: %s"%msg[:2])
+        switch = msg[0]
+        idents, msg = self.session.feed_identities(msg[1:])
+        if not idents:
+            self.log.error("Bad Monitor Message: %s"%msg)
+            return
+        handler = self.monitor_handlers.get(switch, None)
+        if handler is not None:
+            handler(idents, msg)
+        else:
+            self.log.error("Invalid monitor topic: %s"%switch)
+        
+    
+    def dispatch_query(self, msg):
+        """Route registration requests and queries from clients."""
+        idents, msg = self.session.feed_identities(msg)
+        if not idents:
+            self.log.error("Bad Query Message: %s"%msg)
+            return
+        client_id = idents[0]
+        try:
+            msg = self.session.unpack_message(msg, content=True)
+        except:
+            content = error.wrap_exception()
+            self.log.error("Bad Query Message: %s"%msg, exc_info=True)
+            self.session.send(self.query, "hub_error", ident=client_id, 
+                    content=content)
+            return
+        
+        # print client_id, header, parent, content
+        #switch on message type:
+        msg_type = msg['msg_type']
+        self.log.info("client::client %s requested %s"%(client_id, msg_type))
+        handler = self.query_handlers.get(msg_type, None)
+        try:
+            assert handler is not None, "Bad Message Type: %s"%msg_type
+        except:
+            content = error.wrap_exception()
+            self.log.error("Bad Message Type: %s"%msg_type, exc_info=True)
+            self.session.send(self.query, "hub_error", ident=client_id, 
+                    content=content)
+            return
+        else:
+            handler(idents, msg)
+            
+    def dispatch_db(self, msg):
+        """"""
+        raise NotImplementedError
+    
+    #---------------------------------------------------------------------------
+    # handler methods (1 per event)
+    #---------------------------------------------------------------------------
+    
+    #----------------------- Heartbeat --------------------------------------
+    
+    def handle_new_heart(self, heart):
+        """handler to attach to heartbeater.
+        Called when a new heart starts to beat.
+        Triggers completion of registration."""
+        self.log.debug("heartbeat::handle_new_heart(%r)"%heart)
+        if heart not in self.incoming_registrations:
+            self.log.info("heartbeat::ignoring new heart: %r"%heart)
+        else:
+            self.finish_registration(heart)
+        
+    
+    def handle_heart_failure(self, heart):
+        """handler to attach to heartbeater.
+        called when a previously registered heart fails to respond to beat request.
+        triggers unregistration"""
+        self.log.debug("heartbeat::handle_heart_failure(%r)"%heart)
+        eid = self.hearts.get(heart, None)
+        queue = self.engines[eid].queue
+        if eid is None:
+            self.log.info("heartbeat::ignoring heart failure %r"%heart)
+        else:
+            self.unregister_engine(heart, dict(content=dict(id=eid, queue=queue)))
+    
+    #----------------------- MUX Queue Traffic ------------------------------
+    
+    def save_queue_request(self, idents, msg):
+        if len(idents) < 2:
+            self.log.error("invalid identity prefix: %s"%idents)
+            return
+        queue_id, client_id = idents[:2]
+        try:
+            msg = self.session.unpack_message(msg, content=False)
+        except:
+            self.log.error("queue::client %r sent invalid message to %r: %s"%(client_id, queue_id, msg), exc_info=True)
+            return
+        
+        eid = self.by_ident.get(queue_id, None)
+        if eid is None:
+            self.log.error("queue::target %r not registered"%queue_id)
+            self.log.debug("queue::    valid are: %s"%(self.by_ident.keys()))
+            return
+            
+        header = msg['header']
+        msg_id = header['msg_id']
+        record = init_record(msg)
+        record['engine_uuid'] = queue_id
+        record['client_uuid'] = client_id
+        record['queue'] = 'mux'
+
+        try:
+            # it's posible iopub arrived first:
+            existing = self.db.get_record(msg_id)
+            for key,evalue in existing.iteritems():
+                rvalue = record[key]
+                if evalue and rvalue and evalue != rvalue:
+                    self.log.error("conflicting initial state for record: %s:%s <> %s"%(msg_id, rvalue, evalue))
+                elif evalue and not rvalue:
+                    record[key] = evalue
+            self.db.update_record(msg_id, record)
+        except KeyError:
+            self.db.add_record(msg_id, record)
+        
+        self.pending.add(msg_id)
+        self.queues[eid].append(msg_id)
+    
+    def save_queue_result(self, idents, msg):
+        if len(idents) < 2:
+            self.log.error("invalid identity prefix: %s"%idents)
+            return
+            
+        client_id, queue_id = idents[:2]
+        try:
+            msg = self.session.unpack_message(msg, content=False)
+        except:
+            self.log.error("queue::engine %r sent invalid message to %r: %s"%(
+                    queue_id,client_id, msg), exc_info=True)
+            return
+        
+        eid = self.by_ident.get(queue_id, None)
+        if eid is None:
+            self.log.error("queue::unknown engine %r is sending a reply: "%queue_id)
+            # self.log.debug("queue::       %s"%msg[2:])
+            return
+        
+        parent = msg['parent_header']
+        if not parent:
+            return
+        msg_id = parent['msg_id']
+        if msg_id in self.pending:
+            self.pending.remove(msg_id)
+            self.all_completed.add(msg_id)
+            self.queues[eid].remove(msg_id)
+            self.completed[eid].append(msg_id)
+        elif msg_id not in self.all_completed:
+            # it could be a result from a dead engine that died before delivering the
+            # result
+            self.log.warn("queue:: unknown msg finished %s"%msg_id)
+            return
+        # update record anyway, because the unregistration could have been premature
+        rheader = msg['header']
+        completed = datetime.strptime(rheader['date'], ISO8601)
+        started = rheader.get('started', None)
+        if started is not None:
+            started = datetime.strptime(started, ISO8601)
+        result = {
+            'result_header' : rheader,
+            'result_content': msg['content'],
+            'started' : started,
+            'completed' : completed
+        }
+
+        result['result_buffers'] = msg['buffers']
+        self.db.update_record(msg_id, result)
+        
+            
+    #--------------------- Task Queue Traffic ------------------------------
+    
+    def save_task_request(self, idents, msg):
+        """Save the submission of a task."""
+        client_id = idents[0]
+        
+        try:
+            msg = self.session.unpack_message(msg, content=False)
+        except:
+            self.log.error("task::client %r sent invalid task message: %s"%(
+                    client_id, msg), exc_info=True)
+            return
+        record = init_record(msg)
+
+        record['client_uuid'] = client_id
+        record['queue'] = 'task'
+        header = msg['header']
+        msg_id = header['msg_id']
+        self.pending.add(msg_id)
+        try:
+            # it's posible iopub arrived first:
+            existing = self.db.get_record(msg_id)
+            for key,evalue in existing.iteritems():
+                rvalue = record[key]
+                if evalue and rvalue and evalue != rvalue:
+                    self.log.error("conflicting initial state for record: %s:%s <> %s"%(msg_id, rvalue, evalue))
+                elif evalue and not rvalue:
+                    record[key] = evalue
+            self.db.update_record(msg_id, record)
+        except KeyError:
+            self.db.add_record(msg_id, record)
+    
+    def save_task_result(self, idents, msg):
+        """save the result of a completed task."""
+        client_id = idents[0]
+        try:
+            msg = self.session.unpack_message(msg, content=False)
+        except:
+            self.log.error("task::invalid task result message send to %r: %s"%(
+                    client_id, msg), exc_info=True)
+            raise
+            return
+        
+        parent = msg['parent_header']
+        if not parent:
+            # print msg
+            self.log.warn("Task %r had no parent!"%msg)
+            return
+        msg_id = parent['msg_id']
+        
+        header = msg['header']
+        engine_uuid = header.get('engine', None)
+        eid = self.by_ident.get(engine_uuid, None)
+        
+        if msg_id in self.pending:
+            self.pending.remove(msg_id)
+            self.all_completed.add(msg_id)
+            if eid is not None:
+                self.completed[eid].append(msg_id)
+                if msg_id in self.tasks[eid]:
+                    self.tasks[eid].remove(msg_id)
+            completed = datetime.strptime(header['date'], ISO8601)
+            started = header.get('started', None)
+            if started is not None:
+                started = datetime.strptime(started, ISO8601)
+            result = {
+                'result_header' : header,
+                'result_content': msg['content'],
+                'started' : started,
+                'completed' : completed,
+                'engine_uuid': engine_uuid
+            }
+
+            result['result_buffers'] = msg['buffers']
+            self.db.update_record(msg_id, result)
+            
+        else:
+            self.log.debug("task::unknown task %s finished"%msg_id)
+    
+    def save_task_destination(self, idents, msg):
+        try:
+            msg = self.session.unpack_message(msg, content=True)
+        except:
+            self.log.error("task::invalid task tracking message", exc_info=True)
+            return
+        content = msg['content']
+        # print (content)
+        msg_id = content['msg_id']
+        engine_uuid = content['engine_id']
+        eid = self.by_ident[engine_uuid]
+        
+        self.log.info("task::task %s arrived on %s"%(msg_id, eid))
+        # if msg_id in self.mia:
+        #     self.mia.remove(msg_id)
+        # else:
+        #     self.log.debug("task::task %s not listed as MIA?!"%(msg_id))
+        
+        self.tasks[eid].append(msg_id)
+        # self.pending[msg_id][1].update(received=datetime.now(),engine=(eid,engine_uuid))
+        self.db.update_record(msg_id, dict(engine_uuid=engine_uuid))
+    
+    def mia_task_request(self, idents, msg):
+        raise NotImplementedError
+        client_id = idents[0]
+        # content = dict(mia=self.mia,status='ok')
+        # self.session.send('mia_reply', content=content, idents=client_id)
+        
+        
+    #--------------------- IOPub Traffic ------------------------------
+    
+    def save_iopub_message(self, topics, msg):
+        """save an iopub message into the db"""
+        # print (topics)
+        try:
+            msg = self.session.unpack_message(msg, content=True)
+        except:
+            self.log.error("iopub::invalid IOPub message", exc_info=True)
+            return
+        
+        parent = msg['parent_header']
+        if not parent:
+            self.log.error("iopub::invalid IOPub message: %s"%msg)
+            return
+        msg_id = parent['msg_id']
+        msg_type = msg['msg_type']
+        content = msg['content']
+        
+        # ensure msg_id is in db
+        try:
+            rec = self.db.get_record(msg_id)
+        except KeyError:
+            rec = empty_record()
+            rec['msg_id'] = msg_id
+            self.db.add_record(msg_id, rec)
+        # stream
+        d = {}
+        if msg_type == 'stream':
+            name = content['name']
+            s = rec[name] or ''
+            d[name] = s + content['data']
+            
+        elif msg_type == 'pyerr':
+            d['pyerr'] = content
+        else:
+            d[msg_type] = content['data']
+        
+        self.db.update_record(msg_id, d)
+        
+    
+            
+    #-------------------------------------------------------------------------
+    # Registration requests
+    #-------------------------------------------------------------------------
+        
+    def connection_request(self, client_id, msg):
+        """Reply with connection addresses for clients."""
+        self.log.info("client::client %s connected"%client_id)
+        content = dict(status='ok')
+        content.update(self.client_info)
+        jsonable = {}
+        for k,v in self.keytable.iteritems():
+            if v not in self.dead_engines:
+                jsonable[str(k)] = v
+        content['engines'] = jsonable
+        self.session.send(self.query, 'connection_reply', content, parent=msg, ident=client_id)
+    
+    def register_engine(self, reg, msg):
+        """Register a new engine."""
+        content = msg['content']
+        try:
+            queue = content['queue']
+        except KeyError:
+            self.log.error("registration::queue not specified", exc_info=True)
+            return
+        heart = content.get('heartbeat', None)
+        """register a new engine, and create the socket(s) necessary"""
+        eid = self._next_id
+        # print (eid, queue, reg, heart)
+        
+        self.log.debug("registration::register_engine(%i, %r, %r, %r)"%(eid, queue, reg, heart))
+        
+        content = dict(id=eid,status='ok')
+        content.update(self.engine_info)
+        # check if requesting available IDs:
+        if queue in self.by_ident:
+            try:
+                raise KeyError("queue_id %r in use"%queue)
+            except:
+                content = error.wrap_exception()
+                self.log.error("queue_id %r in use"%queue, exc_info=True)
+        elif heart in self.hearts: # need to check unique hearts?
+            try:
+                raise KeyError("heart_id %r in use"%heart)
+            except:
+                self.log.error("heart_id %r in use"%heart, exc_info=True)
+                content = error.wrap_exception()
+        else:
+            for h, pack in self.incoming_registrations.iteritems():
+                if heart == h:
+                    try:
+                        raise KeyError("heart_id %r in use"%heart)
+                    except:
+                        self.log.error("heart_id %r in use"%heart, exc_info=True)
+                        content = error.wrap_exception()
+                    break
+                elif queue == pack[1]:
+                    try:
+                        raise KeyError("queue_id %r in use"%queue)
+                    except:
+                        self.log.error("queue_id %r in use"%queue, exc_info=True)
+                        content = error.wrap_exception()
+                    break
+        
+        msg = self.session.send(self.query, "registration_reply", 
+                content=content, 
+                ident=reg)
+        
+        if content['status'] == 'ok':
+            if heart in self.heartmonitor.hearts:
+                # already beating
+                self.incoming_registrations[heart] = (eid,queue,reg[0],None)
+                self.finish_registration(heart)
+            else:
+                purge = lambda : self._purge_stalled_registration(heart)
+                dc = ioloop.DelayedCallback(purge, self.registration_timeout, self.loop)
+                dc.start()
+                self.incoming_registrations[heart] = (eid,queue,reg[0],dc)
+        else:
+            self.log.error("registration::registration %i failed: %s"%(eid, content['evalue']))
+        return eid
+    
+    def unregister_engine(self, ident, msg):
+        """Unregister an engine that explicitly requested to leave."""
+        try:
+            eid = msg['content']['id']
+        except:
+            self.log.error("registration::bad engine id for unregistration: %s"%ident, exc_info=True)
+            return
+        self.log.info("registration::unregister_engine(%s)"%eid)
+        # print (eid)
+        uuid = self.keytable[eid]
+        content=dict(id=eid, queue=uuid)
+        self.dead_engines.add(uuid)
+        # self.ids.remove(eid)
+        # uuid = self.keytable.pop(eid)
+        # 
+        # ec = self.engines.pop(eid)
+        # self.hearts.pop(ec.heartbeat)
+        # self.by_ident.pop(ec.queue)
+        # self.completed.pop(eid)
+        handleit = lambda : self._handle_stranded_msgs(eid, uuid)
+        dc = ioloop.DelayedCallback(handleit, self.registration_timeout, self.loop)
+        dc.start()
+        ############## TODO: HANDLE IT ################
+        
+        if self.notifier:
+            self.session.send(self.notifier, "unregistration_notification", content=content)
+    
+    def _handle_stranded_msgs(self, eid, uuid):
+        """Handle messages known to be on an engine when the engine unregisters.
+        
+        It is possible that this will fire prematurely - that is, an engine will
+        go down after completing a result, and the client will be notified
+        that the result failed and later receive the actual result.
+        """
+        
+        outstanding = self.queues[eid]
+        
+        for msg_id in outstanding:
+            self.pending.remove(msg_id)
+            self.all_completed.add(msg_id)
+            try:
+                raise error.EngineError("Engine %r died while running task %r"%(eid, msg_id))
+            except:
+                content = error.wrap_exception()
+            # build a fake header:
+            header = {}
+            header['engine'] = uuid
+            header['date'] = datetime.now().strftime(ISO8601)
+            rec = dict(result_content=content, result_header=header, result_buffers=[])
+            rec['completed'] = header['date']
+            rec['engine_uuid'] = uuid
+            self.db.update_record(msg_id, rec)
+    
+    def finish_registration(self, heart):
+        """Second half of engine registration, called after our HeartMonitor
+        has received a beat from the Engine's Heart."""
+        try: 
+            (eid,queue,reg,purge) = self.incoming_registrations.pop(heart)
+        except KeyError:
+            self.log.error("registration::tried to finish nonexistant registration", exc_info=True)
+            return
+        self.log.info("registration::finished registering engine %i:%r"%(eid,queue))
+        if purge is not None:
+            purge.stop()
+        control = queue
+        self.ids.add(eid)
+        self.keytable[eid] = queue
+        self.engines[eid] = EngineConnector(id=eid, queue=queue, registration=reg, 
+                                    control=control, heartbeat=heart)
+        self.by_ident[queue] = eid
+        self.queues[eid] = list()
+        self.tasks[eid] = list()
+        self.completed[eid] = list()
+        self.hearts[heart] = eid
+        content = dict(id=eid, queue=self.engines[eid].queue)
+        if self.notifier:
+            self.session.send(self.notifier, "registration_notification", content=content)
+        self.log.info("engine::Engine Connected: %i"%eid)
+    
+    def _purge_stalled_registration(self, heart):
+        if heart in self.incoming_registrations:
+            eid = self.incoming_registrations.pop(heart)[0]
+            self.log.info("registration::purging stalled registration: %i"%eid)
+        else:
+            pass
+            
+    #-------------------------------------------------------------------------
+    # Client Requests
+    #-------------------------------------------------------------------------
+    
+    def shutdown_request(self, client_id, msg):
+        """handle shutdown request."""
+        self.session.send(self.query, 'shutdown_reply', content={'status': 'ok'}, ident=client_id)
+        # also notify other clients of shutdown
+        self.session.send(self.notifier, 'shutdown_notice', content={'status': 'ok'})
+        dc = ioloop.DelayedCallback(lambda : self._shutdown(), 1000, self.loop)
+        dc.start()
+    
+    def _shutdown(self):
+        self.log.info("hub::hub shutting down.")
+        time.sleep(0.1)
+        sys.exit(0)
+        
+    
+    def check_load(self, client_id, msg):
+        content = msg['content']
+        try:
+            targets = content['targets']
+            targets = self._validate_targets(targets)
+        except:
+            content = error.wrap_exception()
+            self.session.send(self.query, "hub_error", 
+                    content=content, ident=client_id)
+            return
+        
+        content = dict(status='ok')
+        # loads = {}
+        for t in targets:
+            content[bytes(t)] = len(self.queues[t])+len(self.tasks[t])
+        self.session.send(self.query, "load_reply", content=content, ident=client_id)
+            
+    
+    def queue_status(self, client_id, msg):
+        """Return the Queue status of one or more targets.
+        if verbose: return the msg_ids
+        else: return len of each type.
+        keys: queue (pending MUX jobs)
+            tasks (pending Task jobs)
+            completed (finished jobs from both queues)"""
+        content = msg['content']
+        targets = content['targets']
+        try:
+            targets = self._validate_targets(targets)
+        except:
+            content = error.wrap_exception()
+            self.session.send(self.query, "hub_error", 
+                    content=content, ident=client_id)
+            return
+        verbose = content.get('verbose', False)
+        content = dict(status='ok')
+        for t in targets:
+            queue = self.queues[t]
+            completed = self.completed[t]
+            tasks = self.tasks[t]
+            if not verbose:
+                queue = len(queue)
+                completed = len(completed)
+                tasks = len(tasks)
+            content[bytes(t)] = {'queue': queue, 'completed': completed , 'tasks': tasks}
+            # pending
+        self.session.send(self.query, "queue_reply", content=content, ident=client_id)
+    
+    def purge_results(self, client_id, msg):
+        """Purge results from memory. This method is more valuable before we move
+        to a DB based message storage mechanism."""
+        content = msg['content']
+        msg_ids = content.get('msg_ids', [])
+        reply = dict(status='ok')
+        if msg_ids == 'all':
+            self.db.drop_matching_records(dict(completed={'$ne':None}))
+        else:
+            for msg_id in msg_ids:
+                if msg_id in self.all_completed:
+                    self.db.drop_record(msg_id)
+                else:
+                    if msg_id in self.pending:
+                        try:
+                            raise IndexError("msg pending: %r"%msg_id)
+                        except:
+                            reply = error.wrap_exception()
+                    else:
+                        try:
+                            raise IndexError("No such msg: %r"%msg_id)
+                        except:
+                            reply = error.wrap_exception()
+                    break
+            eids = content.get('engine_ids', [])
+            for eid in eids:
+                if eid not in self.engines:
+                    try:
+                        raise IndexError("No such engine: %i"%eid)
+                    except:
+                        reply = error.wrap_exception()
+                    break
+                msg_ids = self.completed.pop(eid)
+                uid = self.engines[eid].queue
+                self.db.drop_matching_records(dict(engine_uuid=uid, completed={'$ne':None}))
+        
+        self.session.send(self.query, 'purge_reply', content=reply, ident=client_id)
+    
+    def resubmit_task(self, client_id, msg, buffers):
+        """Resubmit a task."""
+        raise NotImplementedError
+    
+    def get_results(self, client_id, msg):
+        """Get the result of 1 or more messages."""
+        content = msg['content']
+        msg_ids = sorted(set(content['msg_ids']))
+        statusonly = content.get('status_only', False)
+        pending = []
+        completed = []
+        content = dict(status='ok')
+        content['pending'] = pending
+        content['completed'] = completed
+        buffers = []
+        if not statusonly:
+            content['results'] = {}
+            records = self.db.find_records(dict(msg_id={'$in':msg_ids}))
+        for msg_id in msg_ids:
+            if msg_id in self.pending:
+                pending.append(msg_id)
+            elif msg_id in self.all_completed:
+                completed.append(msg_id)
+                if not statusonly:
+                    rec = records[msg_id]
+                    io_dict = {}
+                    for key in 'pyin pyout pyerr stdout stderr'.split():
+                            io_dict[key] = rec[key]
+                    content[msg_id] = { 'result_content': rec['result_content'],
+                                        'header': rec['header'],
+                                        'result_header' : rec['result_header'],
+                                        'io' : io_dict,
+                                      }
+                    if rec['result_buffers']:
+                        buffers.extend(map(str, rec['result_buffers']))
+            else:
+                try:
+                    raise KeyError('No such message: '+msg_id)
+                except:
+                    content = error.wrap_exception()
+                break
+        self.session.send(self.query, "result_reply", content=content, 
+                                            parent=msg, ident=client_id,
+                                            buffers=buffers)
+

IPython/parallel/controller/mongodb.py

@@ -0,0 +1,80 b''
+"""A TaskRecord backend using mongodb"""
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2010  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+from datetime import datetime
+
+from pymongo import Connection
+from pymongo.binary import Binary
+
+from IPython.utils.traitlets import Dict, List, CUnicode
+
+from .dictdb import BaseDB
+
+#-----------------------------------------------------------------------------
+# MongoDB class
+#-----------------------------------------------------------------------------
+
+class MongoDB(BaseDB):
+    """MongoDB TaskRecord backend."""
+    
+    connection_args = List(config=True)
+    connection_kwargs = Dict(config=True)
+    database = CUnicode(config=True)
+    _table = Dict()
+    
+    def __init__(self, **kwargs):
+        super(MongoDB, self).__init__(**kwargs)
+        self._connection = Connection(*self.connection_args, **self.connection_kwargs)
+        if not self.database:
+            self.database = self.session
+        self._db = self._connection[self.database]
+        self._records = self._db['task_records']
+    
+    def _binary_buffers(self, rec):
+        for key in ('buffers', 'result_buffers'):
+            if key in rec:
+                rec[key] = map(Binary, rec[key])
+    
+    def add_record(self, msg_id, rec):
+        """Add a new Task Record, by msg_id."""
+        # print rec
+        rec = _binary_buffers(rec)
+        obj_id = self._records.insert(rec)
+        self._table[msg_id] = obj_id
+    
+    def get_record(self, msg_id):
+        """Get a specific Task Record, by msg_id."""
+        return self._records.find_one(self._table[msg_id])
+    
+    def update_record(self, msg_id, rec):
+        """Update the data in an existing record."""
+        rec = _binary_buffers(rec)
+        obj_id = self._table[msg_id]
+        self._records.update({'_id':obj_id}, {'$set': rec})
+    
+    def drop_matching_records(self, check):
+        """Remove a record from the DB."""
+        self._records.remove(check)
+        
+    def drop_record(self, msg_id):
+        """Remove a record from the DB."""
+        obj_id = self._table.pop(msg_id)
+        self._records.remove(obj_id)
+    
+    def find_records(self, check, id_only=False):
+        """Find records matching a query dict."""
+        matches = list(self._records.find(check))
+        if id_only:
+            return [ rec['msg_id'] for rec in matches ]
+        else:
+            data = {}
+            for rec in matches:
+                data[rec['msg_id']] = rec
+            return data
+
+

IPython/parallel/controller/scheduler.py

@@ -0,0 +1,592 b''
+"""The Python scheduler for rich scheduling.
+
+The Pure ZMQ scheduler does not allow routing schemes other than LRU,
+nor does it check msg_id DAG dependencies. For those, a slightly slower
+Python Scheduler exists.
+"""
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2010-2011  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#----------------------------------------------------------------------
+# Imports
+#----------------------------------------------------------------------
+
+from __future__ import print_function
+
+import logging
+import sys
+
+from datetime import datetime, timedelta
+from random import randint, random
+from types import FunctionType
+
+try:
+    import numpy
+except ImportError:
+    numpy = None
+
+import zmq
+from zmq.eventloop import ioloop, zmqstream
+
+# local imports
+from IPython.external.decorator import decorator
+from IPython.utils.traitlets import Instance, Dict, List, Set
+
+from IPython.parallel import error
+from IPython.parallel.factory import SessionFactory
+from IPython.parallel.util import connect_logger, local_logger
+
+from .dependency import Dependency
+
+@decorator
+def logged(f,self,*args,**kwargs):
+    # print ("#--------------------")
+    self.log.debug("scheduler::%s(*%s,**%s)"%(f.func_name, args, kwargs))
+    # print ("#--")
+    return f(self,*args, **kwargs)
+
+#----------------------------------------------------------------------
+# Chooser functions
+#----------------------------------------------------------------------
+
+def plainrandom(loads):
+    """Plain random pick."""
+    n = len(loads)
+    return randint(0,n-1)
+
+def lru(loads):
+    """Always pick the front of the line.
+    
+    The content of `loads` is ignored.
+    
+    Assumes LRU ordering of loads, with oldest first.
+    """
+    return 0
+
+def twobin(loads):
+    """Pick two at random, use the LRU of the two.
+    
+    The content of loads is ignored.
+    
+    Assumes LRU ordering of loads, with oldest first.
+    """
+    n = len(loads)
+    a = randint(0,n-1)
+    b = randint(0,n-1)
+    return min(a,b)
+
+def weighted(loads):
+    """Pick two at random using inverse load as weight.
+    
+    Return the less loaded of the two.
+    """
+    # weight 0 a million times more than 1:
+    weights = 1./(1e-6+numpy.array(loads))
+    sums = weights.cumsum()
+    t = sums[-1]
+    x = random()*t
+    y = random()*t
+    idx = 0
+    idy = 0
+    while sums[idx] < x:
+        idx += 1
+    while sums[idy] < y:
+        idy += 1
+    if weights[idy] > weights[idx]:
+        return idy
+    else:
+        return idx
+
+def leastload(loads):
+    """Always choose the lowest load.
+    
+    If the lowest load occurs more than once, the first
+    occurance will be used.  If loads has LRU ordering, this means
+    the LRU of those with the lowest load is chosen.
+    """
+    return loads.index(min(loads))
+
+#---------------------------------------------------------------------
+# Classes
+#---------------------------------------------------------------------
+# store empty default dependency:
+MET = Dependency([])
+
+class TaskScheduler(SessionFactory):
+    """Python TaskScheduler object.
+    
+    This is the simplest object that supports msg_id based
+    DAG dependencies. *Only* task msg_ids are checked, not
+    msg_ids of jobs submitted via the MUX queue.
+    
+    """
+    
+    # input arguments:
+    scheme = Instance(FunctionType, default=leastload) # function for determining the destination
+    client_stream = Instance(zmqstream.ZMQStream) # client-facing stream
+    engine_stream = Instance(zmqstream.ZMQStream) # engine-facing stream
+    notifier_stream = Instance(zmqstream.ZMQStream) # hub-facing sub stream
+    mon_stream = Instance(zmqstream.ZMQStream) # hub-facing pub stream
+    
+    # internals:
+    graph = Dict() # dict by msg_id of [ msg_ids that depend on key ]
+    depending = Dict() # dict by msg_id of (msg_id, raw_msg, after, follow)
+    pending = Dict() # dict by engine_uuid of submitted tasks
+    completed = Dict() # dict by engine_uuid of completed tasks
+    failed = Dict() # dict by engine_uuid of failed tasks
+    destinations = Dict() # dict by msg_id of engine_uuids where jobs ran (reverse of completed+failed)
+    clients = Dict() # dict by msg_id for who submitted the task
+    targets = List() # list of target IDENTs
+    loads = List() # list of engine loads
+    all_completed = Set() # set of all completed tasks
+    all_failed = Set() # set of all failed tasks
+    all_done = Set() # set of all finished tasks=union(completed,failed)
+    all_ids = Set() # set of all submitted task IDs
+    blacklist = Dict() # dict by msg_id of locations where a job has encountered UnmetDependency
+    auditor = Instance('zmq.eventloop.ioloop.PeriodicCallback')
+    
+    
+    def start(self):
+        self.engine_stream.on_recv(self.dispatch_result, copy=False)
+        self._notification_handlers = dict(
+            registration_notification = self._register_engine,
+            unregistration_notification = self._unregister_engine
+        )
+        self.notifier_stream.on_recv(self.dispatch_notification)
+        self.auditor = ioloop.PeriodicCallback(self.audit_timeouts, 2e3, self.loop) # 1 Hz
+        self.auditor.start()
+        self.log.info("Scheduler started...%r"%self)
+    
+    def resume_receiving(self):
+        """Resume accepting jobs."""
+        self.client_stream.on_recv(self.dispatch_submission, copy=False)
+    
+    def stop_receiving(self):
+        """Stop accepting jobs while there are no engines.
+        Leave them in the ZMQ queue."""
+        self.client_stream.on_recv(None)
+    
+    #-----------------------------------------------------------------------
+    # [Un]Registration Handling
+    #-----------------------------------------------------------------------
+    
+    def dispatch_notification(self, msg):
+        """dispatch register/unregister events."""
+        idents,msg = self.session.feed_identities(msg)
+        msg = self.session.unpack_message(msg)
+        msg_type = msg['msg_type']
+        handler = self._notification_handlers.get(msg_type, None)
+        if handler is None:
+            raise Exception("Unhandled message type: %s"%msg_type)
+        else:
+            try:
+                handler(str(msg['content']['queue']))
+            except KeyError:
+                self.log.error("task::Invalid notification msg: %s"%msg)
+    
+    @logged
+    def _register_engine(self, uid):
+        """New engine with ident `uid` became available."""
+        # head of the line:
+        self.targets.insert(0,uid)
+        self.loads.insert(0,0)
+        # initialize sets
+        self.completed[uid] = set()
+        self.failed[uid] = set()
+        self.pending[uid] = {}
+        if len(self.targets) == 1:
+            self.resume_receiving()
+
+    def _unregister_engine(self, uid):
+        """Existing engine with ident `uid` became unavailable."""
+        if len(self.targets) == 1:
+            # this was our only engine
+            self.stop_receiving()
+        
+        # handle any potentially finished tasks:
+        self.engine_stream.flush()
+        
+        self.completed.pop(uid)
+        self.failed.pop(uid)
+        # don't pop destinations, because it might be used later
+        # map(self.destinations.pop, self.completed.pop(uid))
+        # map(self.destinations.pop, self.failed.pop(uid))
+        
+        idx = self.targets.index(uid)
+        self.targets.pop(idx)
+        self.loads.pop(idx)
+        
+        # wait 5 seconds before cleaning up pending jobs, since the results might
+        # still be incoming
+        if self.pending[uid]:
+            dc = ioloop.DelayedCallback(lambda : self.handle_stranded_tasks(uid), 5000, self.loop)
+            dc.start()
+    
+    @logged
+    def handle_stranded_tasks(self, engine):
+        """Deal with jobs resident in an engine that died."""
+        lost = self.pending.pop(engine)
+        
+        for msg_id, (raw_msg, targets, MET, follow, timeout) in lost.iteritems():
+            self.all_failed.add(msg_id)
+            self.all_done.add(msg_id)
+            idents,msg = self.session.feed_identities(raw_msg, copy=False)
+            msg = self.session.unpack_message(msg, copy=False, content=False)
+            parent = msg['header']
+            idents = [idents[0],engine]+idents[1:]
+            # print (idents)
+            try:
+                raise error.EngineError("Engine %r died while running task %r"%(engine, msg_id))
+            except:
+                content = error.wrap_exception()
+            msg = self.session.send(self.client_stream, 'apply_reply', content, 
+                                                    parent=parent, ident=idents)
+            self.session.send(self.mon_stream, msg, ident=['outtask']+idents)
+            self.update_graph(msg_id)
+    
+    
+    #-----------------------------------------------------------------------
+    # Job Submission
+    #-----------------------------------------------------------------------
+    @logged
+    def dispatch_submission(self, raw_msg):
+        """Dispatch job submission to appropriate handlers."""
+        # ensure targets up to date:
+        self.notifier_stream.flush()
+        try:
+            idents, msg = self.session.feed_identities(raw_msg, copy=False)
+            msg = self.session.unpack_message(msg, content=False, copy=False)
+        except:
+            self.log.error("task::Invaid task: %s"%raw_msg, exc_info=True)
+            return
+        
+        # send to monitor
+        self.mon_stream.send_multipart(['intask']+raw_msg, copy=False)
+        
+        header = msg['header']
+        msg_id = header['msg_id']
+        self.all_ids.add(msg_id)
+        
+        # targets
+        targets = set(header.get('targets', []))
+        
+        # time dependencies
+        after = Dependency(header.get('after', []))
+        if after.all:
+            if after.success:
+                after.difference_update(self.all_completed)
+            if after.failure:
+                after.difference_update(self.all_failed)
+        if after.check(self.all_completed, self.all_failed):
+            # recast as empty set, if `after` already met,
+            # to prevent unnecessary set comparisons
+            after = MET
+        
+        # location dependencies
+        follow = Dependency(header.get('follow', []))
+        
+        # turn timeouts into datetime objects:
+        timeout = header.get('timeout', None)
+        if timeout:
+            timeout = datetime.now() + timedelta(0,timeout,0)
+        
+        args = [raw_msg, targets, after, follow, timeout]
+        
+        # validate and reduce dependencies:
+        for dep in after,follow:
+            # check valid:
+            if msg_id in dep or dep.difference(self.all_ids):
+                self.depending[msg_id] = args
+                return self.fail_unreachable(msg_id, error.InvalidDependency)
+            # check if unreachable:
+            if dep.unreachable(self.all_completed, self.all_failed):
+                self.depending[msg_id] = args
+                return self.fail_unreachable(msg_id)
+        
+        if after.check(self.all_completed, self.all_failed):
+            # time deps already met, try to run
+            if not self.maybe_run(msg_id, *args):
+                # can't run yet
+                self.save_unmet(msg_id, *args)
+        else:
+            self.save_unmet(msg_id, *args)
+    
+    # @logged
+    def audit_timeouts(self):
+        """Audit all waiting tasks for expired timeouts."""
+        now = datetime.now()
+        for msg_id in self.depending.keys():
+            # must recheck, in case one failure cascaded to another:
+            if msg_id in self.depending:
+                raw,after,targets,follow,timeout = self.depending[msg_id]
+                if timeout and timeout < now:
+                    self.fail_unreachable(msg_id, timeout=True)
+                
+    @logged
+    def fail_unreachable(self, msg_id, why=error.ImpossibleDependency):
+        """a task has become unreachable, send a reply with an ImpossibleDependency
+        error."""
+        if msg_id not in self.depending:
+            self.log.error("msg %r already failed!"%msg_id)
+            return
+        raw_msg,targets,after,follow,timeout = self.depending.pop(msg_id)
+        for mid in follow.union(after):
+            if mid in self.graph:
+                self.graph[mid].remove(msg_id)
+        
+        # FIXME: unpacking a message I've already unpacked, but didn't save:
+        idents,msg = self.session.feed_identities(raw_msg, copy=False)
+        msg = self.session.unpack_message(msg, copy=False, content=False)
+        header = msg['header']
+        
+        try:
+            raise why()
+        except:
+            content = error.wrap_exception()
+        
+        self.all_done.add(msg_id)
+        self.all_failed.add(msg_id)
+        
+        msg = self.session.send(self.client_stream, 'apply_reply', content, 
+                                                parent=header, ident=idents)
+        self.session.send(self.mon_stream, msg, ident=['outtask']+idents)
+        
+        self.update_graph(msg_id, success=False)
+    
+    @logged
+    def maybe_run(self, msg_id, raw_msg, targets, after, follow, timeout):
+        """check location dependencies, and run if they are met."""
+        blacklist = self.blacklist.setdefault(msg_id, set())
+        if follow or targets or blacklist:
+            # we need a can_run filter
+            def can_run(idx):
+                target = self.targets[idx]
+                # check targets
+                if targets and target not in targets:
+                    return False
+                # check blacklist
+                if target in blacklist:
+                    return False
+                # check follow
+                return follow.check(self.completed[target], self.failed[target])
+            
+            indices = filter(can_run, range(len(self.targets)))
+            if not indices:
+                # couldn't run
+                if follow.all:
+                    # check follow for impossibility
+                    dests = set()
+                    relevant = set()
+                    if follow.success:
+                        relevant = self.all_completed
+                    if follow.failure:
+                        relevant = relevant.union(self.all_failed)
+                    for m in follow.intersection(relevant):
+                        dests.add(self.destinations[m])
+                    if len(dests) > 1:
+                        self.fail_unreachable(msg_id)
+                        return False
+                if targets:
+                    # check blacklist+targets for impossibility
+                    targets.difference_update(blacklist)
+                    if not targets or not targets.intersection(self.targets):
+                        self.fail_unreachable(msg_id)
+                        return False
+                return False
+        else:
+            indices = None
+            
+        self.submit_task(msg_id, raw_msg, targets, follow, timeout, indices)
+        return True
+            
+    @logged
+    def save_unmet(self, msg_id, raw_msg, targets, after, follow, timeout):
+        """Save a message for later submission when its dependencies are met."""
+        self.depending[msg_id] = [raw_msg,targets,after,follow,timeout]
+        # track the ids in follow or after, but not those already finished
+        for dep_id in after.union(follow).difference(self.all_done):
+            if dep_id not in self.graph:
+                self.graph[dep_id] = set()
+            self.graph[dep_id].add(msg_id)
+    
+    @logged
+    def submit_task(self, msg_id, raw_msg, targets, follow, timeout, indices=None):
+        """Submit a task to any of a subset of our targets."""
+        if indices:
+            loads = [self.loads[i] for i in indices]
+        else:
+            loads = self.loads
+        idx = self.scheme(loads)
+        if indices:
+            idx = indices[idx]
+        target = self.targets[idx]
+        # print (target, map(str, msg[:3]))
+        self.engine_stream.send(target, flags=zmq.SNDMORE, copy=False)
+        self.engine_stream.send_multipart(raw_msg, copy=False)
+        self.add_job(idx)
+        self.pending[target][msg_id] = (raw_msg, targets, MET, follow, timeout)
+        content = dict(msg_id=msg_id, engine_id=target)
+        self.session.send(self.mon_stream, 'task_destination', content=content, 
+                        ident=['tracktask',self.session.session])
+    
+    #-----------------------------------------------------------------------
+    # Result Handling
+    #-----------------------------------------------------------------------
+    @logged
+    def dispatch_result(self, raw_msg):
+        """dispatch method for result replies"""
+        try:
+            idents,msg = self.session.feed_identities(raw_msg, copy=False)
+            msg = self.session.unpack_message(msg, content=False, copy=False)
+        except:
+            self.log.error("task::Invaid result: %s"%raw_msg, exc_info=True)
+            return
+        
+        header = msg['header']
+        if header.get('dependencies_met', True):
+            success = (header['status'] == 'ok')
+            self.handle_result(idents, msg['parent_header'], raw_msg, success)
+            # send to Hub monitor
+            self.mon_stream.send_multipart(['outtask']+raw_msg, copy=False)
+        else:
+            self.handle_unmet_dependency(idents, msg['parent_header'])
+        
+    @logged
+    def handle_result(self, idents, parent, raw_msg, success=True):
+        """handle a real task result, either success or failure"""
+        # first, relay result to client
+        engine = idents[0]
+        client = idents[1]
+        # swap_ids for XREP-XREP mirror
+        raw_msg[:2] = [client,engine]
+        # print (map(str, raw_msg[:4]))
+        self.client_stream.send_multipart(raw_msg, copy=False)
+        # now, update our data structures
+        msg_id = parent['msg_id']
+        self.blacklist.pop(msg_id, None)
+        self.pending[engine].pop(msg_id)
+        if success:
+            self.completed[engine].add(msg_id)
+            self.all_completed.add(msg_id)
+        else:
+            self.failed[engine].add(msg_id)
+            self.all_failed.add(msg_id)
+        self.all_done.add(msg_id)
+        self.destinations[msg_id] = engine
+        
+        self.update_graph(msg_id, success)
+        
+    @logged
+    def handle_unmet_dependency(self, idents, parent):
+        """handle an unmet dependency"""
+        engine = idents[0]
+        msg_id = parent['msg_id']
+        
+        if msg_id not in self.blacklist:
+            self.blacklist[msg_id] = set()
+        self.blacklist[msg_id].add(engine)
+        
+        args = self.pending[engine].pop(msg_id)
+        raw,targets,after,follow,timeout = args
+        
+        if self.blacklist[msg_id] == targets:
+            self.depending[msg_id] = args
+            return self.fail_unreachable(msg_id)
+        
+        elif not self.maybe_run(msg_id, *args):
+            # resubmit failed, put it back in our dependency tree
+            self.save_unmet(msg_id, *args)
+        
+    
+    @logged
+    def update_graph(self, dep_id, success=True):
+        """dep_id just finished. Update our dependency
+        graph and submit any jobs that just became runable."""
+        # print ("\n\n***********")
+        # pprint (dep_id)
+        # pprint (self.graph)
+        # pprint (self.depending)
+        # pprint (self.all_completed)
+        # pprint (self.all_failed)
+        # print ("\n\n***********\n\n")
+        if dep_id not in self.graph:
+            return
+        jobs = self.graph.pop(dep_id)
+        
+        for msg_id in jobs:
+            raw_msg, targets, after, follow, timeout = self.depending[msg_id]
+            
+            if after.unreachable(self.all_completed, self.all_failed) or follow.unreachable(self.all_completed, self.all_failed):
+                self.fail_unreachable(msg_id)
+            
+            elif after.check(self.all_completed, self.all_failed): # time deps met, maybe run
+                if self.maybe_run(msg_id, raw_msg, targets, MET, follow, timeout):
+                    
+                    self.depending.pop(msg_id)
+                    for mid in follow.union(after):
+                        if mid in self.graph:
+                            self.graph[mid].remove(msg_id)
+    
+    #----------------------------------------------------------------------
+    # methods to be overridden by subclasses
+    #----------------------------------------------------------------------
+    
+    def add_job(self, idx):
+        """Called after self.targets[idx] just got the job with header.
+        Override with subclasses.  The default ordering is simple LRU.
+        The default loads are the number of outstanding jobs."""
+        self.loads[idx] += 1
+        for lis in (self.targets, self.loads):
+            lis.append(lis.pop(idx))
+            
+    
+    def finish_job(self, idx):
+        """Called after self.targets[idx] just finished a job.
+        Override with subclasses."""
+        self.loads[idx] -= 1
+    
+
+
+def launch_scheduler(in_addr, out_addr, mon_addr, not_addr, config=None,logname='ZMQ', 
+                            log_addr=None, loglevel=logging.DEBUG, scheme='lru',
+                            identity=b'task'):
+    from zmq.eventloop import ioloop
+    from zmq.eventloop.zmqstream import ZMQStream
+    
+    ctx = zmq.Context()
+    loop = ioloop.IOLoop()
+    print (in_addr, out_addr, mon_addr, not_addr)
+    ins = ZMQStream(ctx.socket(zmq.XREP),loop)
+    ins.setsockopt(zmq.IDENTITY, identity)
+    ins.bind(in_addr)
+    
+    outs = ZMQStream(ctx.socket(zmq.XREP),loop)
+    outs.setsockopt(zmq.IDENTITY, identity)
+    outs.bind(out_addr)
+    mons = ZMQStream(ctx.socket(zmq.PUB),loop)
+    mons.connect(mon_addr)
+    nots = ZMQStream(ctx.socket(zmq.SUB),loop)
+    nots.setsockopt(zmq.SUBSCRIBE, '')
+    nots.connect(not_addr)
+    
+    scheme = globals().get(scheme, None)
+    # setup logging
+    if log_addr:
+        connect_logger(logname, ctx, log_addr, root="scheduler", loglevel=loglevel)
+    else:
+        local_logger(logname, loglevel)
+    
+    scheduler = TaskScheduler(client_stream=ins, engine_stream=outs,
+                            mon_stream=mons, notifier_stream=nots,
+                            scheme=scheme, loop=loop, logname=logname,
+                            config=config)
+    scheduler.start()
+    try:
+        loop.start()
+    except KeyboardInterrupt:
+        print ("interrupted, exiting...", file=sys.__stderr__)
+

IPython/parallel/controller/sqlitedb.py

@@ -0,0 +1,284 b''
+"""A TaskRecord backend using sqlite3"""
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2011  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+import json
+import os
+import cPickle as pickle
+from datetime import datetime
+
+import sqlite3
+
+from zmq.eventloop import ioloop
+
+from IPython.utils.traitlets import CUnicode, CStr, Instance, List
+from .dictdb import BaseDB
+from IPython.parallel.util import ISO8601
+
+#-----------------------------------------------------------------------------
+# SQLite operators, adapters, and converters
+#-----------------------------------------------------------------------------
+
+operators = {
+ '$lt' : lambda a,b: "%s < ?",
+ '$gt' : ">",
+ # null is handled weird with ==,!=
+ '$eq' : "IS",
+ '$ne' : "IS NOT",
+ '$lte': "<=",
+ '$gte': ">=",
+ '$in' : ('IS', ' OR '),
+ '$nin': ('IS NOT', ' AND '),
+ # '$all': None,
+ # '$mod': None,
+ # '$exists' : None
+}
+
+def _adapt_datetime(dt):
+    return dt.strftime(ISO8601)
+
+def _convert_datetime(ds):
+    if ds is None:
+        return ds
+    else:
+        return datetime.strptime(ds, ISO8601)
+
+def _adapt_dict(d):
+    return json.dumps(d)
+
+def _convert_dict(ds):
+    if ds is None:
+        return ds
+    else:
+        return json.loads(ds)
+
+def _adapt_bufs(bufs):
+    # this is *horrible*
+    # copy buffers into single list and pickle it:
+    if bufs and isinstance(bufs[0], (bytes, buffer)):
+        return sqlite3.Binary(pickle.dumps(map(bytes, bufs),-1))
+    elif bufs:
+        return bufs
+    else:
+        return None
+
+def _convert_bufs(bs):
+    if bs is None:
+        return []
+    else:
+        return pickle.loads(bytes(bs))
+
+#-----------------------------------------------------------------------------
+# SQLiteDB class
+#-----------------------------------------------------------------------------
+
+class SQLiteDB(BaseDB):
+    """SQLite3 TaskRecord backend."""
+    
+    filename = CUnicode('tasks.db', config=True)
+    location = CUnicode('', config=True)
+    table = CUnicode("", config=True)
+    
+    _db = Instance('sqlite3.Connection')
+    _keys = List(['msg_id' ,
+            'header' ,
+            'content',
+            'buffers',
+            'submitted',
+            'client_uuid' ,
+            'engine_uuid' ,
+            'started',
+            'completed',
+            'resubmitted',
+            'result_header' ,
+            'result_content' ,
+            'result_buffers' ,
+            'queue' ,
+            'pyin' ,
+            'pyout',
+            'pyerr',
+            'stdout',
+            'stderr',
+        ])
+    
+    def __init__(self, **kwargs):
+        super(SQLiteDB, self).__init__(**kwargs)
+        if not self.table:
+            # use session, and prefix _, since starting with # is illegal
+            self.table = '_'+self.session.replace('-','_')
+        if not self.location:
+            if hasattr(self.config.Global, 'cluster_dir'):
+                self.location = self.config.Global.cluster_dir
+            else:
+                self.location = '.'
+        self._init_db()
+        
+        # register db commit as 2s periodic callback
+        # to prevent clogging pipes
+        # assumes we are being run in a zmq ioloop app
+        loop = ioloop.IOLoop.instance()
+        pc = ioloop.PeriodicCallback(self._db.commit, 2000, loop)
+        pc.start()
+    
+    def _defaults(self):
+        """create an empty record"""
+        d = {}
+        for key in self._keys:
+            d[key] = None
+        return d
+    
+    def _init_db(self):
+        """Connect to the database and get new session number."""
+        # register adapters
+        sqlite3.register_adapter(datetime, _adapt_datetime)
+        sqlite3.register_converter('datetime', _convert_datetime)
+        sqlite3.register_adapter(dict, _adapt_dict)
+        sqlite3.register_converter('dict', _convert_dict)
+        sqlite3.register_adapter(list, _adapt_bufs)
+        sqlite3.register_converter('bufs', _convert_bufs)
+        # connect to the db
+        dbfile = os.path.join(self.location, self.filename)
+        self._db = sqlite3.connect(dbfile, detect_types=sqlite3.PARSE_DECLTYPES, 
+            # isolation_level = None)#,
+             cached_statements=64)
+        # print dir(self._db)
+        
+        self._db.execute("""CREATE TABLE IF NOT EXISTS %s 
+                (msg_id text PRIMARY KEY,
+                header dict text,
+                content dict text,
+                buffers bufs blob,
+                submitted datetime text,
+                client_uuid text,
+                engine_uuid text,
+                started datetime text,
+                completed datetime text,
+                resubmitted datetime text,
+                result_header dict text,
+                result_content dict text,
+                result_buffers bufs blob,
+                queue text,
+                pyin text,
+                pyout text,
+                pyerr text,
+                stdout text,
+                stderr text)
+                """%self.table)
+        # self._db.execute("""CREATE TABLE IF NOT EXISTS %s_buffers 
+        #         (msg_id text, result integer, buffer blob)
+        #         """%self.table)
+        self._db.commit()
+    
+    def _dict_to_list(self, d):
+        """turn a mongodb-style record dict into a list."""
+        
+        return [ d[key] for key in self._keys ]
+    
+    def _list_to_dict(self, line):
+        """Inverse of dict_to_list"""
+        d = self._defaults()
+        for key,value in zip(self._keys, line):
+            d[key] = value
+        
+        return d
+    
+    def _render_expression(self, check):
+        """Turn a mongodb-style search dict into an SQL query."""
+        expressions = []
+        args = []
+        
+        skeys = set(check.keys())
+        skeys.difference_update(set(self._keys))
+        skeys.difference_update(set(['buffers', 'result_buffers']))
+        if skeys:
+            raise KeyError("Illegal testing key(s): %s"%skeys)
+        
+        for name,sub_check in check.iteritems():
+            if isinstance(sub_check, dict):
+                for test,value in sub_check.iteritems():
+                    try:
+                        op = operators[test]
+                    except KeyError:
+                        raise KeyError("Unsupported operator: %r"%test)
+                    if isinstance(op, tuple):
+                        op, join = op
+                    expr = "%s %s ?"%(name, op)
+                    if isinstance(value, (tuple,list)):
+                        expr = '( %s )'%( join.join([expr]*len(value)) )
+                        args.extend(value)
+                    else:
+                        args.append(value)
+                    expressions.append(expr)
+            else:
+                # it's an equality check
+                expressions.append("%s IS ?"%name)
+                args.append(sub_check)
+        
+        expr = " AND ".join(expressions)
+        return expr, args
+    
+    def add_record(self, msg_id, rec):
+        """Add a new Task Record, by msg_id."""
+        d = self._defaults()
+        d.update(rec)
+        d['msg_id'] = msg_id
+        line = self._dict_to_list(d)
+        tups = '(%s)'%(','.join(['?']*len(line)))
+        self._db.execute("INSERT INTO %s VALUES %s"%(self.table, tups), line)
+        # self._db.commit()
+    
+    def get_record(self, msg_id):
+        """Get a specific Task Record, by msg_id."""
+        cursor = self._db.execute("""SELECT * FROM %s WHERE msg_id==?"""%self.table, (msg_id,))
+        line = cursor.fetchone()
+        if line is None:
+            raise KeyError("No such msg: %r"%msg_id)
+        return self._list_to_dict(line)
+    
+    def update_record(self, msg_id, rec):
+        """Update the data in an existing record."""
+        query = "UPDATE %s SET "%self.table
+        sets = []
+        keys = sorted(rec.keys())
+        values = []
+        for key in keys:
+            sets.append('%s = ?'%key)
+            values.append(rec[key])
+        query += ', '.join(sets)
+        query += ' WHERE msg_id == %r'%msg_id
+        self._db.execute(query, values)
+        # self._db.commit()
+    
+    def drop_record(self, msg_id):
+        """Remove a record from the DB."""
+        self._db.execute("""DELETE FROM %s WHERE mgs_id==?"""%self.table, (msg_id,))
+        # self._db.commit()
+    
+    def drop_matching_records(self, check):
+        """Remove a record from the DB."""
+        expr,args = self._render_expression(check)
+        query = "DELETE FROM %s WHERE %s"%(self.table, expr)
+        self._db.execute(query,args)
+        # self._db.commit()
+        
+    def find_records(self, check, id_only=False):
+        """Find records matching a query dict."""
+        req = 'msg_id' if id_only else '*'
+        expr,args = self._render_expression(check)
+        query = """SELECT %s FROM %s WHERE %s"""%(req, self.table, expr)
+        cursor = self._db.execute(query, args)
+        matches = cursor.fetchall()
+        if id_only:
+            return [ m[0] for m in matches ]
+        else:
+            records = {}
+            for line in matches:
+                rec = self._list_to_dict(line)
+                records[rec['msg_id']] = rec
+            return records
+
+__all__ = ['SQLiteDB']
  No newline at end of file

IPython/parallel/engine/engine.py

@@ -0,0 +1,156 b''
+#!/usr/bin/env python
+"""A simple engine that talks to a controller over 0MQ.
+it handles registration, etc. and launches a kernel
+connected to the Controller's Schedulers.
+"""
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2010-2011  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+from __future__ import print_function
+
+import sys
+import time
+
+import zmq
+from zmq.eventloop import ioloop, zmqstream
+
+# internal
+from IPython.utils.traitlets import Instance, Str, Dict, Int, Type, CFloat
+# from IPython.utils.localinterfaces import LOCALHOST 
+
+from IPython.parallel.controller.heartmonitor import Heart
+from IPython.parallel.factory import RegistrationFactory
+from IPython.parallel.streamsession import Message
+from IPython.parallel.util import disambiguate_url
+
+from .streamkernel import Kernel
+
+class EngineFactory(RegistrationFactory):
+    """IPython engine"""
+    
+    # configurables:
+    user_ns=Dict(config=True)
+    out_stream_factory=Type('IPython.zmq.iostream.OutStream', config=True)
+    display_hook_factory=Type('IPython.zmq.displayhook.DisplayHook', config=True)
+    location=Str(config=True)
+    timeout=CFloat(2,config=True)
+    
+    # not configurable:
+    id=Int(allow_none=True)
+    registrar=Instance('zmq.eventloop.zmqstream.ZMQStream')
+    kernel=Instance(Kernel)
+    
+    
+    def __init__(self, **kwargs):
+        super(EngineFactory, self).__init__(**kwargs)
+        ctx = self.context
+        
+        reg = ctx.socket(zmq.XREQ)
+        reg.setsockopt(zmq.IDENTITY, self.ident)
+        reg.connect(self.url)
+        self.registrar = zmqstream.ZMQStream(reg, self.loop)
+        
+    def register(self):
+        """send the registration_request"""
+        
+        self.log.info("registering")
+        content = dict(queue=self.ident, heartbeat=self.ident, control=self.ident)
+        self.registrar.on_recv(self.complete_registration)
+        # print (self.session.key)
+        self.session.send(self.registrar, "registration_request",content=content)
+    
+    def complete_registration(self, msg):
+        # print msg
+        self._abort_dc.stop()
+        ctx = self.context
+        loop = self.loop
+        identity = self.ident
+        
+        idents,msg = self.session.feed_identities(msg)
+        msg = Message(self.session.unpack_message(msg))
+        
+        if msg.content.status == 'ok':
+            self.id = int(msg.content.id)
+            
+            # create Shell Streams (MUX, Task, etc.):
+            queue_addr = msg.content.mux
+            shell_addrs = [ str(queue_addr) ]
+            task_addr = msg.content.task
+            if task_addr:
+                shell_addrs.append(str(task_addr))
+            
+            # Uncomment this to go back to two-socket model
+            # shell_streams = []
+            # for addr in shell_addrs:
+            #     stream = zmqstream.ZMQStream(ctx.socket(zmq.XREP), loop)
+            #     stream.setsockopt(zmq.IDENTITY, identity)
+            #     stream.connect(disambiguate_url(addr, self.location))
+            #     shell_streams.append(stream)
+            
+            # Now use only one shell stream for mux and tasks
+            stream = zmqstream.ZMQStream(ctx.socket(zmq.XREP), loop)
+            stream.setsockopt(zmq.IDENTITY, identity)
+            shell_streams = [stream]
+            for addr in shell_addrs:
+                stream.connect(disambiguate_url(addr, self.location))
+            # end single stream-socket
+            
+            # control stream:
+            control_addr = str(msg.content.control)
+            control_stream = zmqstream.ZMQStream(ctx.socket(zmq.XREP), loop)
+            control_stream.setsockopt(zmq.IDENTITY, identity)
+            control_stream.connect(disambiguate_url(control_addr, self.location))
+            
+            # create iopub stream:
+            iopub_addr = msg.content.iopub
+            iopub_stream = zmqstream.ZMQStream(ctx.socket(zmq.PUB), loop)
+            iopub_stream.setsockopt(zmq.IDENTITY, identity)
+            iopub_stream.connect(disambiguate_url(iopub_addr, self.location))
+            
+            # launch heartbeat
+            hb_addrs = msg.content.heartbeat
+            # print (hb_addrs)
+            
+            # # Redirect input streams and set a display hook.
+            if self.out_stream_factory:
+                sys.stdout = self.out_stream_factory(self.session, iopub_stream, u'stdout')
+                sys.stdout.topic = 'engine.%i.stdout'%self.id
+                sys.stderr = self.out_stream_factory(self.session, iopub_stream, u'stderr')
+                sys.stderr.topic = 'engine.%i.stderr'%self.id
+            if self.display_hook_factory:
+                sys.displayhook = self.display_hook_factory(self.session, iopub_stream)
+                sys.displayhook.topic = 'engine.%i.pyout'%self.id
+            
+            self.kernel = Kernel(config=self.config, int_id=self.id, ident=self.ident, session=self.session, 
+                    control_stream=control_stream, shell_streams=shell_streams, iopub_stream=iopub_stream, 
+                    loop=loop, user_ns = self.user_ns, logname=self.log.name)
+            self.kernel.start()
+            hb_addrs = [ disambiguate_url(addr, self.location) for addr in hb_addrs ]
+            heart = Heart(*map(str, hb_addrs), heart_id=identity)
+            # ioloop.DelayedCallback(heart.start, 1000, self.loop).start()
+            heart.start()
+            
+            
+        else:
+            self.log.fatal("Registration Failed: %s"%msg)
+            raise Exception("Registration Failed: %s"%msg)
+        
+        self.log.info("Completed registration with id %i"%self.id)
+    
+    
+    def abort(self):
+        self.log.fatal("Registration timed out")
+        self.session.send(self.registrar, "unregistration_request", content=dict(id=self.id))
+        time.sleep(1)
+        sys.exit(255)
+    
+    def start(self):
+        dc = ioloop.DelayedCallback(self.register, 0, self.loop)
+        dc.start()
+        self._abort_dc = ioloop.DelayedCallback(self.abort, self.timeout*1000, self.loop)
+        self._abort_dc.start()
+

IPython/parallel/engine/kernelstarter.py

@@ -0,0 +1,225 b''
+"""KernelStarter class that intercepts Control Queue messages, and handles process management."""
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2010-2011  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+from zmq.eventloop import ioloop
+
+from IPython.parallel.streamsession import StreamSession
+
+class KernelStarter(object):
+    """Object for resetting/killing the Kernel."""
+    
+    
+    def __init__(self, session, upstream, downstream, *kernel_args, **kernel_kwargs):
+        self.session = session
+        self.upstream = upstream
+        self.downstream = downstream
+        self.kernel_args = kernel_args
+        self.kernel_kwargs = kernel_kwargs
+        self.handlers = {}
+        for method in 'shutdown_request shutdown_reply'.split():
+            self.handlers[method] = getattr(self, method)
+    
+    def start(self):
+        self.upstream.on_recv(self.dispatch_request)
+        self.downstream.on_recv(self.dispatch_reply)
+        
+    #--------------------------------------------------------------------------
+    # Dispatch methods
+    #--------------------------------------------------------------------------
+    
+    def dispatch_request(self, raw_msg):
+        idents, msg = self.session.feed_identities()
+        try:
+            msg = self.session.unpack_message(msg, content=False)
+        except:
+            print ("bad msg: %s"%msg)
+        
+        msgtype = msg['msg_type']
+        handler = self.handlers.get(msgtype, None)
+        if handler is None:
+            self.downstream.send_multipart(raw_msg, copy=False)
+        else:
+            handler(msg)
+        
+    def dispatch_reply(self, raw_msg):
+        idents, msg = self.session.feed_identities()
+        try:
+            msg = self.session.unpack_message(msg, content=False)
+        except:
+            print ("bad msg: %s"%msg)
+        
+        msgtype = msg['msg_type']
+        handler = self.handlers.get(msgtype, None)
+        if handler is None:
+            self.upstream.send_multipart(raw_msg, copy=False)
+        else:
+            handler(msg)
+    
+    #--------------------------------------------------------------------------
+    # Handlers
+    #--------------------------------------------------------------------------
+    
+    def shutdown_request(self, msg):
+        """"""
+        self.downstream.send_multipart(msg)
+    
+    #--------------------------------------------------------------------------
+    # Kernel process management methods, from KernelManager:
+    #--------------------------------------------------------------------------
+    
+    def _check_local(addr):
+        if isinstance(addr, tuple):
+            addr = addr[0]
+        return addr in LOCAL_IPS
+    
+    def start_kernel(self, **kw):
+        """Starts a kernel process and configures the manager to use it.
+
+        If random ports (port=0) are being used, this method must be called
+        before the channels are created.
+
+        Parameters:
+        -----------
+        ipython : bool, optional (default True)
+             Whether to use an IPython kernel instead of a plain Python kernel.
+        """
+        self.kernel = Process(target=make_kernel, args=self.kernel_args,
+                                            kwargs=self.kernel_kwargs)
+
+    def shutdown_kernel(self, restart=False):
+        """ Attempts to the stop the kernel process cleanly. If the kernel
+        cannot be stopped, it is killed, if possible.
+        """
+        # FIXME: Shutdown does not work on Windows due to ZMQ errors!
+        if sys.platform == 'win32':
+            self.kill_kernel()
+            return
+
+        # Don't send any additional kernel kill messages immediately, to give
+        # the kernel a chance to properly execute shutdown actions. Wait for at
+        # most 1s, checking every 0.1s.
+        self.xreq_channel.shutdown(restart=restart)
+        for i in range(10):
+            if self.is_alive:
+                time.sleep(0.1)
+            else:
+                break
+        else:
+            # OK, we've waited long enough.
+            if self.has_kernel:
+                self.kill_kernel()
+    
+    def restart_kernel(self, now=False):
+        """Restarts a kernel with the same arguments that were used to launch
+        it. If the old kernel was launched with random ports, the same ports
+        will be used for the new kernel.
+
+        Parameters
+        ----------
+        now : bool, optional
+          If True, the kernel is forcefully restarted *immediately*, without
+          having a chance to do any cleanup action.  Otherwise the kernel is
+          given 1s to clean up before a forceful restart is issued.
+
+          In all cases the kernel is restarted, the only difference is whether
+          it is given a chance to perform a clean shutdown or not.
+        """
+        if self._launch_args is None:
+            raise RuntimeError("Cannot restart the kernel. "
+                               "No previous call to 'start_kernel'.")
+        else:
+            if self.has_kernel:
+                if now:
+                    self.kill_kernel()
+                else:
+                    self.shutdown_kernel(restart=True)
+            self.start_kernel(**self._launch_args)
+
+            # FIXME: Messages get dropped in Windows due to probable ZMQ bug
+            # unless there is some delay here.
+            if sys.platform == 'win32':
+                time.sleep(0.2)
+
+    @property
+    def has_kernel(self):
+        """Returns whether a kernel process has been specified for the kernel
+        manager.
+        """
+        return self.kernel is not None
+
+    def kill_kernel(self):
+        """ Kill the running kernel. """
+        if self.has_kernel:
+            # Pause the heart beat channel if it exists.
+            if self._hb_channel is not None:
+                self._hb_channel.pause()
+
+            # Attempt to kill the kernel.
+            try:
+                self.kernel.kill()
+            except OSError, e:
+                # In Windows, we will get an Access Denied error if the process
+                # has already terminated. Ignore it.
+                if not (sys.platform == 'win32' and e.winerror == 5):
+                    raise
+            self.kernel = None
+        else:
+            raise RuntimeError("Cannot kill kernel. No kernel is running!")
+
+    def interrupt_kernel(self):
+        """ Interrupts the kernel. Unlike ``signal_kernel``, this operation is
+        well supported on all platforms.
+        """
+        if self.has_kernel:
+            if sys.platform == 'win32':
+                from parentpoller import ParentPollerWindows as Poller
+                Poller.send_interrupt(self.kernel.win32_interrupt_event)
+            else:
+                self.kernel.send_signal(signal.SIGINT)
+        else:
+            raise RuntimeError("Cannot interrupt kernel. No kernel is running!")
+
+    def signal_kernel(self, signum):
+        """ Sends a signal to the kernel. Note that since only SIGTERM is
+        supported on Windows, this function is only useful on Unix systems.
+        """
+        if self.has_kernel:
+            self.kernel.send_signal(signum)
+        else:
+            raise RuntimeError("Cannot signal kernel. No kernel is running!")
+
+    @property
+    def is_alive(self):
+        """Is the kernel process still running?"""
+        # FIXME: not using a heartbeat means this method is broken for any
+        # remote kernel, it's only capable of handling local kernels.
+        if self.has_kernel:
+            if self.kernel.poll() is None:
+                return True
+            else:
+                return False
+        else:
+            # We didn't start the kernel with this KernelManager so we don't
+            # know if it is running. We should use a heartbeat for this case.
+            return True
+
+
+def make_starter(up_addr, down_addr, *args, **kwargs):
+    """entry point function for launching a kernelstarter in a subprocess"""
+    loop = ioloop.IOLoop.instance()
+    ctx = zmq.Context()
+    session = StreamSession()
+    upstream = zmqstream.ZMQStream(ctx.socket(zmq.XREQ),loop)
+    upstream.connect(up_addr)
+    downstream = zmqstream.ZMQStream(ctx.socket(zmq.XREQ),loop)
+    downstream.connect(down_addr)
+    
+    starter = KernelStarter(session, upstream, downstream, *args, **kwargs)
+    starter.start()
+    loop.start()
+    
  No newline at end of file

IPython/parallel/engine/streamkernel.py

@@ -0,0 +1,423 b''
+#!/usr/bin/env python
+"""
+Kernel adapted from kernel.py to use ZMQ Streams
+"""
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2010-2011  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+# Standard library imports.
+from __future__ import print_function
+
+import sys
+import time
+
+from code import CommandCompiler
+from datetime import datetime
+from pprint import pprint
+
+# System library imports.
+import zmq
+from zmq.eventloop import ioloop, zmqstream
+
+# Local imports.
+from IPython.utils.traitlets import Instance, List, Int, Dict, Set, Str
+from IPython.zmq.completer import KernelCompleter
+
+from IPython.parallel.error import wrap_exception
+from IPython.parallel.factory import SessionFactory
+from IPython.parallel.util import serialize_object, unpack_apply_message, ISO8601
+
+def printer(*args):
+    pprint(args, stream=sys.__stdout__)
+
+
+class _Passer:
+    """Empty class that implements `send()` that does nothing."""
+    def send(self, *args, **kwargs):
+        pass
+    send_multipart = send
+    
+
+#-----------------------------------------------------------------------------
+# Main kernel class
+#-----------------------------------------------------------------------------
+
+class Kernel(SessionFactory):
+
+    #---------------------------------------------------------------------------
+    # Kernel interface
+    #---------------------------------------------------------------------------
+    
+    # kwargs:
+    int_id = Int(-1, config=True)
+    user_ns = Dict(config=True)
+    exec_lines = List(config=True)
+    
+    control_stream = Instance(zmqstream.ZMQStream)
+    task_stream = Instance(zmqstream.ZMQStream)
+    iopub_stream = Instance(zmqstream.ZMQStream)
+    client = Instance('IPython.parallel.Client')
+    
+    # internals
+    shell_streams = List()
+    compiler = Instance(CommandCompiler, (), {})
+    completer = Instance(KernelCompleter)
+    
+    aborted = Set()
+    shell_handlers = Dict()
+    control_handlers = Dict()
+    
+    def _set_prefix(self):
+        self.prefix = "engine.%s"%self.int_id
+    
+    def _connect_completer(self):
+        self.completer = KernelCompleter(self.user_ns)
+    
+    def __init__(self, **kwargs):
+        super(Kernel, self).__init__(**kwargs)
+        self._set_prefix()
+        self._connect_completer()
+        
+        self.on_trait_change(self._set_prefix, 'id')
+        self.on_trait_change(self._connect_completer, 'user_ns')
+        
+        # Build dict of handlers for message types
+        for msg_type in ['execute_request', 'complete_request', 'apply_request', 
+                'clear_request']:
+            self.shell_handlers[msg_type] = getattr(self, msg_type)
+        
+        for msg_type in ['shutdown_request', 'abort_request']+self.shell_handlers.keys():
+            self.control_handlers[msg_type] = getattr(self, msg_type)
+        
+        self._initial_exec_lines()
+    
+    def _wrap_exception(self, method=None):
+        e_info = dict(engine_uuid=self.ident, engine_id=self.int_id, method=method)
+        content=wrap_exception(e_info)
+        return content
+    
+    def _initial_exec_lines(self):
+        s = _Passer()
+        content = dict(silent=True, user_variable=[],user_expressions=[])
+        for line in self.exec_lines:
+            self.log.debug("executing initialization: %s"%line)
+            content.update({'code':line})
+            msg = self.session.msg('execute_request', content)
+            self.execute_request(s, [], msg)
+        
+        
+    #-------------------- control handlers -----------------------------
+    def abort_queues(self):
+        for stream in self.shell_streams:
+            if stream:
+                self.abort_queue(stream)
+    
+    def abort_queue(self, stream):
+        while True:
+            try:
+                msg = self.session.recv(stream, zmq.NOBLOCK,content=True)
+            except zmq.ZMQError as e:
+                if e.errno == zmq.EAGAIN:
+                    break
+                else:
+                    return
+            else:
+                if msg is None:
+                    return
+                else:
+                    idents,msg = msg
+                
+                # assert self.reply_socketly_socket.rcvmore(), "Unexpected missing message part."
+                # msg = self.reply_socket.recv_json()
+            self.log.info("Aborting:")
+            self.log.info(str(msg))
+            msg_type = msg['msg_type']
+            reply_type = msg_type.split('_')[0] + '_reply'
+            # reply_msg = self.session.msg(reply_type, {'status' : 'aborted'}, msg)
+            # self.reply_socket.send(ident,zmq.SNDMORE)
+            # self.reply_socket.send_json(reply_msg)
+            reply_msg = self.session.send(stream, reply_type, 
+                        content={'status' : 'aborted'}, parent=msg, ident=idents)[0]
+            self.log.debug(str(reply_msg))
+            # We need to wait a bit for requests to come in. This can probably
+            # be set shorter for true asynchronous clients.
+            time.sleep(0.05)
+    
+    def abort_request(self, stream, ident, parent):
+        """abort a specifig msg by id"""
+        msg_ids = parent['content'].get('msg_ids', None)
+        if isinstance(msg_ids, basestring):
+            msg_ids = [msg_ids]
+        if not msg_ids:
+            self.abort_queues()
+        for mid in msg_ids:
+            self.aborted.add(str(mid))
+        
+        content = dict(status='ok')
+        reply_msg = self.session.send(stream, 'abort_reply', content=content, 
+                parent=parent, ident=ident)
+        self.log.debug(str(reply_msg))
+    
+    def shutdown_request(self, stream, ident, parent):
+        """kill ourself.  This should really be handled in an external process"""
+        try:
+            self.abort_queues()
+        except:
+            content = self._wrap_exception('shutdown')
+        else:
+            content = dict(parent['content'])
+            content['status'] = 'ok'
+        msg = self.session.send(stream, 'shutdown_reply',
+                                content=content, parent=parent, ident=ident)
+        self.log.debug(str(msg))
+        dc = ioloop.DelayedCallback(lambda : sys.exit(0), 1000, self.loop)
+        dc.start()
+    
+    def dispatch_control(self, msg):
+        idents,msg = self.session.feed_identities(msg, copy=False)
+        try:
+            msg = self.session.unpack_message(msg, content=True, copy=False)
+        except:
+            self.log.error("Invalid Message", exc_info=True)
+            return
+        
+        header = msg['header']
+        msg_id = header['msg_id']
+        
+        handler = self.control_handlers.get(msg['msg_type'], None)
+        if handler is None:
+            self.log.error("UNKNOWN CONTROL MESSAGE TYPE: %r"%msg['msg_type'])
+        else:
+            handler(self.control_stream, idents, msg)
+    
+
+    #-------------------- queue helpers ------------------------------
+    
+    def check_dependencies(self, dependencies):
+        if not dependencies:
+            return True
+        if len(dependencies) == 2 and dependencies[0] in 'any all'.split():
+            anyorall = dependencies[0]
+            dependencies = dependencies[1]
+        else:
+            anyorall = 'all'
+        results = self.client.get_results(dependencies,status_only=True)
+        if results['status'] != 'ok':
+            return False
+        
+        if anyorall == 'any':
+            if not results['completed']:
+                return False
+        else:
+            if results['pending']:
+                return False
+        
+        return True
+    
+    def check_aborted(self, msg_id):
+        return msg_id in self.aborted
+    
+    #-------------------- queue handlers -----------------------------
+    
+    def clear_request(self, stream, idents, parent):
+        """Clear our namespace."""
+        self.user_ns = {}
+        msg = self.session.send(stream, 'clear_reply', ident=idents, parent=parent, 
+                content = dict(status='ok'))
+        self._initial_exec_lines()
+    
+    def execute_request(self, stream, ident, parent):
+        self.log.debug('execute request %s'%parent)
+        try:
+            code = parent[u'content'][u'code']
+        except:
+            self.log.error("Got bad msg: %s"%parent, exc_info=True)
+            return
+        self.session.send(self.iopub_stream, u'pyin', {u'code':code},parent=parent,
+                            ident='%s.pyin'%self.prefix)
+        started = datetime.now().strftime(ISO8601)
+        try:
+            comp_code = self.compiler(code, '<zmq-kernel>')
+            # allow for not overriding displayhook
+            if hasattr(sys.displayhook, 'set_parent'):
+                sys.displayhook.set_parent(parent)
+                sys.stdout.set_parent(parent)
+                sys.stderr.set_parent(parent)
+            exec comp_code in self.user_ns, self.user_ns
+        except:
+            exc_content = self._wrap_exception('execute')
+            # exc_msg = self.session.msg(u'pyerr', exc_content, parent)
+            self.session.send(self.iopub_stream, u'pyerr', exc_content, parent=parent,
+                            ident='%s.pyerr'%self.prefix)
+            reply_content = exc_content
+        else:
+            reply_content = {'status' : 'ok'}
+        
+        reply_msg = self.session.send(stream, u'execute_reply', reply_content, parent=parent, 
+                    ident=ident, subheader = dict(started=started))
+        self.log.debug(str(reply_msg))
+        if reply_msg['content']['status'] == u'error':
+            self.abort_queues()
+
+    def complete_request(self, stream, ident, parent):
+        matches = {'matches' : self.complete(parent),
+                   'status' : 'ok'}
+        completion_msg = self.session.send(stream, 'complete_reply',
+                                           matches, parent, ident)
+        # print >> sys.__stdout__, completion_msg
+
+    def complete(self, msg):
+        return self.completer.complete(msg.content.line, msg.content.text)
+    
+    def apply_request(self, stream, ident, parent):
+        # flush previous reply, so this request won't block it
+        stream.flush(zmq.POLLOUT)
+        
+        try:
+            content = parent[u'content']
+            bufs = parent[u'buffers']
+            msg_id = parent['header']['msg_id']
+            # bound = parent['header'].get('bound', False)
+        except:
+            self.log.error("Got bad msg: %s"%parent, exc_info=True)
+            return
+        # pyin_msg = self.session.msg(u'pyin',{u'code':code}, parent=parent)
+        # self.iopub_stream.send(pyin_msg)
+        # self.session.send(self.iopub_stream, u'pyin', {u'code':code},parent=parent)
+        sub = {'dependencies_met' : True, 'engine' : self.ident,
+                'started': datetime.now().strftime(ISO8601)}
+        try:
+            # allow for not overriding displayhook
+            if hasattr(sys.displayhook, 'set_parent'):
+                sys.displayhook.set_parent(parent)
+                sys.stdout.set_parent(parent)
+                sys.stderr.set_parent(parent)
+            # exec "f(*args,**kwargs)" in self.user_ns, self.user_ns
+            working = self.user_ns
+            # suffix = 
+            prefix = "_"+str(msg_id).replace("-","")+"_"
+            
+            f,args,kwargs = unpack_apply_message(bufs, working, copy=False)
+            # if bound:
+            #     bound_ns = Namespace(working)
+            #     args = [bound_ns]+list(args)
+
+            fname = getattr(f, '__name__', 'f')
+            
+            fname = prefix+"f"
+            argname = prefix+"args"
+            kwargname = prefix+"kwargs"
+            resultname = prefix+"result"
+            
+            ns = { fname : f, argname : args, kwargname : kwargs , resultname : None }
+            # print ns
+            working.update(ns)
+            code = "%s=%s(*%s,**%s)"%(resultname, fname, argname, kwargname)
+            try:
+                exec code in working,working
+                result = working.get(resultname)
+            finally:
+                for key in ns.iterkeys():
+                    working.pop(key)
+            # if bound:
+            #     working.update(bound_ns)
+            
+            packed_result,buf = serialize_object(result)
+            result_buf = [packed_result]+buf
+        except:
+            exc_content = self._wrap_exception('apply')
+            # exc_msg = self.session.msg(u'pyerr', exc_content, parent)
+            self.session.send(self.iopub_stream, u'pyerr', exc_content, parent=parent,
+                                ident='%s.pyerr'%self.prefix)
+            reply_content = exc_content
+            result_buf = []
+            
+            if exc_content['ename'] == 'UnmetDependency':
+                sub['dependencies_met'] = False
+        else:
+            reply_content = {'status' : 'ok'}
+        
+        # put 'ok'/'error' status in header, for scheduler introspection:
+        sub['status'] = reply_content['status']
+        
+        reply_msg = self.session.send(stream, u'apply_reply', reply_content, 
+                    parent=parent, ident=ident,buffers=result_buf, subheader=sub)
+        
+        # flush i/o
+        # should this be before reply_msg is sent, like in the single-kernel code, 
+        # or should nothing get in the way of real results?
+        sys.stdout.flush()
+        sys.stderr.flush()
+    
+    def dispatch_queue(self, stream, msg):
+        self.control_stream.flush()
+        idents,msg = self.session.feed_identities(msg, copy=False)
+        try:
+            msg = self.session.unpack_message(msg, content=True, copy=False)
+        except:
+            self.log.error("Invalid Message", exc_info=True)
+            return
+            
+        
+        header = msg['header']
+        msg_id = header['msg_id']
+        if self.check_aborted(msg_id):
+            self.aborted.remove(msg_id)
+            # is it safe to assume a msg_id will not be resubmitted?
+            reply_type = msg['msg_type'].split('_')[0] + '_reply'
+            reply_msg = self.session.send(stream, reply_type, 
+                        content={'status' : 'aborted'}, parent=msg, ident=idents)
+            return
+        handler = self.shell_handlers.get(msg['msg_type'], None)
+        if handler is None:
+            self.log.error("UNKNOWN MESSAGE TYPE: %r"%msg['msg_type'])
+        else:
+            handler(stream, idents, msg)
+    
+    def start(self):
+        #### stream mode:
+        if self.control_stream:
+            self.control_stream.on_recv(self.dispatch_control, copy=False)
+            self.control_stream.on_err(printer)
+        
+        def make_dispatcher(stream):
+            def dispatcher(msg):
+                return self.dispatch_queue(stream, msg)
+            return dispatcher
+        
+        for s in self.shell_streams:
+            s.on_recv(make_dispatcher(s), copy=False)
+            s.on_err(printer)
+        
+        if self.iopub_stream:
+            self.iopub_stream.on_err(printer)
+        
+        #### while True mode:
+        # while True:
+        #     idle = True
+        #     try:
+        #         msg = self.shell_stream.socket.recv_multipart(
+        #                     zmq.NOBLOCK, copy=False)
+        #     except zmq.ZMQError, e:
+        #         if e.errno != zmq.EAGAIN:
+        #             raise e
+        #     else:
+        #         idle=False
+        #         self.dispatch_queue(self.shell_stream, msg)
+        #             
+        #     if not self.task_stream.empty():
+        #         idle=False
+        #         msg = self.task_stream.recv_multipart()
+        #         self.dispatch_queue(self.task_stream, msg)
+        #     if idle:
+        #         # don't busywait
+        #         time.sleep(1e-3)
+

IPython/parallel/error.py

@@ -0,0 +1,313 b''
+# encoding: utf-8
+
+"""Classes and functions for kernel related errors and exceptions."""
+from __future__ import print_function
+
+import sys
+import traceback
+
+__docformat__ = "restructuredtext en"
+
+# Tell nose to skip this module
+__test__ = {}
+
+#-------------------------------------------------------------------------------
+#  Copyright (C) 2008  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-------------------------------------------------------------------------------
+
+#-------------------------------------------------------------------------------
+# Error classes
+#-------------------------------------------------------------------------------
+class IPythonError(Exception):
+    """Base exception that all of our exceptions inherit from.
+
+    This can be raised by code that doesn't have any more specific
+    information."""
+
+    pass
+
+# Exceptions associated with the controller objects
+class ControllerError(IPythonError): pass
+
+class ControllerCreationError(ControllerError): pass
+
+
+# Exceptions associated with the Engines
+class EngineError(IPythonError): pass
+
+class EngineCreationError(EngineError): pass
+
+class KernelError(IPythonError):
+    pass
+
+class NotDefined(KernelError):
+    def __init__(self, name):
+        self.name = name
+        self.args = (name,)
+
+    def __repr__(self):
+        return '<NotDefined: %s>' % self.name
+    
+    __str__ = __repr__
+
+
+class QueueCleared(KernelError):
+    pass
+
+
+class IdInUse(KernelError):
+    pass
+
+
+class ProtocolError(KernelError):
+    pass
+
+
+class ConnectionError(KernelError):
+    pass
+
+
+class InvalidEngineID(KernelError):
+    pass
+
+
+class NoEnginesRegistered(KernelError):
+    pass
+
+
+class InvalidClientID(KernelError):
+    pass
+
+
+class InvalidDeferredID(KernelError):
+    pass
+
+
+class SerializationError(KernelError):
+    pass
+
+
+class MessageSizeError(KernelError):
+    pass
+
+
+class PBMessageSizeError(MessageSizeError):
+    pass
+
+
+class ResultNotCompleted(KernelError):
+    pass
+
+
+class ResultAlreadyRetrieved(KernelError):
+    pass
+
+class ClientError(KernelError):
+    pass
+
+
+class TaskAborted(KernelError):
+    pass
+
+
+class TaskTimeout(KernelError):
+    pass
+
+
+class NotAPendingResult(KernelError):
+    pass
+
+
+class UnpickleableException(KernelError):
+    pass
+
+
+class AbortedPendingDeferredError(KernelError):
+    pass
+
+
+class InvalidProperty(KernelError):
+    pass
+
+
+class MissingBlockArgument(KernelError):
+    pass
+
+
+class StopLocalExecution(KernelError):
+    pass
+
+
+class SecurityError(KernelError):
+    pass
+
+
+class FileTimeoutError(KernelError):
+    pass
+
+class TimeoutError(KernelError):
+    pass
+
+class UnmetDependency(KernelError):
+    pass
+
+class ImpossibleDependency(UnmetDependency):
+    pass
+
+class DependencyTimeout(ImpossibleDependency):
+    pass
+
+class InvalidDependency(ImpossibleDependency):
+    pass
+
+class RemoteError(KernelError):
+    """Error raised elsewhere"""
+    ename=None
+    evalue=None
+    traceback=None
+    engine_info=None
+    
+    def __init__(self, ename, evalue, traceback, engine_info=None):
+        self.ename=ename
+        self.evalue=evalue
+        self.traceback=traceback
+        self.engine_info=engine_info or {}
+        self.args=(ename, evalue)
+    
+    def __repr__(self):
+        engineid = self.engine_info.get('engine_id', ' ')
+        return "<Remote[%s]:%s(%s)>"%(engineid, self.ename, self.evalue)
+        
+    def __str__(self):
+        sig = "%s(%s)"%(self.ename, self.evalue)
+        if self.traceback:
+            return sig + '\n' + self.traceback
+        else:
+            return sig
+    
+
+class TaskRejectError(KernelError):
+    """Exception to raise when a task should be rejected by an engine.
+    
+    This exception can be used to allow a task running on an engine to test
+    if the engine (or the user's namespace on the engine) has the needed
+    task dependencies.  If not, the task should raise this exception.  For
+    the task to be retried on another engine, the task should be created
+    with the `retries` argument > 1.
+    
+    The advantage of this approach over our older properties system is that
+    tasks have full access to the user's namespace on the engines and the
+    properties don't have to be managed or tested by the controller.
+    """
+
+
+class CompositeError(RemoteError):
+    """Error for representing possibly multiple errors on engines"""
+    def __init__(self, message, elist):
+        Exception.__init__(self, *(message, elist))
+        # Don't use pack_exception because it will conflict with the .message
+        # attribute that is being deprecated in 2.6 and beyond.
+        self.msg = message
+        self.elist = elist
+        self.args = [ e[0] for e in elist ]
+
+    def _get_engine_str(self, ei):
+        if not ei:
+            return '[Engine Exception]'
+        else:
+            return '[%s:%s]: ' % (ei['engine_id'], ei['method'])
+
+    def _get_traceback(self, ev):
+        try:
+            tb = ev._ipython_traceback_text
+        except AttributeError:
+            return 'No traceback available'
+        else:
+            return tb
+
+    def __str__(self):
+        s = str(self.msg)
+        for en, ev, etb, ei in self.elist:
+            engine_str = self._get_engine_str(ei)
+            s = s + '\n' + engine_str + en + ': ' + str(ev)
+        return s
+    
+    def __repr__(self):
+        return "CompositeError(%i)"%len(self.elist)
+
+    def print_tracebacks(self, excid=None):
+        if excid is None:
+            for (en,ev,etb,ei) in self.elist:
+                print (self._get_engine_str(ei))
+                print (etb or 'No traceback available')
+                print ()
+        else:
+            try:
+                en,ev,etb,ei = self.elist[excid]
+            except:
+                raise IndexError("an exception with index %i does not exist"%excid)
+            else:
+                print (self._get_engine_str(ei))
+                print (etb or 'No traceback available')
+    
+    def raise_exception(self, excid=0):
+        try:
+            en,ev,etb,ei = self.elist[excid]
+        except:
+            raise IndexError("an exception with index %i does not exist"%excid)
+        else:
+            raise RemoteError(en, ev, etb, ei)
+
+
+def collect_exceptions(rdict_or_list, method='unspecified'):
+    """check a result dict for errors, and raise CompositeError if any exist.
+    Passthrough otherwise."""
+    elist = []
+    if isinstance(rdict_or_list, dict):
+        rlist = rdict_or_list.values()
+    else:
+        rlist = rdict_or_list
+    for r in rlist:
+        if isinstance(r, RemoteError):
+            en, ev, etb, ei = r.ename, r.evalue, r.traceback, r.engine_info
+            # Sometimes we could have CompositeError in our list.  Just take
+            # the errors out of them and put them in our new list.  This 
+            # has the effect of flattening lists of CompositeErrors into one
+            # CompositeError
+            if en=='CompositeError':
+                for e in ev.elist:
+                    elist.append(e)
+            else:
+                elist.append((en, ev, etb, ei))
+    if len(elist)==0:
+        return rdict_or_list
+    else:
+        msg = "one or more exceptions from call to method: %s" % (method)
+        # This silliness is needed so the debugger has access to the exception
+        # instance (e in this case)
+        try:
+            raise CompositeError(msg, elist)
+        except CompositeError as e:
+            raise e
+
+def wrap_exception(engine_info={}):
+    etype, evalue, tb = sys.exc_info()
+    stb = traceback.format_exception(etype, evalue, tb)
+    exc_content = {
+        'status' : 'error',
+        'traceback' : stb,
+        'ename' : unicode(etype.__name__),
+        'evalue' : unicode(evalue),
+        'engine_info' : engine_info
+    }
+    return exc_content
+
+def unwrap_exception(content):
+    err = RemoteError(content['ename'], content['evalue'], 
+                ''.join(content['traceback']),
+                content.get('engine_info', {}))
+    return err
+

IPython/parallel/factory.py

@@ -0,0 +1,152 b''
+"""Base config factories."""
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2009  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+
+import logging
+import os
+import uuid
+
+from zmq.eventloop.ioloop import IOLoop
+
+from IPython.config.configurable import Configurable
+from IPython.utils.importstring import import_item
+from IPython.utils.traitlets import Str,Int,Instance, CUnicode, CStr
+
+import IPython.parallel.streamsession as ss
+from IPython.parallel.util import select_random_ports
+
+#-----------------------------------------------------------------------------
+# Classes
+#-----------------------------------------------------------------------------
+class LoggingFactory(Configurable):
+    """A most basic class, that has a `log` (type:`Logger`) attribute, set via a `logname` Trait."""
+    log = Instance('logging.Logger', ('ZMQ', logging.WARN))
+    logname = CUnicode('ZMQ')
+    def _logname_changed(self, name, old, new):
+        self.log = logging.getLogger(new)
+    
+
+class SessionFactory(LoggingFactory):
+    """The Base factory from which every factory in IPython.parallel inherits"""
+    
+    packer = Str('',config=True)
+    unpacker = Str('',config=True)
+    ident = CStr('',config=True)
+    def _ident_default(self):
+        return str(uuid.uuid4())
+    username = CUnicode(os.environ.get('USER','username'),config=True)
+    exec_key = CUnicode('',config=True)
+    # not configurable:
+    context = Instance('zmq.Context', (), {})
+    session = Instance('IPython.parallel.streamsession.StreamSession')
+    loop = Instance('zmq.eventloop.ioloop.IOLoop', allow_none=False)
+    def _loop_default(self):
+        return IOLoop.instance()
+    
+    
+    def __init__(self, **kwargs):
+        super(SessionFactory, self).__init__(**kwargs)
+        exec_key = self.exec_key or None
+        # set the packers:
+        if not self.packer:
+            packer_f = unpacker_f = None
+        elif self.packer.lower() == 'json':
+            packer_f = ss.json_packer
+            unpacker_f = ss.json_unpacker
+        elif self.packer.lower() == 'pickle':
+            packer_f = ss.pickle_packer
+            unpacker_f = ss.pickle_unpacker
+        else:
+            packer_f = import_item(self.packer)
+            unpacker_f = import_item(self.unpacker)
+        
+        # construct the session
+        self.session = ss.StreamSession(self.username, self.ident, packer=packer_f, unpacker=unpacker_f, key=exec_key)
+    
+
+class RegistrationFactory(SessionFactory):
+    """The Base Configurable for objects that involve registration."""
+    
+    url = Str('', config=True) # url takes precedence over ip,regport,transport
+    transport = Str('tcp', config=True)
+    ip = Str('127.0.0.1', config=True)
+    regport = Instance(int, config=True)
+    def _regport_default(self):
+        # return 10101
+        return select_random_ports(1)[0]
+    
+    def __init__(self, **kwargs):
+        super(RegistrationFactory, self).__init__(**kwargs)
+        self._propagate_url()
+        self._rebuild_url()
+        self.on_trait_change(self._propagate_url, 'url')
+        self.on_trait_change(self._rebuild_url, 'ip')
+        self.on_trait_change(self._rebuild_url, 'transport')
+        self.on_trait_change(self._rebuild_url, 'regport')
+    
+    def _rebuild_url(self):
+        self.url = "%s://%s:%i"%(self.transport, self.ip, self.regport)
+        
+    def _propagate_url(self):
+        """Ensure self.url contains full transport://interface:port"""
+        if self.url:
+            iface = self.url.split('://',1)
+            if len(iface) == 2:
+                self.transport,iface = iface
+            iface = iface.split(':')
+            self.ip = iface[0]
+            if iface[1]:
+                self.regport = int(iface[1])
+
+#-----------------------------------------------------------------------------
+# argparse argument extenders
+#-----------------------------------------------------------------------------
+
+
+def add_session_arguments(parser):
+    paa = parser.add_argument
+    paa('--ident',
+        type=str, dest='SessionFactory.ident', 
+        help='set the ZMQ and session identity [default: random uuid]',
+        metavar='identity')
+    # paa('--execkey',
+    #     type=str, dest='SessionFactory.exec_key', 
+    #     help='path to a file containing an execution key.',
+    #     metavar='execkey')
+    paa('--packer',
+        type=str, dest='SessionFactory.packer', 
+        help='method to serialize messages: {json,pickle} [default: json]',
+        metavar='packer')
+    paa('--unpacker',
+        type=str, dest='SessionFactory.unpacker', 
+        help='inverse function of `packer`.  Only necessary when using something other than json|pickle',
+        metavar='packer')
+
+def add_registration_arguments(parser):
+    paa = parser.add_argument
+    paa('--ip',
+        type=str, dest='RegistrationFactory.ip',
+        help="The IP used for registration [default: localhost]",
+        metavar='ip')
+    paa('--transport',
+        type=str, dest='RegistrationFactory.transport',
+        help="The ZeroMQ transport used for registration [default: tcp]",
+        metavar='transport')
+    paa('--url',
+        type=str, dest='RegistrationFactory.url', 
+        help='set transport,ip,regport in one go, e.g. tcp://127.0.0.1:10101',
+        metavar='url')
+    paa('--regport',
+        type=int, dest='RegistrationFactory.regport',
+        help="The port used for registration [default: 10101]",
+        metavar='ip')

IPython/parallel/scripts/__init__.py

@@ -0,0 +1,16 b''
+# encoding: utf-8
+
+""""""
+
+__docformat__ = "restructuredtext en"
+
+#-------------------------------------------------------------------------------
+#  Copyright (C) 2008  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-------------------------------------------------------------------------------
+
+#-------------------------------------------------------------------------------
+# Imports
+#-------------------------------------------------------------------------------
  No newline at end of file

IPython/parallel/scripts/ipcluster

@@ -0,0 +1,18 b''
+#!/usr/bin/env python
+# encoding: utf-8
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2009  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+
+from IPython.parallel.apps.ipclusterapp import launch_new_instance
+
+launch_new_instance()

IPython/parallel/scripts/ipcontroller

@@ -0,0 +1,18 b''
+#!/usr/bin/env python
+# encoding: utf-8
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2009  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+
+from IPython.parallel.apps.ipcontrollerapp import launch_new_instance
+
+launch_new_instance()

IPython/parallel/scripts/ipengine

@@ -0,0 +1,20 b''
+#!/usr/bin/env python
+# encoding: utf-8
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2009  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+
+from IPython.parallel.apps.ipengineapp import launch_new_instance
+
+launch_new_instance()
+
+

IPython/parallel/scripts/iplogger

@@ -0,0 +1,20 b''
+#!/usr/bin/env python
+# encoding: utf-8
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2009  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+
+from IPython.parallel.apps.iploggerapp import launch_new_instance
+
+launch_new_instance()
+
+

IPython/parallel/streamsession.py

@@ -0,0 +1,418 b''
+#!/usr/bin/env python
+"""edited session.py to work with streams, and move msg_type to the header
+"""
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2010-2011  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+
+import os
+import pprint
+import uuid
+from datetime import datetime
+
+try:
+    import cPickle
+    pickle = cPickle
+except:
+    cPickle = None
+    import pickle
+
+import zmq
+from zmq.utils import jsonapi
+from zmq.eventloop.zmqstream import ZMQStream
+
+from .util import ISO8601
+
+# packer priority: jsonlib[2], cPickle, simplejson/json, pickle
+json_name = '' if not jsonapi.jsonmod else jsonapi.jsonmod.__name__
+if json_name in ('jsonlib', 'jsonlib2'):
+    use_json = True
+elif json_name:
+    if cPickle is None:
+        use_json = True
+    else:
+        use_json = False
+else:
+    use_json = False
+
+def squash_unicode(obj):
+    if isinstance(obj,dict):
+        for key in obj.keys():
+            obj[key] = squash_unicode(obj[key])
+            if isinstance(key, unicode):
+                obj[squash_unicode(key)] = obj.pop(key)
+    elif isinstance(obj, list):
+        for i,v in enumerate(obj):
+            obj[i] = squash_unicode(v)
+    elif isinstance(obj, unicode):
+        obj = obj.encode('utf8')
+    return obj
+
+json_packer = jsonapi.dumps
+json_unpacker = lambda s: squash_unicode(jsonapi.loads(s))
+
+pickle_packer = lambda o: pickle.dumps(o,-1)
+pickle_unpacker = pickle.loads
+
+if use_json:
+    default_packer = json_packer
+    default_unpacker = json_unpacker
+else:
+    default_packer = pickle_packer
+    default_unpacker = pickle_unpacker
+
+
+DELIM="<IDS|MSG>"
+
+class Message(object):
+    """A simple message object that maps dict keys to attributes.
+
+    A Message can be created from a dict and a dict from a Message instance
+    simply by calling dict(msg_obj)."""
+    
+    def __init__(self, msg_dict):
+        dct = self.__dict__
+        for k, v in dict(msg_dict).iteritems():
+            if isinstance(v, dict):
+                v = Message(v)
+            dct[k] = v
+
+    # Having this iterator lets dict(msg_obj) work out of the box.
+    def __iter__(self):
+        return iter(self.__dict__.iteritems())
+    
+    def __repr__(self):
+        return repr(self.__dict__)
+
+    def __str__(self):
+        return pprint.pformat(self.__dict__)
+
+    def __contains__(self, k):
+        return k in self.__dict__
+
+    def __getitem__(self, k):
+        return self.__dict__[k]
+
+
+def msg_header(msg_id, msg_type, username, session):
+    date=datetime.now().strftime(ISO8601)
+    return locals()
+
+def extract_header(msg_or_header):
+    """Given a message or header, return the header."""
+    if not msg_or_header:
+        return {}
+    try:
+        # See if msg_or_header is the entire message.
+        h = msg_or_header['header']
+    except KeyError:
+        try:
+            # See if msg_or_header is just the header
+            h = msg_or_header['msg_id']
+        except KeyError:
+            raise
+        else:
+            h = msg_or_header
+    if not isinstance(h, dict):
+        h = dict(h)
+    return h
+
+class StreamSession(object):
+    """tweaked version of IPython.zmq.session.Session, for development in Parallel"""
+    debug=False
+    key=None
+    
+    def __init__(self, username=None, session=None, packer=None, unpacker=None, key=None, keyfile=None):
+        if username is None:
+            username = os.environ.get('USER','username')
+        self.username = username
+        if session is None:
+            self.session = str(uuid.uuid4())
+        else:
+            self.session = session
+        self.msg_id = str(uuid.uuid4())
+        if packer is None:
+            self.pack = default_packer
+        else:
+            if not callable(packer):
+                raise TypeError("packer must be callable, not %s"%type(packer))
+            self.pack = packer
+        
+        if unpacker is None:
+            self.unpack = default_unpacker
+        else:
+            if not callable(unpacker):
+                raise TypeError("unpacker must be callable, not %s"%type(unpacker))
+            self.unpack = unpacker
+        
+        if key is not None and keyfile is not None:
+            raise TypeError("Must specify key OR keyfile, not both")
+        if keyfile is not None:
+            with open(keyfile) as f:
+                self.key = f.read().strip()
+        else:
+            self.key = key
+        if isinstance(self.key, unicode):
+            self.key = self.key.encode('utf8')
+        # print key, keyfile, self.key
+        self.none = self.pack({})
+            
+    def msg_header(self, msg_type):
+        h = msg_header(self.msg_id, msg_type, self.username, self.session)
+        self.msg_id = str(uuid.uuid4())
+        return h
+
+    def msg(self, msg_type, content=None, parent=None, subheader=None):
+        msg = {}
+        msg['header'] = self.msg_header(msg_type)
+        msg['msg_id'] = msg['header']['msg_id']
+        msg['parent_header'] = {} if parent is None else extract_header(parent)
+        msg['msg_type'] = msg_type
+        msg['content'] = {} if content is None else content
+        sub = {} if subheader is None else subheader
+        msg['header'].update(sub)
+        return msg
+
+    def check_key(self, msg_or_header):
+        """Check that a message's header has the right key"""
+        if self.key is None:
+            return True
+        header = extract_header(msg_or_header)
+        return header.get('key', None) == self.key
+            
+        
+    def send(self, stream, msg_or_type, content=None, buffers=None, parent=None, subheader=None, ident=None, track=False):
+        """Build and send a message via stream or socket.
+        
+        Parameters
+        ----------
+        
+        stream : zmq.Socket or ZMQStream
+            the socket-like object used to send the data
+        msg_or_type : str or Message/dict
+            Normally, msg_or_type will be a msg_type unless a message is being sent more
+            than once.
+        
+        content : dict or None
+            the content of the message (ignored if msg_or_type is a message)
+        buffers : list or None
+            the already-serialized buffers to be appended to the message
+        parent : Message or dict or None
+            the parent or parent header describing the parent of this message
+        subheader : dict or None
+            extra header keys for this message's header
+        ident : bytes or list of bytes
+            the zmq.IDENTITY routing path
+        track : bool
+            whether to track.  Only for use with Sockets, because ZMQStream objects cannot track messages.
+        
+        Returns
+        -------
+        msg : message dict
+            the constructed message
+        (msg,tracker) : (message dict, MessageTracker)
+            if track=True, then a 2-tuple will be returned, the first element being the constructed
+            message, and the second being the MessageTracker
+            
+        """
+
+        if not isinstance(stream, (zmq.Socket, ZMQStream)):
+            raise TypeError("stream must be Socket or ZMQStream, not %r"%type(stream))
+        elif track and isinstance(stream, ZMQStream):
+            raise TypeError("ZMQStream cannot track messages")
+        
+        if isinstance(msg_or_type, (Message, dict)):
+            # we got a Message, not a msg_type
+            # don't build a new Message
+            msg = msg_or_type
+            content = msg['content']
+        else:
+            msg = self.msg(msg_or_type, content, parent, subheader)
+        
+        buffers = [] if buffers is None else buffers
+        to_send = []
+        if isinstance(ident, list):
+            # accept list of idents
+            to_send.extend(ident)
+        elif ident is not None:
+            to_send.append(ident)
+        to_send.append(DELIM)
+        if self.key is not None:
+            to_send.append(self.key)
+        to_send.append(self.pack(msg['header']))
+        to_send.append(self.pack(msg['parent_header']))
+        
+        if content is None:
+            content = self.none
+        elif isinstance(content, dict):
+            content = self.pack(content)
+        elif isinstance(content, bytes):
+            # content is already packed, as in a relayed message
+            pass
+        else:
+            raise TypeError("Content incorrect type: %s"%type(content))
+        to_send.append(content)
+        flag = 0
+        if buffers:
+            flag = zmq.SNDMORE
+            _track = False
+        else:
+            _track=track
+        if track:
+            tracker = stream.send_multipart(to_send, flag, copy=False, track=_track)
+        else:
+            tracker = stream.send_multipart(to_send, flag, copy=False)
+        for b in buffers[:-1]:
+            stream.send(b, flag, copy=False)
+        if buffers:
+            if track:
+                tracker = stream.send(buffers[-1], copy=False, track=track)
+            else:
+                tracker = stream.send(buffers[-1], copy=False)
+                
+        # omsg = Message(msg)
+        if self.debug:
+            pprint.pprint(msg)
+            pprint.pprint(to_send)
+            pprint.pprint(buffers)
+        
+        msg['tracker'] = tracker
+        
+        return msg
+    
+    def send_raw(self, stream, msg, flags=0, copy=True, ident=None):
+        """Send a raw message via ident path.
+        
+        Parameters
+        ----------
+        msg : list of sendable buffers"""
+        to_send = []
+        if isinstance(ident, bytes):
+            ident = [ident]
+        if ident is not None:
+            to_send.extend(ident)
+        to_send.append(DELIM)
+        if self.key is not None:
+            to_send.append(self.key)
+        to_send.extend(msg)
+        stream.send_multipart(msg, flags, copy=copy)
+    
+    def recv(self, socket, mode=zmq.NOBLOCK, content=True, copy=True):
+        """receives and unpacks a message
+        returns [idents], msg"""
+        if isinstance(socket, ZMQStream):
+            socket = socket.socket
+        try:
+            msg = socket.recv_multipart(mode)
+        except zmq.ZMQError as e:
+            if e.errno == zmq.EAGAIN:
+                # We can convert EAGAIN to None as we know in this case
+                # recv_multipart won't return None.
+                return None
+            else:
+                raise
+        # return an actual Message object
+        # determine the number of idents by trying to unpack them.
+        # this is terrible:
+        idents, msg = self.feed_identities(msg, copy)
+        try:
+            return idents, self.unpack_message(msg, content=content, copy=copy)
+        except Exception as e:
+            print (idents, msg)
+            # TODO: handle it
+            raise e
+    
+    def feed_identities(self, msg, copy=True):
+        """feed until DELIM is reached, then return the prefix as idents and remainder as
+        msg. This is easily broken by setting an IDENT to DELIM, but that would be silly.
+        
+        Parameters
+        ----------
+        msg : a list of Message or bytes objects
+            the message to be split
+        copy : bool
+            flag determining whether the arguments are bytes or Messages
+        
+        Returns
+        -------
+        (idents,msg) : two lists
+            idents will always be a list of bytes - the indentity prefix
+            msg will be a list of bytes or Messages, unchanged from input
+            msg should be unpackable via self.unpack_message at this point.
+        """
+        ikey = int(self.key is not None)
+        minlen = 3 + ikey
+        msg = list(msg)
+        idents = []
+        while len(msg) > minlen:
+            if copy:
+                s = msg[0]
+            else:
+                s = msg[0].bytes
+            if s == DELIM:
+                msg.pop(0)
+                break
+            else:
+                idents.append(s)
+                msg.pop(0)
+                
+        return idents, msg
+    
+    def unpack_message(self, msg, content=True, copy=True):
+        """Return a message object from the format
+        sent by self.send.
+        
+        Parameters:
+        -----------
+        
+        content : bool (True)
+            whether to unpack the content dict (True), 
+            or leave it serialized (False)
+        
+        copy : bool (True)
+            whether to return the bytes (True), 
+            or the non-copying Message object in each place (False)
+        
+        """
+        ikey = int(self.key is not None)
+        minlen = 3 + ikey
+        message = {}
+        if not copy:
+            for i in range(minlen):
+                msg[i] = msg[i].bytes
+        if ikey:
+            if not self.key == msg[0]:
+                raise KeyError("Invalid Session Key: %s"%msg[0])
+        if not len(msg) >= minlen:
+            raise TypeError("malformed message, must have at least %i elements"%minlen)
+        message['header'] = self.unpack(msg[ikey+0])
+        message['msg_type'] = message['header']['msg_type']
+        message['parent_header'] = self.unpack(msg[ikey+1])
+        if content:
+            message['content'] = self.unpack(msg[ikey+2])
+        else:
+            message['content'] = msg[ikey+2]
+        
+        message['buffers'] = msg[ikey+3:]# [ m.buffer for m in msg[3:] ]
+        return message
+        
+
+def test_msg2obj():
+    am = dict(x=1)
+    ao = Message(am)
+    assert ao.x == am['x']
+
+    am['y'] = dict(z=1)
+    ao = Message(am)
+    assert ao.y.z == am['y']['z']
+    
+    k1, k2 = 'y', 'z'
+    assert ao[k1][k2] == am[k1][k2]
+    
+    am2 = dict(ao)
+    assert am['x'] == am2['x']
+    assert am['y']['z'] == am2['y']['z']

IPython/parallel/tests/__init__.py

@@ -0,0 +1,69 b''
+"""toplevel setup/teardown for parallel tests."""
+
+#-------------------------------------------------------------------------------
+#  Copyright (C) 2011  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-------------------------------------------------------------------------------
+
+#-------------------------------------------------------------------------------
+# Imports
+#-------------------------------------------------------------------------------
+
+import tempfile
+import time
+from subprocess import Popen, PIPE, STDOUT
+
+from IPython.parallel import Client
+
+processes = []
+blackhole = tempfile.TemporaryFile()
+
+# nose setup/teardown
+
+def setup():
+    cp = Popen('ipcontroller --profile iptest -r --log-level 10 --log-to-file'.split(), stdout=blackhole, stderr=STDOUT)
+    processes.append(cp)
+    time.sleep(.5)
+    add_engines(1)
+    c = Client(profile='iptest')
+    while not c.ids:
+        time.sleep(.1)
+        c.spin()
+    c.close()
+
+def add_engines(n=1, profile='iptest'):
+    rc = Client(profile=profile)
+    base = len(rc)
+    eps = []
+    for i in range(n):
+        ep = Popen(['ipengine']+ ['--profile', profile, '--log-level', '10', '--log-to-file'], stdout=blackhole, stderr=STDOUT)
+        # ep.start()
+        processes.append(ep)
+        eps.append(ep)
+    while len(rc) < base+n:
+        time.sleep(.1)
+        rc.spin()
+    rc.close()
+    return eps
+
+def teardown():
+    time.sleep(1)
+    while processes:
+        p = processes.pop()
+        if p.poll() is None:
+            try:
+                p.terminate()
+            except Exception, e:
+                print e
+                pass
+        if p.poll() is None:
+            time.sleep(.25)
+        if p.poll() is None:
+            try:
+                print 'killing'
+                p.kill()
+            except:
+                print "couldn't shutdown process: ", p
+    

IPython/parallel/tests/clienttest.py

@@ -0,0 +1,115 b''
+"""base class for parallel client tests"""
+
+#-------------------------------------------------------------------------------
+#  Copyright (C) 2011  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-------------------------------------------------------------------------------
+
+import sys
+import tempfile
+import time
+
+from nose import SkipTest
+
+import zmq
+from zmq.tests import BaseZMQTestCase
+
+from IPython.external.decorator import decorator
+
+from IPython.parallel import error
+from IPython.parallel import Client
+from IPython.parallel.tests import processes,add_engines
+
+# simple tasks for use in apply tests
+
+def segfault():
+    """this will segfault"""
+    import ctypes
+    ctypes.memset(-1,0,1)
+
+def wait(n):
+    """sleep for a time"""
+    import time
+    time.sleep(n)
+    return n
+
+def raiser(eclass):
+    """raise an exception"""
+    raise eclass()
+
+# test decorator for skipping tests when libraries are unavailable
+def skip_without(*names):
+    """skip a test if some names are not importable"""
+    @decorator
+    def skip_without_names(f, *args, **kwargs):
+        """decorator to skip tests in the absence of numpy."""
+        for name in names:
+            try:
+                __import__(name)
+            except ImportError:
+                raise SkipTest
+        return f(*args, **kwargs)
+    return skip_without_names
+
+class ClusterTestCase(BaseZMQTestCase):
+    
+    def add_engines(self, n=1, block=True):
+        """add multiple engines to our cluster"""
+        self.engines.extend(add_engines(n))
+        if block:
+            self.wait_on_engines()
+    
+    def wait_on_engines(self, timeout=5):
+        """wait for our engines to connect."""
+        n = len(self.engines)+self.base_engine_count
+        tic = time.time()
+        while time.time()-tic < timeout and len(self.client.ids) < n:
+            time.sleep(0.1)
+        
+        assert not len(self.client.ids) < n, "waiting for engines timed out"
+    
+    def connect_client(self):
+        """connect a client with my Context, and track its sockets for cleanup"""
+        c = Client(profile='iptest', context=self.context)
+        for name in filter(lambda n:n.endswith('socket'), dir(c)):
+            s = getattr(c, name)
+            s.setsockopt(zmq.LINGER, 0)
+            self.sockets.append(s)
+        return c
+    
+    def assertRaisesRemote(self, etype, f, *args, **kwargs):
+        try:
+            try:
+                f(*args, **kwargs)
+            except error.CompositeError as e:
+                e.raise_exception()
+        except error.RemoteError as e:
+            self.assertEquals(etype.__name__, e.ename, "Should have raised %r, but raised %r"%(e.ename, etype.__name__))
+        else:
+            self.fail("should have raised a RemoteError")
+            
+    def setUp(self):
+        BaseZMQTestCase.setUp(self)
+        self.client = self.connect_client()
+        self.base_engine_count=len(self.client.ids)
+        self.engines=[]
+    
+    def tearDown(self):
+        # self.client.clear(block=True)
+        # close fds:
+        for e in filter(lambda e: e.poll() is not None, processes):
+            processes.remove(e)
+        
+        # allow flushing of incoming messages to prevent crash on socket close
+        self.client.wait(timeout=2)
+        # time.sleep(2)
+        self.client.spin()
+        self.client.close()
+        BaseZMQTestCase.tearDown(self)
+        # this will be redundant when pyzmq merges PR #88
+        # self.context.term()
+        # print tempfile.TemporaryFile().fileno(),
+        # sys.stdout.flush()
+        
  No newline at end of file

IPython/parallel/tests/test_asyncresult.py

@@ -0,0 +1,69 b''
+"""Tests for asyncresult.py"""
+
+#-------------------------------------------------------------------------------
+#  Copyright (C) 2011  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-------------------------------------------------------------------------------
+
+#-------------------------------------------------------------------------------
+# Imports
+#-------------------------------------------------------------------------------
+
+
+from IPython.parallel.error import TimeoutError
+
+from IPython.parallel.tests import add_engines
+from .clienttest import ClusterTestCase
+
+def setup():
+    add_engines(2)
+
+def wait(n):
+    import time
+    time.sleep(n)
+    return n
+
+class AsyncResultTest(ClusterTestCase):
+
+    def test_single_result(self):
+        eid = self.client.ids[-1]
+        ar = self.client[eid].apply_async(lambda : 42)
+        self.assertEquals(ar.get(), 42)
+        ar = self.client[[eid]].apply_async(lambda : 42)
+        self.assertEquals(ar.get(), [42])
+        ar = self.client[-1:].apply_async(lambda : 42)
+        self.assertEquals(ar.get(), [42])
+
+    def test_get_after_done(self):
+        ar = self.client[-1].apply_async(lambda : 42)
+        self.assertFalse(ar.ready())
+        ar.wait()
+        self.assertTrue(ar.ready())
+        self.assertEquals(ar.get(), 42)
+        self.assertEquals(ar.get(), 42)
+
+    def test_get_before_done(self):
+        ar = self.client[-1].apply_async(wait, 0.1)
+        self.assertRaises(TimeoutError, ar.get, 0)
+        ar.wait(0)
+        self.assertFalse(ar.ready())
+        self.assertEquals(ar.get(), 0.1)
+
+    def test_get_after_error(self):
+        ar = self.client[-1].apply_async(lambda : 1/0)
+        ar.wait()
+        self.assertRaisesRemote(ZeroDivisionError, ar.get)
+        self.assertRaisesRemote(ZeroDivisionError, ar.get)
+        self.assertRaisesRemote(ZeroDivisionError, ar.get_dict)
+    
+    def test_get_dict(self):
+        n = len(self.client)
+        ar = self.client[:].apply_async(lambda : 5)
+        self.assertEquals(ar.get(), [5]*n)
+        d = ar.get_dict()
+        self.assertEquals(sorted(d.keys()), sorted(self.client.ids))
+        for eid,r in d.iteritems():
+            self.assertEquals(r, 5)
+

IPython/parallel/tests/test_client.py

@@ -0,0 +1,147 b''
+"""Tests for parallel client.py"""
+
+#-------------------------------------------------------------------------------
+#  Copyright (C) 2011  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-------------------------------------------------------------------------------
+
+#-------------------------------------------------------------------------------
+# Imports
+#-------------------------------------------------------------------------------
+
+import time
+from tempfile import mktemp
+
+import zmq
+
+from IPython.parallel.client import client as clientmod
+from IPython.parallel import error
+from IPython.parallel import AsyncResult, AsyncHubResult
+from IPython.parallel import LoadBalancedView, DirectView
+
+from clienttest import ClusterTestCase, segfault, wait, add_engines
+
+def setup():
+    add_engines(4)
+
+class TestClient(ClusterTestCase):
+    
+    def test_ids(self):
+        n = len(self.client.ids)
+        self.add_engines(3)
+        self.assertEquals(len(self.client.ids), n+3)
+    
+    def test_view_indexing(self):
+        """test index access for views"""
+        self.add_engines(2)
+        targets = self.client._build_targets('all')[-1]
+        v = self.client[:]
+        self.assertEquals(v.targets, targets)
+        t = self.client.ids[2]
+        v = self.client[t]
+        self.assert_(isinstance(v, DirectView))
+        self.assertEquals(v.targets, t)
+        t = self.client.ids[2:4]
+        v = self.client[t]
+        self.assert_(isinstance(v, DirectView))
+        self.assertEquals(v.targets, t)
+        v = self.client[::2]
+        self.assert_(isinstance(v, DirectView))
+        self.assertEquals(v.targets, targets[::2])
+        v = self.client[1::3]
+        self.assert_(isinstance(v, DirectView))
+        self.assertEquals(v.targets, targets[1::3])
+        v = self.client[:-3]
+        self.assert_(isinstance(v, DirectView))
+        self.assertEquals(v.targets, targets[:-3])
+        v = self.client[-1]
+        self.assert_(isinstance(v, DirectView))
+        self.assertEquals(v.targets, targets[-1])
+        self.assertRaises(TypeError, lambda : self.client[None])
+    
+    def test_lbview_targets(self):
+        """test load_balanced_view targets"""
+        v = self.client.load_balanced_view()
+        self.assertEquals(v.targets, None)
+        v = self.client.load_balanced_view(-1)
+        self.assertEquals(v.targets, [self.client.ids[-1]])
+        v = self.client.load_balanced_view('all')
+        self.assertEquals(v.targets, self.client.ids)
+    
+    def test_targets(self):
+        """test various valid targets arguments"""
+        build = self.client._build_targets
+        ids = self.client.ids
+        idents,targets = build(None)
+        self.assertEquals(ids, targets)
+    
+    def test_clear(self):
+        """test clear behavior"""
+        # self.add_engines(2)
+        v = self.client[:]
+        v.block=True
+        v.push(dict(a=5))
+        v.pull('a')
+        id0 = self.client.ids[-1]
+        self.client.clear(targets=id0)
+        self.client[:-1].pull('a')
+        self.assertRaisesRemote(NameError, self.client[id0].get, 'a')
+        self.client.clear(block=True)
+        for i in self.client.ids:
+            # print i
+            self.assertRaisesRemote(NameError, self.client[i].get, 'a')
+    
+    def test_get_result(self):
+        """test getting results from the Hub."""
+        c = clientmod.Client(profile='iptest')
+        # self.add_engines(1)
+        t = c.ids[-1]
+        ar = c[t].apply_async(wait, 1)
+        # give the monitor time to notice the message
+        time.sleep(.25)
+        ahr = self.client.get_result(ar.msg_ids)
+        self.assertTrue(isinstance(ahr, AsyncHubResult))
+        self.assertEquals(ahr.get(), ar.get())
+        ar2 = self.client.get_result(ar.msg_ids)
+        self.assertFalse(isinstance(ar2, AsyncHubResult))
+        c.close()
+    
+    def test_ids_list(self):
+        """test client.ids"""
+        # self.add_engines(2)
+        ids = self.client.ids
+        self.assertEquals(ids, self.client._ids)
+        self.assertFalse(ids is self.client._ids)
+        ids.remove(ids[-1])
+        self.assertNotEquals(ids, self.client._ids)
+    
+    def test_queue_status(self):
+        # self.addEngine(4)
+        ids = self.client.ids
+        id0 = ids[0]
+        qs = self.client.queue_status(targets=id0)
+        self.assertTrue(isinstance(qs, dict))
+        self.assertEquals(sorted(qs.keys()), ['completed', 'queue', 'tasks'])
+        allqs = self.client.queue_status()
+        self.assertTrue(isinstance(allqs, dict))
+        self.assertEquals(sorted(allqs.keys()), self.client.ids)
+        for eid,qs in allqs.items():
+            self.assertTrue(isinstance(qs, dict))
+            self.assertEquals(sorted(qs.keys()), ['completed', 'queue', 'tasks'])
+
+    def test_shutdown(self):
+        # self.addEngine(4)
+        ids = self.client.ids
+        id0 = ids[0]
+        self.client.shutdown(id0, block=True)
+        while id0 in self.client.ids:
+            time.sleep(0.1)
+            self.client.spin()
+        
+        self.assertRaises(IndexError, lambda : self.client[id0])
+        
+    def test_result_status(self):
+        pass
+        # to be written

IPython/parallel/tests/test_dependency.py

@@ -0,0 +1,101 b''
+"""Tests for dependency.py"""
+
+__docformat__ = "restructuredtext en"
+
+#-------------------------------------------------------------------------------
+#  Copyright (C) 2011  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-------------------------------------------------------------------------------
+
+#-------------------------------------------------------------------------------
+# Imports
+#-------------------------------------------------------------------------------
+
+# import
+import os
+
+from IPython.utils.pickleutil import can, uncan
+
+import IPython.parallel as pmod
+from IPython.parallel.util import interactive
+
+from IPython.parallel.tests import add_engines
+from .clienttest import ClusterTestCase
+
+def setup():
+    add_engines(1)
+
[e-mail]('time')
+def wait(n):
+    time.sleep(n)
+    return n
+
+mixed = map(str, range(10))
+completed = map(str, range(0,10,2))
+failed = map(str, range(1,10,2))
+
+class DependencyTest(ClusterTestCase):
+    
+    def setUp(self):
+        ClusterTestCase.setUp(self)
+        self.user_ns = {'__builtins__' : __builtins__}
+        self.view = self.client.load_balanced_view()
+        self.dview = self.client[-1]
+        self.succeeded = set(map(str, range(0,25,2)))
+        self.failed = set(map(str, range(1,25,2)))
+    
+    def assertMet(self, dep):
+        self.assertTrue(dep.check(self.succeeded, self.failed), "Dependency should be met")
+        
+    def assertUnmet(self, dep):
+        self.assertFalse(dep.check(self.succeeded, self.failed), "Dependency should not be met")
+        
+    def assertUnreachable(self, dep):
+        self.assertTrue(dep.unreachable(self.succeeded, self.failed), "Dependency should be unreachable")
+    
+    def assertReachable(self, dep):
+        self.assertFalse(dep.unreachable(self.succeeded, self.failed), "Dependency should be reachable")
+    
+    def cancan(self, f):
+        """decorator to pass through canning into self.user_ns"""
+        return uncan(can(f), self.user_ns)
+    
+    def test_require_imports(self):
+        """test that @require imports names"""
+        @self.cancan
+        @pmod.require('urllib')
+        @interactive
+        def encode(dikt):
+            return urllib.urlencode(dikt)
+        # must pass through canning to properly connect namespaces
+        self.assertEquals(encode(dict(a=5)), 'a=5')
+    
+    def test_success_only(self):
+        dep = pmod.Dependency(mixed, success=True, failure=False)
+        self.assertUnmet(dep)
+        self.assertUnreachable(dep)
+        dep.all=False
+        self.assertMet(dep)
+        self.assertReachable(dep)
+        dep = pmod.Dependency(completed, success=True, failure=False)
+        self.assertMet(dep)
+        self.assertReachable(dep)
+        dep.all=False
+        self.assertMet(dep)
+        self.assertReachable(dep)
+
+    def test_failure_only(self):
+        dep = pmod.Dependency(mixed, success=False, failure=True)
+        self.assertUnmet(dep)
+        self.assertUnreachable(dep)
+        dep.all=False
+        self.assertMet(dep)
+        self.assertReachable(dep)
+        dep = pmod.Dependency(completed, success=False, failure=True)
+        self.assertUnmet(dep)
+        self.assertUnreachable(dep)
+        dep.all=False
+        self.assertUnmet(dep)
+        self.assertUnreachable(dep)

IPython/parallel/tests/test_newserialized.py

@@ -0,0 +1,108 b''
+"""test serialization with newserialized"""
+
+#-------------------------------------------------------------------------------
+#  Copyright (C) 2011  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-------------------------------------------------------------------------------
+
+#-------------------------------------------------------------------------------
+# Imports
+#-------------------------------------------------------------------------------
+
+from unittest import TestCase
+
+from IPython.testing.decorators import parametric
+from IPython.utils import newserialized as ns
+from IPython.utils.pickleutil import can, uncan, CannedObject, CannedFunction
+from IPython.parallel.tests.clienttest import skip_without
+
+
+class CanningTestCase(TestCase):
+    def test_canning(self):
+        d = dict(a=5,b=6)
+        cd = can(d)
+        self.assertTrue(isinstance(cd, dict))
+
+    def test_canned_function(self):
+        f = lambda : 7
+        cf = can(f)
+        self.assertTrue(isinstance(cf, CannedFunction))
+    
+    @parametric
+    def test_can_roundtrip(cls):
+        objs = [
+            dict(),
+            set(),
+            list(),
+            ['a',1,['a',1],u'e'],
+        ]
+        return map(cls.run_roundtrip, objs)
+    
+    @classmethod
+    def run_roundtrip(self, obj):
+        o = uncan(can(obj))
+        assert o == obj, "failed assertion: %r == %r"%(o,obj)
+    
+    def test_serialized_interfaces(self):
+
+        us = {'a':10, 'b':range(10)}
+        s = ns.serialize(us)
+        uus = ns.unserialize(s)
+        self.assertTrue(isinstance(s, ns.SerializeIt))
+        self.assertEquals(uus, us)
+
+    def test_pickle_serialized(self):
+        obj = {'a':1.45345, 'b':'asdfsdf', 'c':10000L}
+        original = ns.UnSerialized(obj)
+        originalSer = ns.SerializeIt(original)
+        firstData = originalSer.getData()
+        firstTD = originalSer.getTypeDescriptor()
+        firstMD = originalSer.getMetadata()
+        self.assertEquals(firstTD, 'pickle')
+        self.assertEquals(firstMD, {})
+        unSerialized = ns.UnSerializeIt(originalSer)
+        secondObj = unSerialized.getObject()
+        for k, v in secondObj.iteritems():
+            self.assertEquals(obj[k], v)
+        secondSer = ns.SerializeIt(ns.UnSerialized(secondObj))
+        self.assertEquals(firstData, secondSer.getData())
+        self.assertEquals(firstTD, secondSer.getTypeDescriptor() )
+        self.assertEquals(firstMD, secondSer.getMetadata())
+    
+    @skip_without('numpy')
+    def test_ndarray_serialized(self):
+        import numpy
+        a = numpy.linspace(0.0, 1.0, 1000)
+        unSer1 = ns.UnSerialized(a)
+        ser1 = ns.SerializeIt(unSer1)
+        td = ser1.getTypeDescriptor()
+        self.assertEquals(td, 'ndarray')
+        md = ser1.getMetadata()
+        self.assertEquals(md['shape'], a.shape)
+        self.assertEquals(md['dtype'], a.dtype.str)
+        buff = ser1.getData()
+        self.assertEquals(buff, numpy.getbuffer(a))
+        s = ns.Serialized(buff, td, md)
+        final = ns.unserialize(s)
+        self.assertEquals(numpy.getbuffer(a), numpy.getbuffer(final))
+        self.assertTrue((a==final).all())
+        self.assertEquals(a.dtype.str, final.dtype.str)
+        self.assertEquals(a.shape, final.shape)
+        # test non-copying:
+        a[2] = 1e9
+        self.assertTrue((a==final).all())
+    
+    def test_uncan_function_globals(self):
+        """test that uncanning a module function restores it into its module"""
+        from re import search
+        cf = can(search)
+        csearch = uncan(cf)
+        self.assertEqual(csearch.__module__, search.__module__)
+        self.assertNotEqual(csearch('asd', 'asdf'), None)
+        csearch = uncan(cf, dict(a=5))
+        self.assertEqual(csearch.__module__, search.__module__)
+        self.assertNotEqual(csearch('asd', 'asdf'), None)
+        
+        
  No newline at end of file

IPython/parallel/tests/test_streamsession.py

@@ -0,0 +1,111 b''
+"""test building messages with streamsession"""
+
+#-------------------------------------------------------------------------------
+#  Copyright (C) 2011  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-------------------------------------------------------------------------------
+
+#-------------------------------------------------------------------------------
+# Imports
+#-------------------------------------------------------------------------------
+
+import os
+import uuid
+import zmq
+
+from zmq.tests import BaseZMQTestCase
+from zmq.eventloop.zmqstream import ZMQStream
+# from IPython.zmq.tests import SessionTestCase
+from IPython.parallel import streamsession as ss
+
+class SessionTestCase(BaseZMQTestCase):
+    
+    def setUp(self):
+        BaseZMQTestCase.setUp(self)
+        self.session = ss.StreamSession()
+
+class TestSession(SessionTestCase):
+    
+    def test_msg(self):
+        """message format"""
+        msg = self.session.msg('execute')
+        thekeys = set('header msg_id parent_header msg_type content'.split())
+        s = set(msg.keys())
+        self.assertEquals(s, thekeys)
+        self.assertTrue(isinstance(msg['content'],dict))
+        self.assertTrue(isinstance(msg['header'],dict))
+        self.assertTrue(isinstance(msg['parent_header'],dict))
+        self.assertEquals(msg['msg_type'], 'execute')
+        
+            
+    
+    def test_args(self):
+        """initialization arguments for StreamSession"""
+        s = self.session
+        self.assertTrue(s.pack is ss.default_packer)
+        self.assertTrue(s.unpack is ss.default_unpacker)
+        self.assertEquals(s.username, os.environ.get('USER', 'username'))
+        
+        s = ss.StreamSession(username=None)
+        self.assertEquals(s.username, os.environ.get('USER', 'username'))
+        
+        self.assertRaises(TypeError, ss.StreamSession, packer='hi')
+        self.assertRaises(TypeError, ss.StreamSession, unpacker='hi')
+        u = str(uuid.uuid4())
+        s = ss.StreamSession(username='carrot', session=u)
+        self.assertEquals(s.session, u)
+        self.assertEquals(s.username, 'carrot')
+        
+    def test_tracking(self):
+        """test tracking messages"""
+        a,b = self.create_bound_pair(zmq.PAIR, zmq.PAIR)
+        s = self.session
+        stream = ZMQStream(a)
+        msg = s.send(a, 'hello', track=False)
+        self.assertTrue(msg['tracker'] is None)
+        msg = s.send(a, 'hello', track=True)
+        self.assertTrue(isinstance(msg['tracker'], zmq.MessageTracker))
+        M = zmq.Message(b'hi there', track=True)
+        msg = s.send(a, 'hello', buffers=[M], track=True)
+        t = msg['tracker']
+        self.assertTrue(isinstance(t, zmq.MessageTracker))
+        self.assertRaises(zmq.NotDone, t.wait, .1)
+        del M
+        t.wait(1) # this will raise
+        
+        
+    # def test_rekey(self):
+    #     """rekeying dict around json str keys"""
+    #     d = {'0': uuid.uuid4(), 0:uuid.uuid4()}
+    #     self.assertRaises(KeyError, ss.rekey, d)
+    #     
+    #     d = {'0': uuid.uuid4(), 1:uuid.uuid4(), 'asdf':uuid.uuid4()}
+    #     d2 = {0:d['0'],1:d[1],'asdf':d['asdf']}
+    #     rd = ss.rekey(d)
+    #     self.assertEquals(d2,rd)
+    #     
+    #     d = {'1.5':uuid.uuid4(),'1':uuid.uuid4()}
+    #     d2 = {1.5:d['1.5'],1:d['1']}
+    #     rd = ss.rekey(d)
+    #     self.assertEquals(d2,rd)
+    #     
+    #     d = {'1.0':uuid.uuid4(),'1':uuid.uuid4()}
+    #     self.assertRaises(KeyError, ss.rekey, d)
+    # 
+    def test_unique_msg_ids(self):
+        """test that messages receive unique ids"""
+        ids = set()
+        for i in range(2**12):
+            h = self.session.msg_header('test')
+            msg_id = h['msg_id']
+            self.assertTrue(msg_id not in ids)
+            ids.add(msg_id)
+    
+    def test_feed_identities(self):
+        """scrub the front for zmq IDENTITIES"""
+        theids = "engine client other".split()
+        content = dict(code='whoda',stuff=object())
+        themsg = self.session.msg('execute',content=content)
+        pmsg = theids

IPython/parallel/tests/test_view.py

@@ -0,0 +1,301 b''
+"""test View objects"""
+#-------------------------------------------------------------------------------
+#  Copyright (C) 2011  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-------------------------------------------------------------------------------
+
+#-------------------------------------------------------------------------------
+# Imports
+#-------------------------------------------------------------------------------
+
+import time
+from tempfile import mktemp
+
+import zmq
+
+from IPython import parallel  as pmod
+from IPython.parallel import error
+from IPython.parallel import AsyncResult, AsyncHubResult, AsyncMapResult
+from IPython.parallel import LoadBalancedView, DirectView
+from IPython.parallel.util import interactive
+
+from IPython.parallel.tests import add_engines
+
+from .clienttest import ClusterTestCase, segfault, wait, skip_without
+
+def setup():
+    add_engines(3)
+
+class TestView(ClusterTestCase):
+    
+    def test_segfault_task(self):
+        """test graceful handling of engine death (balanced)"""
+        # self.add_engines(1)
+        ar = self.client[-1].apply_async(segfault)
+        self.assertRaisesRemote(error.EngineError, ar.get)
+        eid = ar.engine_id
+        while eid in self.client.ids:
+            time.sleep(.01)
+            self.client.spin()
+    
+    def test_segfault_mux(self):
+        """test graceful handling of engine death (direct)"""
+        # self.add_engines(1)
+        eid = self.client.ids[-1]
+        ar = self.client[eid].apply_async(segfault)
+        self.assertRaisesRemote(error.EngineError, ar.get)
+        eid = ar.engine_id
+        while eid in self.client.ids:
+            time.sleep(.01)
+            self.client.spin()
+    
+    def test_push_pull(self):
+        """test pushing and pulling"""
+        data = dict(a=10, b=1.05, c=range(10), d={'e':(1,2),'f':'hi'})
+        t = self.client.ids[-1]
+        v = self.client[t]
+        push = v.push
+        pull = v.pull
+        v.block=True
+        nengines = len(self.client)
+        push({'data':data})
+        d = pull('data')
+        self.assertEquals(d, data)
+        self.client[:].push({'data':data})
+        d = self.client[:].pull('data', block=True)
+        self.assertEquals(d, nengines*[data])
+        ar = push({'data':data}, block=False)
+        self.assertTrue(isinstance(ar, AsyncResult))
+        r = ar.get()
+        ar = self.client[:].pull('data', block=False)
+        self.assertTrue(isinstance(ar, AsyncResult))
+        r = ar.get()
+        self.assertEquals(r, nengines*[data])
+        self.client[:].push(dict(a=10,b=20))
+        r = self.client[:].pull(('a','b'))
+        self.assertEquals(r, nengines*[[10,20]])
+    
+    def test_push_pull_function(self):
+        "test pushing and pulling functions"
+        def testf(x):
+            return 2.0*x
+        
+        t = self.client.ids[-1]
+        self.client[t].block=True
+        push = self.client[t].push
+        pull = self.client[t].pull
+        execute = self.client[t].execute
+        push({'testf':testf})
+        r = pull('testf')
+        self.assertEqual(r(1.0), testf(1.0))
+        execute('r = testf(10)')
+        r = pull('r')
+        self.assertEquals(r, testf(10))
+        ar = self.client[:].push({'testf':testf}, block=False)
+        ar.get()
+        ar = self.client[:].pull('testf', block=False)
+        rlist = ar.get()
+        for r in rlist:
+            self.assertEqual(r(1.0), testf(1.0))
+        execute("def g(x): return x*x")
+        r = pull(('testf','g'))
+        self.assertEquals((r[0](10),r[1](10)), (testf(10), 100))
+    
+    def test_push_function_globals(self):
+        """test that pushed functions have access to globals"""
+        @interactive
+        def geta():
+            return a
+        # self.add_engines(1)
+        v = self.client[-1]
+        v.block=True
+        v['f'] = geta
+        self.assertRaisesRemote(NameError, v.execute, 'b=f()')
+        v.execute('a=5')
+        v.execute('b=f()')
+        self.assertEquals(v['b'], 5)
+    
+    def test_push_function_defaults(self):
+        """test that pushed functions preserve default args"""
+        def echo(a=10):
+            return a
+        v = self.client[-1]
+        v.block=True
+        v['f'] = echo
+        v.execute('b=f()')
+        self.assertEquals(v['b'], 10)
+
+    def test_get_result(self):
+        """test getting results from the Hub."""
+        c = pmod.Client(profile='iptest')
+        # self.add_engines(1)
+        t = c.ids[-1]
+        v = c[t]
+        v2 = self.client[t]
+        ar = v.apply_async(wait, 1)
+        # give the monitor time to notice the message
+        time.sleep(.25)
+        ahr = v2.get_result(ar.msg_ids)
+        self.assertTrue(isinstance(ahr, AsyncHubResult))
+        self.assertEquals(ahr.get(), ar.get())
+        ar2 = v2.get_result(ar.msg_ids)
+        self.assertFalse(isinstance(ar2, AsyncHubResult))
+        c.spin()
+        c.close()
+    
+    def test_run_newline(self):
+        """test that run appends newline to files"""
+        tmpfile = mktemp()
+        with open(tmpfile, 'w') as f:
+            f.write("""def g():
+                return 5
+                """)
+        v = self.client[-1]
+        v.run(tmpfile, block=True)
+        self.assertEquals(v.apply_sync(lambda f: f(), pmod.Reference('g')), 5)
+
+    def test_apply_tracked(self):
+        """test tracking for apply"""
+        # self.add_engines(1)
+        t = self.client.ids[-1]
+        v = self.client[t]
+        v.block=False
+        def echo(n=1024*1024, **kwargs):
+            with v.temp_flags(**kwargs):
+                return v.apply(lambda x: x, 'x'*n)
+        ar = echo(1, track=False)
+        self.assertTrue(isinstance(ar._tracker, zmq.MessageTracker))
+        self.assertTrue(ar.sent)
+        ar = echo(track=True)
+        self.assertTrue(isinstance(ar._tracker, zmq.MessageTracker))
+        self.assertEquals(ar.sent, ar._tracker.done)
+        ar._tracker.wait()
+        self.assertTrue(ar.sent)
+        
+    def test_push_tracked(self):
+        t = self.client.ids[-1]
+        ns = dict(x='x'*1024*1024)
+        v = self.client[t]
+        ar = v.push(ns, block=False, track=False)
+        self.assertTrue(isinstance(ar._tracker, zmq.MessageTracker))
+        self.assertTrue(ar.sent)
+        
+        ar = v.push(ns, block=False, track=True)
+        self.assertTrue(isinstance(ar._tracker, zmq.MessageTracker))
+        self.assertEquals(ar.sent, ar._tracker.done)
+        ar._tracker.wait()
+        self.assertTrue(ar.sent)
+        ar.get()
+        
+    def test_scatter_tracked(self):
+        t = self.client.ids
+        x='x'*1024*1024
+        ar = self.client[t].scatter('x', x, block=False, track=False)
+        self.assertTrue(isinstance(ar._tracker, zmq.MessageTracker))
+        self.assertTrue(ar.sent)
+        
+        ar = self.client[t].scatter('x', x, block=False, track=True)
+        self.assertTrue(isinstance(ar._tracker, zmq.MessageTracker))
+        self.assertEquals(ar.sent, ar._tracker.done)
+        ar._tracker.wait()
+        self.assertTrue(ar.sent)
+        ar.get()
+    
+    def test_remote_reference(self):
+        v = self.client[-1]
+        v['a'] = 123
+        ra = pmod.Reference('a')
+        b = v.apply_sync(lambda x: x, ra)
+        self.assertEquals(b, 123)
+
+
+    def test_scatter_gather(self):
+        view = self.client[:]
+        seq1 = range(16)
+        view.scatter('a', seq1)
+        seq2 = view.gather('a', block=True)
+        self.assertEquals(seq2, seq1)
+        self.assertRaisesRemote(NameError, view.gather, 'asdf', block=True)
+    
+    @skip_without('numpy')
+    def test_scatter_gather_numpy(self):
+        import numpy
+        from numpy.testing.utils import assert_array_equal, assert_array_almost_equal
+        view = self.client[:]
+        a = numpy.arange(64)
+        view.scatter('a', a)
+        b = view.gather('a', block=True)
+        assert_array_equal(b, a)
+    
+    def test_map(self):
+        view = self.client[:]
+        def f(x):
+            return x**2
+        data = range(16)
+        r = view.map_sync(f, data)
+        self.assertEquals(r, map(f, data))
+    
+    def test_scatterGatherNonblocking(self):
+        data = range(16)
+        view = self.client[:]
+        view.scatter('a', data, block=False)
+        ar = view.gather('a', block=False)
+        self.assertEquals(ar.get(), data)
+    
+    @skip_without('numpy')
+    def test_scatter_gather_numpy_nonblocking(self):
+        import numpy
+        from numpy.testing.utils import assert_array_equal, assert_array_almost_equal
+        a = numpy.arange(64)
+        view = self.client[:]
+        ar = view.scatter('a', a, block=False)
+        self.assertTrue(isinstance(ar, AsyncResult))
+        amr = view.gather('a', block=False)
+        self.assertTrue(isinstance(amr, AsyncMapResult))
+        assert_array_equal(amr.get(), a)
+
+    def test_execute(self):
+        view = self.client[:]
+        # self.client.debug=True
+        execute = view.execute
+        ar = execute('c=30', block=False)
+        self.assertTrue(isinstance(ar, AsyncResult))
+        ar = execute('d=[0,1,2]', block=False)
+        self.client.wait(ar, 1)
+        self.assertEquals(len(ar.get()), len(self.client))
+        for c in view['c']:
+            self.assertEquals(c, 30)
+    
+    def test_abort(self):
+        view = self.client[-1]
+        ar = view.execute('import time; time.sleep(0.25)', block=False)
+        ar2 = view.apply_async(lambda : 2)
+        ar3 = view.apply_async(lambda : 3)
+        view.abort(ar2)
+        view.abort(ar3.msg_ids)
+        self.assertRaises(error.TaskAborted, ar2.get)
+        self.assertRaises(error.TaskAborted, ar3.get)
+        
+    def test_temp_flags(self):
+        view = self.client[-1]
+        view.block=True
+        with view.temp_flags(block=False):
+            self.assertFalse(view.block)
+        self.assertTrue(view.block)
+    
+    def test_importer(self):
+        view = self.client[-1]
+        view.clear(block=True)
+        with view.importer:
+            import re
+        
+        @interactive
+        def findall(pat, s):
+            # this globals() step isn't necessary in real code
+            # only to prevent a closure in the test
+            return globals()['re'].findall(pat, s)
+        
+        self.assertEquals(view.apply_sync(findall, '\w+', 'hello world'), 'hello world'.split())
+    

IPython/parallel/util.py

@@ -0,0 +1,462 b''
+"""some generic utilities for dealing with classes, urls, and serialization"""
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2010-2011  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+# Standard library imports.
+import logging
+import os
+import re
+import stat
+import socket
+import sys
+from signal import signal, SIGINT, SIGABRT, SIGTERM
+try:
+    from signal import SIGKILL
+except ImportError:
+    SIGKILL=None
+
+try:
+    import cPickle
+    pickle = cPickle
+except:
+    cPickle = None
+    import pickle
+
+# System library imports
+import zmq
+from zmq.log import handlers
+
+# IPython imports
+from IPython.utils.pickleutil import can, uncan, canSequence, uncanSequence
+from IPython.utils.newserialized import serialize, unserialize
+from IPython.zmq.log import EnginePUBHandler
+
+# globals
+ISO8601="%Y-%m-%dT%H:%M:%S.%f"
+
+#-----------------------------------------------------------------------------
+# Classes
+#-----------------------------------------------------------------------------
+
+class Namespace(dict):
+    """Subclass of dict for attribute access to keys."""
+    
+    def __getattr__(self, key):
+        """getattr aliased to getitem"""
+        if key in self.iterkeys():
+            return self[key]
+        else:
+            raise NameError(key)
+
+    def __setattr__(self, key, value):
+        """setattr aliased to setitem, with strict"""
+        if hasattr(dict, key):
+            raise KeyError("Cannot override dict keys %r"%key)
+        self[key] = value
+    
+
+class ReverseDict(dict):
+    """simple double-keyed subset of dict methods."""
+    
+    def __init__(self, *args, **kwargs):
+        dict.__init__(self, *args, **kwargs)
+        self._reverse = dict()
+        for key, value in self.iteritems():
+            self._reverse[value] = key
+    
+    def __getitem__(self, key):
+        try:
+            return dict.__getitem__(self, key)
+        except KeyError:
+            return self._reverse[key]
+    
+    def __setitem__(self, key, value):
+        if key in self._reverse:
+            raise KeyError("Can't have key %r on both sides!"%key)
+        dict.__setitem__(self, key, value)
+        self._reverse[value] = key
+    
+    def pop(self, key):
+        value = dict.pop(self, key)
+        self._reverse.pop(value)
+        return value
+    
+    def get(self, key, default=None):
+        try:
+            return self[key]
+        except KeyError:
+            return default
+
+#-----------------------------------------------------------------------------
+# Functions
+#-----------------------------------------------------------------------------
+
+def validate_url(url):
+    """validate a url for zeromq"""
+    if not isinstance(url, basestring):
+        raise TypeError("url must be a string, not %r"%type(url))
+    url = url.lower()
+    
+    proto_addr = url.split('://')
+    assert len(proto_addr) == 2, 'Invalid url: %r'%url
+    proto, addr = proto_addr
+    assert proto in ['tcp','pgm','epgm','ipc','inproc'], "Invalid protocol: %r"%proto
+    
+    # domain pattern adapted from http://www.regexlib.com/REDetails.aspx?regexp_id=391
+    # author: Remi Sabourin
+    pat = re.compile(r'^([\w\d]([\w\d\-]{0,61}[\w\d])?\.)*[\w\d]([\w\d\-]{0,61}[\w\d])?$')
+    
+    if proto == 'tcp':
+        lis = addr.split(':')
+        assert len(lis) == 2, 'Invalid url: %r'%url
+        addr,s_port = lis
+        try:
+            port = int(s_port)
+        except ValueError:
+            raise AssertionError("Invalid port %r in url: %r"%(port, url))
+        
+        assert addr == '*' or pat.match(addr) is not None, 'Invalid url: %r'%url
+        
+    else:
+        # only validate tcp urls currently
+        pass
+    
+    return True
+
+
+def validate_url_container(container):
+    """validate a potentially nested collection of urls."""
+    if isinstance(container, basestring):
+        url = container
+        return validate_url(url)
+    elif isinstance(container, dict):
+        container = container.itervalues()
+    
+    for element in container:
+        validate_url_container(element)
+
+
+def split_url(url):
+    """split a zmq url (tcp://ip:port) into ('tcp','ip','port')."""
+    proto_addr = url.split('://')
+    assert len(proto_addr) == 2, 'Invalid url: %r'%url
+    proto, addr = proto_addr
+    lis = addr.split(':')
+    assert len(lis) == 2, 'Invalid url: %r'%url
+    addr,s_port = lis
+    return proto,addr,s_port
+    
+def disambiguate_ip_address(ip, location=None):
+    """turn multi-ip interfaces '0.0.0.0' and '*' into connectable
+    ones, based on the location (default interpretation of location is localhost)."""
+    if ip in ('0.0.0.0', '*'):
+        external_ips = socket.gethostbyname_ex(socket.gethostname())[2]
+        if location is None or location in external_ips:
+            ip='127.0.0.1'
+        elif location:
+            return location
+    return ip
+
+def disambiguate_url(url, location=None):
+    """turn multi-ip interfaces '0.0.0.0' and '*' into connectable
+    ones, based on the location (default interpretation is localhost).
+    
+    This is for zeromq urls, such as tcp://*:10101."""
+    try:
+        proto,ip,port = split_url(url)
+    except AssertionError:
+        # probably not tcp url; could be ipc, etc.
+        return url
+    
+    ip = disambiguate_ip_address(ip,location)
+    
+    return "%s://%s:%s"%(proto,ip,port)
+
+
+def rekey(dikt):
+    """Rekey a dict that has been forced to use str keys where there should be
+    ints by json.  This belongs in the jsonutil added by fperez."""
+    for k in dikt.iterkeys():
+        if isinstance(k, str):
+            ik=fk=None
+            try:
+                ik = int(k)
+            except ValueError:
+                try:
+                    fk = float(k)
+                except ValueError:
+                    continue
+            if ik is not None:
+                nk = ik
+            else:
+                nk = fk
+            if nk in dikt:
+                raise KeyError("already have key %r"%nk)
+            dikt[nk] = dikt.pop(k)
+    return dikt
+
+def serialize_object(obj, threshold=64e-6):
+    """Serialize an object into a list of sendable buffers.
+    
+    Parameters
+    ----------
+    
+    obj : object
+        The object to be serialized
+    threshold : float
+        The threshold for not double-pickling the content.
+        
+    
+    Returns
+    -------
+    ('pmd', [bufs]) :
+        where pmd is the pickled metadata wrapper,
+        bufs is a list of data buffers
+    """
+    databuffers = []
+    if isinstance(obj, (list, tuple)):
+        clist = canSequence(obj)
+        slist = map(serialize, clist)
+        for s in slist:
+            if s.typeDescriptor in ('buffer', 'ndarray') or s.getDataSize() > threshold:
+                databuffers.append(s.getData())
+                s.data = None
+        return pickle.dumps(slist,-1), databuffers
+    elif isinstance(obj, dict):
+        sobj = {}
+        for k in sorted(obj.iterkeys()):
+            s = serialize(can(obj[k]))
+            if s.typeDescriptor in ('buffer', 'ndarray') or s.getDataSize() > threshold:
+                databuffers.append(s.getData())
+                s.data = None
+            sobj[k] = s
+        return pickle.dumps(sobj,-1),databuffers
+    else:
+        s = serialize(can(obj))
+        if s.typeDescriptor in ('buffer', 'ndarray') or s.getDataSize() > threshold:
+            databuffers.append(s.getData())
+            s.data = None
+        return pickle.dumps(s,-1),databuffers
+            
+        
+def unserialize_object(bufs):
+    """reconstruct an object serialized by serialize_object from data buffers."""
+    bufs = list(bufs)
+    sobj = pickle.loads(bufs.pop(0))
+    if isinstance(sobj, (list, tuple)):
+        for s in sobj:
+            if s.data is None:
+                s.data = bufs.pop(0)
+        return uncanSequence(map(unserialize, sobj)), bufs
+    elif isinstance(sobj, dict):
+        newobj = {}
+        for k in sorted(sobj.iterkeys()):
+            s = sobj[k]
+            if s.data is None:
+                s.data = bufs.pop(0)
+            newobj[k] = uncan(unserialize(s))
+        return newobj, bufs
+    else:
+        if sobj.data is None:
+            sobj.data = bufs.pop(0)
+        return uncan(unserialize(sobj)), bufs
+
+def pack_apply_message(f, args, kwargs, threshold=64e-6):
+    """pack up a function, args, and kwargs to be sent over the wire
+    as a series of buffers. Any object whose data is larger than `threshold`
+    will not have their data copied (currently only numpy arrays support zero-copy)"""
+    msg = [pickle.dumps(can(f),-1)]
+    databuffers = [] # for large objects
+    sargs, bufs = serialize_object(args,threshold)
+    msg.append(sargs)
+    databuffers.extend(bufs)
+    skwargs, bufs = serialize_object(kwargs,threshold)
+    msg.append(skwargs)
+    databuffers.extend(bufs)
+    msg.extend(databuffers)
+    return msg
+
+def unpack_apply_message(bufs, g=None, copy=True):
+    """unpack f,args,kwargs from buffers packed by pack_apply_message()
+    Returns: original f,args,kwargs"""
+    bufs = list(bufs) # allow us to pop
+    assert len(bufs) >= 3, "not enough buffers!"
+    if not copy:
+        for i in range(3):
+            bufs[i] = bufs[i].bytes
+    cf = pickle.loads(bufs.pop(0))
+    sargs = list(pickle.loads(bufs.pop(0)))
+    skwargs = dict(pickle.loads(bufs.pop(0)))
+    # print sargs, skwargs
+    f = uncan(cf, g)
+    for sa in sargs:
+        if sa.data is None:
+            m = bufs.pop(0)
+            if sa.getTypeDescriptor() in ('buffer', 'ndarray'):
+                if copy:
+                    sa.data = buffer(m)
+                else:
+                    sa.data = m.buffer
+            else:
+                if copy:
+                    sa.data = m
+                else:
+                    sa.data = m.bytes
+    
+    args = uncanSequence(map(unserialize, sargs), g)
+    kwargs = {}
+    for k in sorted(skwargs.iterkeys()):
+        sa = skwargs[k]
+        if sa.data is None:
+            m = bufs.pop(0)
+            if sa.getTypeDescriptor() in ('buffer', 'ndarray'):
+                if copy:
+                    sa.data = buffer(m)
+                else:
+                    sa.data = m.buffer
+            else:
+                if copy:
+                    sa.data = m
+                else:
+                    sa.data = m.bytes
+
+        kwargs[k] = uncan(unserialize(sa), g)
+    
+    return f,args,kwargs
+
+#--------------------------------------------------------------------------
+# helpers for implementing old MEC API via view.apply
+#--------------------------------------------------------------------------
+
+def interactive(f):
+    """decorator for making functions appear as interactively defined.
+    This results in the function being linked to the user_ns as globals()
+    instead of the module globals().
+    """
+    f.__module__ = '__main__'
+    return f
+
+@interactive
+def _push(ns):
+    """helper method for implementing `client.push` via `client.apply`"""
+    globals().update(ns)
+
+@interactive
+def _pull(keys):
+    """helper method for implementing `client.pull` via `client.apply`"""
+    user_ns = globals()
+    if isinstance(keys, (list,tuple, set)):
+        for key in keys:
+            if not user_ns.has_key(key):
+                raise NameError("name '%s' is not defined"%key)
+        return map(user_ns.get, keys)
+    else:
+        if not user_ns.has_key(keys):
+            raise NameError("name '%s' is not defined"%keys)
+        return user_ns.get(keys)
+
+@interactive
+def _execute(code):
+    """helper method for implementing `client.execute` via `client.apply`"""
+    exec code in globals()
+
+#--------------------------------------------------------------------------
+# extra process management utilities
+#--------------------------------------------------------------------------
+
+_random_ports = set()
+
+def select_random_ports(n):
+    """Selects and return n random ports that are available."""
+    ports = []
+    for i in xrange(n):
+        sock = socket.socket()
+        sock.bind(('', 0))
+        while sock.getsockname()[1] in _random_ports:
+            sock.close()
+            sock = socket.socket()
+            sock.bind(('', 0))
+        ports.append(sock)
+    for i, sock in enumerate(ports):
+        port = sock.getsockname()[1]
+        sock.close()
+        ports[i] = port
+        _random_ports.add(port)
+    return ports
+
+def signal_children(children):
+    """Relay interupt/term signals to children, for more solid process cleanup."""
+    def terminate_children(sig, frame):
+        logging.critical("Got signal %i, terminating children..."%sig)
+        for child in children:
+            child.terminate()
+        
+        sys.exit(sig != SIGINT)
+        # sys.exit(sig)
+    for sig in (SIGINT, SIGABRT, SIGTERM):
+        signal(sig, terminate_children)
+
+def generate_exec_key(keyfile):
+    import uuid
+    newkey = str(uuid.uuid4())
+    with open(keyfile, 'w') as f:
+        # f.write('ipython-key ')
+        f.write(newkey+'\n')
+    # set user-only RW permissions (0600)
+    # this will have no effect on Windows
+    os.chmod(keyfile, stat.S_IRUSR|stat.S_IWUSR)
+
+
+def integer_loglevel(loglevel):
+    try:
+        loglevel = int(loglevel)
+    except ValueError:
+        if isinstance(loglevel, str):
+            loglevel = getattr(logging, loglevel)
+    return loglevel
+
+def connect_logger(logname, context, iface, root="ip", loglevel=logging.DEBUG):
+    logger = logging.getLogger(logname)
+    if any([isinstance(h, handlers.PUBHandler) for h in logger.handlers]):
+        # don't add a second PUBHandler
+        return
+    loglevel = integer_loglevel(loglevel)
+    lsock = context.socket(zmq.PUB)
+    lsock.connect(iface)
+    handler = handlers.PUBHandler(lsock)
+    handler.setLevel(loglevel)
+    handler.root_topic = root
+    logger.addHandler(handler)
+    logger.setLevel(loglevel)
+
+def connect_engine_logger(context, iface, engine, loglevel=logging.DEBUG):
+    logger = logging.getLogger()
+    if any([isinstance(h, handlers.PUBHandler) for h in logger.handlers]):
+        # don't add a second PUBHandler
+        return
+    loglevel = integer_loglevel(loglevel)
+    lsock = context.socket(zmq.PUB)
+    lsock.connect(iface)
+    handler = EnginePUBHandler(engine, lsock)
+    handler.setLevel(loglevel)
+    logger.addHandler(handler)
+    logger.setLevel(loglevel)
+
+def local_logger(logname, loglevel=logging.DEBUG):
+    loglevel = integer_loglevel(loglevel)
+    logger = logging.getLogger(logname)
+    if any([isinstance(h, logging.StreamHandler) for h in logger.handlers]):
+        # don't add a second StreamHandler
+        return
+    handler = logging.StreamHandler()
+    handler.setLevel(loglevel)
+    logger.addHandler(handler)
+    logger.setLevel(loglevel)

.gitignore

@@ -1,11 +1,12 b''
 build
 ./dist
 docs/dist
-docs/build/*
+docs/build
+docs/_build
 docs/source/api/generated
 docs/gh-pages
 *.py[co]
 build
 *.egg-info
 *~
 *.bak

IPython/config/default/ipcluster_config.py

@@ -1,184 +1,241 b''
 import os
 
 c = get_config()
 
 #-----------------------------------------------------------------------------
 # Select which launchers to use
 #-----------------------------------------------------------------------------
 
 # This allows you to control what method is used to start the controller
 # and engines.  The following methods are currently supported:
 # - Start as a regular process on localhost.
 # - Start using mpiexec.
 # - Start using the Windows HPC Server 2008 scheduler
-# - Start using PBS
-# - Start using SSH (currently broken)
+# - Start using PBS/SGE
+# - Start using SSH
 
 
 # The selected launchers can be configured below.
 
 # Options are:
 # - LocalControllerLauncher
 # - MPIExecControllerLauncher
 # - PBSControllerLauncher
+# - SGEControllerLauncher
 # - WindowsHPCControllerLauncher
-# c.Global.controller_launcher = 'IPython.kernel.launcher.LocalControllerLauncher'
+# c.Global.controller_launcher = 'IPython.parallel.apps.launcher.LocalControllerLauncher'
+# c.Global.controller_launcher = 'IPython.parallel.apps.launcher.PBSControllerLauncher'
 
 # Options are:
 # - LocalEngineSetLauncher
 # - MPIExecEngineSetLauncher
 # - PBSEngineSetLauncher
+# - SGEEngineSetLauncher
 # - WindowsHPCEngineSetLauncher
-# c.Global.engine_launcher = 'IPython.kernel.launcher.LocalEngineSetLauncher'
+# c.Global.engine_launcher = 'IPython.parallel.apps.launcher.LocalEngineSetLauncher'
 
 #-----------------------------------------------------------------------------
 # Global configuration
 #-----------------------------------------------------------------------------
 
 # The default number of engines that will be started. This is overridden by
 # the -n command line option: "ipcluster start -n 4"
 # c.Global.n = 2
 
 # Log to a file in cluster_dir/log, otherwise just log to sys.stdout.
 # c.Global.log_to_file = False
 
 # Remove old logs from cluster_dir/log before starting.
 # c.Global.clean_logs = True
 
 # The working directory for the process. The application will use os.chdir
 # to change to this directory before starting.
 # c.Global.work_dir = os.getcwd()
 
 
 #-----------------------------------------------------------------------------
 # Local process launchers
 #-----------------------------------------------------------------------------
 
 # The command line arguments to call the controller with.
 # c.LocalControllerLauncher.controller_args = \
 #    ['--log-to-file','--log-level', '40']
 
 # The working directory for the controller
 # c.LocalEngineSetLauncher.work_dir = u''
 
 # Command line argument passed to the engines.
 # c.LocalEngineSetLauncher.engine_args = ['--log-to-file','--log-level', '40']
 
 #-----------------------------------------------------------------------------
 # MPIExec launchers
 #-----------------------------------------------------------------------------
 
-# The mpiexec/mpirun command to use in started the controller.
-# c.MPIExecControllerLauncher.mpi_cmd = ['mpiexec']
+# The mpiexec/mpirun command to use in both the controller and engines.
+# c.MPIExecLauncher.mpi_cmd = ['mpiexec']
 
 # Additional arguments to pass to the actual mpiexec command.
+# c.MPIExecLauncher.mpi_args = []
+
+# The mpiexec/mpirun command and args can be overridden if they should be different
+# for controller and engines.
+# c.MPIExecControllerLauncher.mpi_cmd = ['mpiexec']
 # c.MPIExecControllerLauncher.mpi_args = []
+# c.MPIExecEngineSetLauncher.mpi_cmd = ['mpiexec']
+# c.MPIExecEngineSetLauncher.mpi_args = []
 
 # The command line argument to call the controller with.
 # c.MPIExecControllerLauncher.controller_args = \
 #     ['--log-to-file','--log-level', '40']
 
-
-# The mpiexec/mpirun command to use in started the controller.
-# c.MPIExecEngineSetLauncher.mpi_cmd = ['mpiexec']
-
-# Additional arguments to pass to the actual mpiexec command.
-# c.MPIExecEngineSetLauncher.mpi_args = []
-
 # Command line argument passed to the engines.
 # c.MPIExecEngineSetLauncher.engine_args = ['--log-to-file','--log-level', '40']
 
 # The default number of engines to start if not given elsewhere.
 # c.MPIExecEngineSetLauncher.n = 1
 
 #-----------------------------------------------------------------------------
 # SSH launchers
 #-----------------------------------------------------------------------------
 
-# Todo
+# ipclusterz can be used to launch controller and engines remotely via ssh.
+# Note that currently ipclusterz does not do any file distribution, so if
+# machines are not on a shared filesystem, config and json files must be
+# distributed.  For this reason, the reuse_files defaults to True on an
+# ssh-launched Controller.  This flag can be overridded by the program_args
+# attribute of c.SSHControllerLauncher.
+
+# set the ssh cmd for launching remote commands. The default is ['ssh']
+# c.SSHLauncher.ssh_cmd = ['ssh']
+
+# set the ssh cmd for launching remote commands. The default is ['ssh']
+# c.SSHLauncher.ssh_args = ['tt']
+
+# Set the user and hostname for the controller
+# c.SSHControllerLauncher.hostname = 'controller.example.com'
+# c.SSHControllerLauncher.user = os.environ.get('USER','username')
+
+# Set the arguments to be passed to ipcontrollerz
+# note that remotely launched ipcontrollerz will not get the contents of 
+# the local ipcontrollerz_config.py unless it resides on the *remote host*
+# in the location specified by the --cluster_dir argument.
+# c.SSHControllerLauncher.program_args = ['-r', '-ip', '0.0.0.0', '--cluster_dir', '/path/to/cd']
+
+# Set the default args passed to ipenginez for SSH launched engines
+# c.SSHEngineSetLauncher.engine_args = ['--mpi', 'mpi4py']
 
+# SSH engines are launched as a dict of locations/n-engines.
+# if a value is a tuple instead of an int, it is assumed to be of the form
+# (n, [args]), setting the arguments to passed to ipenginez on `host`.
+# otherwise, c.SSHEngineSetLauncher.engine_args will be used as the default.
+
+# In this case, there will be 3 engines at my.example.com, and
+# 2 at [e-mail] with a special json connector location.
+# c.SSHEngineSetLauncher.engines = {'my.example.com' : 3,
+#                                   '[e-mail]' : (2, ['-f', '/path/to/ipcontroller-engine.json']}
+#                                  }
 
 #-----------------------------------------------------------------------------
 # Unix batch (PBS) schedulers launchers
 #-----------------------------------------------------------------------------
 
+# SGE and PBS are very similar. All configurables in this section called 'PBS*'
+# also exist as 'SGE*'.
+
 # The command line program to use to submit a PBS job.
-# c.PBSControllerLauncher.submit_command = 'qsub'
+# c.PBSLauncher.submit_command = ['qsub']
 
 # The command line program to use to delete a PBS job.
-# c.PBSControllerLauncher.delete_command = 'qdel'
+# c.PBSLauncher.delete_command = ['qdel']
+
+# The PBS queue in which the job should run
+# c.PBSLauncher.queue = 'myqueue'
 
 # A regular expression that takes the output of qsub and find the job id.
-# c.PBSControllerLauncher.job_id_regexp = r'\d+'
+# c.PBSLauncher.job_id_regexp = r'\d+'
+
+# If for some reason the Controller and Engines have different options above, they
+# can be set as c.PBSControllerLauncher.<option> etc.
+
+# PBS and SGE have default templates, but you can specify your own, either as strings
+# or from files, as described here:
 
 # The batch submission script used to start the controller. This is where
-# environment variables would be setup, etc. This string is interpolated using
+# environment variables would be setup, etc. This string is interpreted using
 # the Itpl module in IPython.external. Basically, you can use ${n} for the
 # number of engine and ${cluster_dir} for the cluster_dir.
-# c.PBSControllerLauncher.batch_template = """"""
+# c.PBSControllerLauncher.batch_template = """
+# #PBS -N ipcontroller
+# #PBS -q $queue
+# 
+# ipcontrollerz --cluster-dir $cluster_dir
+# """
+
+# You can also load this template from a file
+# c.PBSControllerLauncher.batch_template_file = u"/path/to/my/template.sh"
 
 # The name of the instantiated batch script that will actually be used to
 # submit the job. This will be written to the cluster directory.
-# c.PBSControllerLauncher.batch_file_name = u'pbs_batch_script_controller'
-
-
-# The command line program to use to submit a PBS job.
-# c.PBSEngineSetLauncher.submit_command = 'qsub'
-
-# The command line program to use to delete a PBS job.
-# c.PBSEngineSetLauncher.delete_command = 'qdel'
-
-# A regular expression that takes the output of qsub and find the job id.
-# c.PBSEngineSetLauncher.job_id_regexp = r'\d+'
+# c.PBSControllerLauncher.batch_file_name = u'pbs_controller'
 
 # The batch submission script used to start the engines. This is where
-# environment variables would be setup, etc. This string is interpolated using
+# environment variables would be setup, etc. This string is interpreted using
 # the Itpl module in IPython.external. Basically, you can use ${n} for the
 # number of engine and ${cluster_dir} for the cluster_dir.
-# c.PBSEngineSetLauncher.batch_template = """"""
+# c.PBSEngineSetLauncher.batch_template = """
+# #PBS -N ipcontroller
+# #PBS -l nprocs=$n
+# 
+# ipenginez --cluster-dir $cluster_dir$s
+# """
+
+# You can also load this template from a file
+# c.PBSControllerLauncher.batch_template_file = u"/path/to/my/template.sh"
 
 # The name of the instantiated batch script that will actually be used to
 # submit the job. This will be written to the cluster directory.
-# c.PBSEngineSetLauncher.batch_file_name = u'pbs_batch_script_engines'
+# c.PBSEngineSetLauncher.batch_file_name = u'pbs_engines'
+
+
 
 #-----------------------------------------------------------------------------
 # Windows HPC Server 2008 launcher configuration
 #-----------------------------------------------------------------------------
 
 # c.IPControllerJob.job_name = 'IPController'
 # c.IPControllerJob.is_exclusive = False
 # c.IPControllerJob.username = r'USERDOMAIN\USERNAME'
 # c.IPControllerJob.priority = 'Highest'
 # c.IPControllerJob.requested_nodes = ''
 # c.IPControllerJob.project = 'MyProject'
 
 # c.IPControllerTask.task_name = 'IPController'
 # c.IPControllerTask.controller_cmd = [u'ipcontroller.exe']
 # c.IPControllerTask.controller_args = ['--log-to-file', '--log-level', '40']
 # c.IPControllerTask.environment_variables = {}
 
 # c.WindowsHPCControllerLauncher.scheduler = 'HEADNODE'
 # c.WindowsHPCControllerLauncher.job_file_name = u'ipcontroller_job.xml'
 
 
 # c.IPEngineSetJob.job_name = 'IPEngineSet'
 # c.IPEngineSetJob.is_exclusive = False
 # c.IPEngineSetJob.username = r'USERDOMAIN\USERNAME'
 # c.IPEngineSetJob.priority = 'Highest'
 # c.IPEngineSetJob.requested_nodes = ''
 # c.IPEngineSetJob.project = 'MyProject'
 
 # c.IPEngineTask.task_name = 'IPEngine'
 # c.IPEngineTask.engine_cmd = [u'ipengine.exe']
 # c.IPEngineTask.engine_args = ['--log-to-file', '--log-level', '40']
 # c.IPEngineTask.environment_variables = {}
 
 # c.WindowsHPCEngineSetLauncher.scheduler = 'HEADNODE'
 # c.WindowsHPCEngineSetLauncher.job_file_name = u'ipengineset_job.xml'
 
 
 
 
 
 
 

IPython/config/default/ipcontroller_config.py

@@ -1,136 +1,180 b''
 from IPython.config.loader import Config
 
 c = get_config()
 
 #-----------------------------------------------------------------------------
 # Global configuration
 #-----------------------------------------------------------------------------
 
 # Basic Global config attributes
 
 # Start up messages are logged to stdout using the logging module.
 # These all happen before the twisted reactor is started and are
 # useful for debugging purposes. Can be (10=DEBUG,20=INFO,30=WARN,40=CRITICAL) 
 # and smaller is more verbose.
 # c.Global.log_level = 20
 
 # Log to a file in cluster_dir/log, otherwise just log to sys.stdout.
 # c.Global.log_to_file = False
 
 # Remove old logs from cluster_dir/log before starting.
 # c.Global.clean_logs = True
 
 # A list of Python statements that will be run before starting the 
 # controller. This is provided because occasionally certain things need to 
 # be imported in the controller for pickling to work.
 # c.Global.import_statements = ['import math']
 
-# Reuse the controller's FURL files. If False, FURL files are regenerated
+# Reuse the controller's JSON files. If False, JSON files are regenerated
 # each time the controller is run. If True, they will be reused, *but*, you
 # also must set the network ports by hand. If set, this will override the
 # values set for the client and engine connections below.
-# c.Global.reuse_furls = True
+# c.Global.reuse_files = True
 
-# Enable SSL encryption on all connections to the controller. If set, this
-# will override the values set for the client and engine connections below.
+# Enable exec_key authentication on all messages. Default is True
 # c.Global.secure = True
 
 # The working directory for the process. The application will use os.chdir
 # to change to this directory before starting.
 # c.Global.work_dir = os.getcwd()
 
+# The log url for logging to an `iploggerz` application.  This will override
+# log-to-file.
+# c.Global.log_url = 'tcp://127.0.0.1:20202'
+
+# The specific external IP that is used to disambiguate multi-interface URLs.
+# The default behavior is to guess from external IPs gleaned from `socket`.
+# c.Global.location = '192.168.1.123'
+
+# The ssh server remote clients should use to connect to this controller.
+# It must be a machine that can see the interface specified in client_ip.
+# The default for client_ip is localhost, in which case the sshserver must
+# be an external IP of the controller machine.
+# c.Global.sshserver = 'controller.example.com'
+
+# the url to use for registration.  If set, this overrides engine-ip,
+# engine-transport client-ip,client-transport, and regport.
+# c.RegistrationFactory.url = 'tcp://*:12345'
+
+# the port to use for registration.  Clients and Engines both use this
+# port for registration.
+# c.RegistrationFactory.regport = 10101
+
 #-----------------------------------------------------------------------------
-# Configure the client services
+# Configure the Task Scheduler
 #-----------------------------------------------------------------------------
 
-# Basic client service config attributes
+# The routing scheme. 'pure' will use the pure-ZMQ scheduler. Any other 
+# value will use a Python scheduler with various routing schemes.
+# python schemes are: lru, weighted, random, twobin. Default is 'weighted'.
+# Note that the pure ZMQ scheduler does not support many features, such as
+# dying engines, dependencies, or engine-subset load-balancing.
+# c.ControllerFactory.scheme = 'pure'
 
-# The network interface the controller will listen on for client connections.
-# This should be an IP address or hostname of the controller's host. The empty
-# string means listen on all interfaces.
-# c.FCClientServiceFactory.ip = ''
+# The pure ZMQ scheduler can limit the number of outstanding tasks per engine
+# by using the ZMQ HWM option.  This allows engines with long-running tasks
+# to not steal too many tasks from other engines. The default is 0, which
+# means agressively distribute messages, never waiting for them to finish.
+# c.ControllerFactory.hwm = 1
 
-# The TCP/IP port the controller will listen on for client connections. If 0
-# a random port will be used. If the controller's host has a firewall running
-# it must allow incoming traffic on this port.
-# c.FCClientServiceFactory.port = 0
+# Whether to use Threads or Processes to start the Schedulers.  Threads will
+# use less resources, but potentially reduce throughput. Default is to 
+# use processes.  Note that the a Python scheduler will always be in a Process.
+# c.ControllerFactory.usethreads
 
-# The client learns how to connect to the controller by looking at the
-# location field embedded in the FURL. If this field is empty, all network
-# interfaces that the controller is listening on will be listed. To have the
-# client connect on a particular interface, list it here.
-# c.FCClientServiceFactory.location = ''
+#-----------------------------------------------------------------------------
+# Configure the Hub
+#-----------------------------------------------------------------------------
+
+# Which class to use for the db backend.  Currently supported are DictDB (the
+# default), and MongoDB. Uncomment this line to enable MongoDB, which will
+# slow-down the Hub's responsiveness, but also reduce its memory footprint.
+# c.HubFactory.db_class = 'IPython.parallel.controller.mongodb.MongoDB'
 
-# Use SSL encryption for the client connection.
-# c.FCClientServiceFactory.secure =  True
+# The heartbeat ping frequency.  This is the frequency (in ms) at which the
+# Hub pings engines for heartbeats.  This determines how quickly the Hub
+# will react to engines coming and going.  A lower number means faster response
+# time, but more network activity.  The default is 100ms
+# c.HubFactory.ping = 100
 
-# Reuse the client FURL each time the controller is started. If set, you must
-# also pick a specific network port above (FCClientServiceFactory.port).
-# c.FCClientServiceFactory.reuse_furls = False
+# HubFactory queue port pairs, to set by name: mux, iopub, control, task.  Set
+# each as a tuple of length 2 of ints.  The default is to find random
+# available ports
+# c.HubFactory.mux = (10102,10112)
 
 #-----------------------------------------------------------------------------
-# Configure the engine services
+# Configure the client connections
 #-----------------------------------------------------------------------------
 
-# Basic config attributes for the engine services.
+# Basic client connection config attributes
 
-# The network interface the controller will listen on for engine connections.
-# This should be an IP address or hostname of the controller's host. The empty
-# string means listen on all interfaces.
-# c.FCEngineServiceFactory.ip = ''
+# The network interface the controller will listen on for client connections.
+# This should be an IP address or interface on the controller. An asterisk
+# means listen on all interfaces. The transport can be any transport
+# supported by zeromq (tcp,epgm,pgm,ib,ipc):
+# c.HubFactory.client_ip = '*'
+# c.HubFactory.client_transport = 'tcp'
 
-# The TCP/IP port the controller will listen on for engine connections. If 0
-# a random port will be used. If the controller's host has a firewall running
-# it must allow incoming traffic on this port.
-# c.FCEngineServiceFactory.port = 0
+# individual client ports to configure by name: query_port, notifier_port
+# c.HubFactory.query_port = 12345
 
-# The engine learns how to connect to the controller by looking at the
-# location field embedded in the FURL. If this field is empty, all network
-# interfaces that the controller is listening on will be listed. To have the
-# client connect on a particular interface, list it here.
-# c.FCEngineServiceFactory.location = ''
+#-----------------------------------------------------------------------------
+# Configure the engine connections
+#-----------------------------------------------------------------------------
 
-# Use SSL encryption for the engine connection.
-# c.FCEngineServiceFactory.secure = True
+# Basic config attributes for the engine connections.
 
-# Reuse the client FURL each time the controller is started. If set, you must
-# also pick a specific network port above (FCClientServiceFactory.port).
-# c.FCEngineServiceFactory.reuse_furls = False
+# The network interface the controller will listen on for engine connections.
+# This should be an IP address or interface on the controller. An asterisk
+# means listen on all interfaces. The transport can be any transport
+# supported by zeromq (tcp,epgm,pgm,ib,ipc):
+# c.HubFactory.engine_ip = '*'
+# c.HubFactory.engine_transport = 'tcp'
+
+# set the engine heartbeat ports to use:
+# c.HubFactory.hb = (10303,10313)
 
 #-----------------------------------------------------------------------------
-# Developer level configuration attributes
+# Configure the TaskRecord database backend
 #-----------------------------------------------------------------------------
 
-# You shouldn't have to modify anything in this section. These attributes
-# are more for developers who want to change the behavior of the controller
-# at a fundamental level.
-
-# c.FCClientServiceFactory.cert_file = u'ipcontroller-client.pem'
-
-# default_client_interfaces = Config()
-# default_client_interfaces.Task.interface_chain = [
-#     'IPython.kernel.task.ITaskController',
-#     'IPython.kernel.taskfc.IFCTaskController'
-# ]
-# 
-# default_client_interfaces.Task.furl_file = u'ipcontroller-tc.furl'
-# 
-# default_client_interfaces.MultiEngine.interface_chain = [
-#     'IPython.kernel.multiengine.IMultiEngine',
-#     'IPython.kernel.multienginefc.IFCSynchronousMultiEngine'
-# ]
-# 
-# default_client_interfaces.MultiEngine.furl_file = u'ipcontroller-mec.furl'
-# 
-# c.FCEngineServiceFactory.interfaces = default_client_interfaces
-
-# c.FCEngineServiceFactory.cert_file = u'ipcontroller-engine.pem'
-
-# default_engine_interfaces = Config()
-# default_engine_interfaces.Default.interface_chain = [
-#     'IPython.kernel.enginefc.IFCControllerBase'
-# ]
-# 
-# default_engine_interfaces.Default.furl_file = u'ipcontroller-engine.furl'
-# 
-# c.FCEngineServiceFactory.interfaces = default_engine_interfaces
+# For memory/persistance reasons, tasks can be stored out-of-memory in a database.
+# Currently, only sqlite and mongodb are supported as backends, but the interface
+# is fairly simple, so advanced developers could write their own backend.
+
+# ----- in-memory configuration --------
+# this line restores the default behavior: in-memory storage of all results.
+# c.HubFactory.db_class = 'IPython.parallel.controller.dictdb.DictDB'
+
+# ----- sqlite configuration --------
+# use this line to activate sqlite:
+# c.HubFactory.db_class = 'IPython.parallel.controller.sqlitedb.SQLiteDB'
+
+# You can specify the name of the db-file.  By default, this will be located
+# in the active cluster_dir, e.g. ~/.ipython/clusterz_default/tasks.db
+# c.SQLiteDB.filename = 'tasks.db'
+
+# You can also specify the location of the db-file, if you want it to be somewhere
+# other than the cluster_dir.
+# c.SQLiteDB.location = '/scratch/'
+
+# This will specify the name of the table for the controller to use.  The default
+# behavior is to use the session ID of the SessionFactory object (a uuid). Overriding
+# this will result in results persisting for multiple sessions.
+# c.SQLiteDB.table = 'results'
+
+# ----- mongodb configuration --------
+# use this line to activate mongodb:
+# c.HubFactory.db_class = 'IPython.parallel.controller.mongodb.MongoDB'
+
+# You can specify the args and kwargs pymongo will use when creating the Connection.
+# For more information on what these options might be, see pymongo documentation.
+# c.MongoDB.connection_kwargs = {}
+# c.MongoDB.connection_args = []
+
+# This will specify the name of the mongo database for the controller to use.  The default
+# behavior is to use the session ID of the SessionFactory object (a uuid). Overriding
+# this will result in task results persisting through multiple sessions.
+# c.MongoDB.database = 'ipythondb'
+
+

IPython/config/default/ipengine_config.py

@@ -1,90 +1,85 b''
 c = get_config()
 
 #-----------------------------------------------------------------------------
 # Global configuration
 #-----------------------------------------------------------------------------
 
 # Start up messages are logged to stdout using the logging module.
 # These all happen before the twisted reactor is started and are
 # useful for debugging purposes. Can be (10=DEBUG,20=INFO,30=WARN,40=CRITICAL) 
 # and smaller is more verbose.
 # c.Global.log_level = 20
 
 # Log to a file in cluster_dir/log, otherwise just log to sys.stdout.
 # c.Global.log_to_file = False
 
 # Remove old logs from cluster_dir/log before starting.
 # c.Global.clean_logs = True
 
 # A list of strings that will be executed in the users namespace on the engine
 # before it connects to the controller.
 # c.Global.exec_lines = ['import numpy']
 
 # The engine will try to connect to the controller multiple times, to allow
 # the controller time to startup and write its FURL file. These parameters 
 # control the number of retries (connect_max_tries) and the initial delay
 # (connect_delay) between attemps. The actual delay between attempts gets
 # longer each time by a factor of 1.5 (delay[i] = 1.5*delay[i-1])
 # those attemps.
 # c.Global.connect_delay = 0.1
 # c.Global.connect_max_tries = 15
 
-# By default, the engine will look for the controller's FURL file in its own
-# cluster directory. Sometimes, the FURL file will be elsewhere and this 
-# attribute can be set to the full path of the FURL file.
-# c.Global.furl_file = u''
+# By default, the engine will look for the controller's JSON file in its own
+# cluster directory. Sometimes, the JSON file will be elsewhere and this 
+# attribute can be set to the full path of the JSON file.
+# c.Global.url_file = u'/path/to/my/ipcontroller-engine.json'
 
 # The working directory for the process. The application will use os.chdir
 # to change to this directory before starting.
 # c.Global.work_dir = os.getcwd()
 
 #-----------------------------------------------------------------------------
 # MPI configuration
 #-----------------------------------------------------------------------------
 
 # Upon starting the engine can be configured to call MPI_Init. This section
 # configures that.
 
 # Select which MPI section to execute to setup MPI. The value of this 
 # attribute must match the name of another attribute in the MPI config 
 # section (mpi4py, pytrilinos, etc.). This can also be set by the --mpi
 # command line option.
 # c.MPI.use = ''
 
 # Initialize MPI using mpi4py. To use this, set c.MPI.use = 'mpi4py' to use
 # --mpi=mpi4py at the command line.
 # c.MPI.mpi4py = """from mpi4py import MPI as mpi
 # mpi.size = mpi.COMM_WORLD.Get_size()
 # mpi.rank = mpi.COMM_WORLD.Get_rank()
 # """
 
 # Initialize MPI using pytrilinos. To use this, set c.MPI.use = 'pytrilinos' 
 # to use --mpi=pytrilinos at the command line.
 # c.MPI.pytrilinos = """from PyTrilinos import Epetra
 # class SimpleStruct:
 # pass
 # mpi = SimpleStruct()
 # mpi.rank = 0
 # mpi.size = 0
 # """
 
 #-----------------------------------------------------------------------------
 # Developer level configuration attributes
 #-----------------------------------------------------------------------------
 
 # You shouldn't have to modify anything in this section. These attributes
 # are more for developers who want to change the behavior of the controller
 # at a fundamental level.
 
 # You should not have to change these attributes.
 
-# c.Global.shell_class = 'IPython.kernel.core.interpreter.Interpreter'
-
-# c.Global.furl_file_name = u'ipcontroller-engine.furl'
-
-
-
+# c.Global.url_file_name = u'ipcontroller-engine.furl'