propagate iopub to clients

IPython/zmq/displayhook.py

@@ -1,21 +1,23 b''
 import __builtin__
 
 from session import extract_header
 
 class DisplayHook(object):
 
+    topic=None
+
     def __init__(self, session, pub_socket):
         self.session = session
         self.pub_socket = pub_socket
         self.parent_header = {}
 
     def __call__(self, obj):
         if obj is None:
             return
 
         __builtin__._ = obj
         msg = self.session.send(self.pub_socket, u'pyout', {u'data':repr(obj)},
-                               parent=self.parent_header)
+                               parent=self.parent_header, ident=self.topic)
 
     def set_parent(self, parent):
         self.parent_header = extract_header(parent)
  No newline at end of file

IPython/zmq/iostream.py

@@ -1,96 +1,97 b''
 import logging
 import sys
 import time
 from cStringIO import StringIO
 
 from session import extract_header, Message
 
 from IPython.utils import io
 
 #-----------------------------------------------------------------------------
 # Globals
 #-----------------------------------------------------------------------------
 
 # Module-level logger
 logger =  logging.getLogger(__name__)
 
 #-----------------------------------------------------------------------------
 # Stream classes
 #-----------------------------------------------------------------------------
 
 class OutStream(object):
     """A file like object that publishes the stream to a 0MQ PUB socket."""
 
     # The time interval between automatic flushes, in seconds.
     flush_interval = 0.05
-    
+    topic=None
+
     def __init__(self, session, pub_socket, name):
         self.session = session
         self.pub_socket = pub_socket
         self.name = name
         self.parent_header = {}
         self._new_buffer()
 
     def set_parent(self, parent):
         self.parent_header = extract_header(parent)
 
     def close(self):
         self.pub_socket = None
 
     def flush(self):
         #io.rprint('>>>flushing output buffer: %s<<<' % self.name)  # dbg
         if self.pub_socket is None:
             raise ValueError(u'I/O operation on closed file')
         else:
             data = self._buffer.getvalue()
             if data:
                 # Make sure that we're handling unicode
                 if not isinstance(data, unicode):
                     enc = sys.stdin.encoding or sys.getdefaultencoding()
                     data = data.decode(enc, 'replace')
                 content = {u'name':self.name, u'data':data}
-                msg = self.session.send(self.pub_socket, u'stream',
-                                        content=content,
-                                        parent=self.parent_header)
+                msg = self.session.send(self.pub_socket, u'stream', content=content,
+                                       parent=self.parent_header, ident=self.topic)
                 logger.debug(msg)
+
                 self._buffer.close()
                 self._new_buffer()
 
     def isatty(self):
         return False
 
     def next(self):
         raise IOError('Read not supported on a write only stream.')
 
     def read(self, size=-1):
         raise IOError('Read not supported on a write only stream.')
 
     def readline(self, size=-1):
         raise IOError('Read not supported on a write only stream.')
 
     def write(self, string):
         if self.pub_socket is None:
             raise ValueError('I/O operation on closed file')
         else:
             # We can only send raw bytes, not unicode objects, so we encode
             # into utf-8 for all frontends if we get unicode inputs.
             if type(string) == unicode:
                 string = string.encode('utf-8')
                 
             self._buffer.write(string)
             current_time = time.time()
             if self._start <= 0:
                 self._start = current_time
             elif current_time - self._start > self.flush_interval:
                 self.flush()
 
     def writelines(self, sequence):
         if self.pub_socket is None:
             raise ValueError('I/O operation on closed file')
         else:
             for string in sequence:
                 self.write(string)
 
     def _new_buffer(self):
         self._buffer = StringIO()
         self._start = -1

IPython/zmq/parallel/asyncresult.py

@@ -1,186 +1,188 b''
 """AsyncResult objects for the client"""
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2010  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 from IPython.external.decorator import decorator
 import error
 
 #-----------------------------------------------------------------------------
 # Classes
 #-----------------------------------------------------------------------------
 
 @decorator
 def check_ready(f, self, *args, **kwargs):
     """Call spin() to sync state prior to calling the method."""
     self.wait(0)
     if not self._ready:
         raise error.TimeoutError("result not ready")
     return f(self, *args, **kwargs)
 
 class AsyncResult(object):
     """Class for representing results of non-blocking calls.
     
     Provides the same interface as :py:class:`multiprocessing.AsyncResult`.
     """
     def __init__(self, client, msg_ids, fname=''):
         self._client = client
         self.msg_ids = msg_ids
         self._fname=fname
         self._ready = False
         self._success = None
     
     def __repr__(self):
         if self._ready:
             return "<%s: finished>"%(self.__class__.__name__)
         else:
             return "<%s: %s>"%(self.__class__.__name__,self._fname)
     
     
     def _reconstruct_result(self, res):
         """
         Override me in subclasses for turning a list of results
         into the expected form.
         """
         if len(self.msg_ids) == 1:
             return res[0]
         else:
             return res
         
     def get(self, timeout=-1):
         """Return the result when it arrives. 
         
         If `timeout` is not ``None`` and the result does not arrive within
         `timeout` seconds then ``TimeoutError`` is raised. If the
         remote call raised an exception then that exception will be reraised
         by get().
         """
         if not self.ready():
             self.wait(timeout)
         
         if self._ready:
             if self._success:
                 return self._result
             else:
                 raise self._exception
         else:
             raise error.TimeoutError("Result not ready.")
     
     def ready(self):
         """Return whether the call has completed."""
         if not self._ready:
             self.wait(0)
         return self._ready
     
     def wait(self, timeout=-1):
         """Wait until the result is available or until `timeout` seconds pass.
         """
         if self._ready:
             return
         self._ready = self._client.barrier(self.msg_ids, timeout)
         if self._ready:
             try:
                 results = map(self._client.results.get, self.msg_ids)
                 self._result = results
                 results = error.collect_exceptions(results, self._fname)
                 self._result = self._reconstruct_result(results)
             except Exception, e:
                 self._exception = e
                 self._success = False
             else:
                 self._success = True
             finally:
                 self._metadata = map(self._client.metadata.get, self.msg_ids)
             
     
     def successful(self):
         """Return whether the call completed without raising an exception. 
         
         Will raise ``AssertionError`` if the result is not ready.
         """
         assert self._ready
         return self._success
     
     #----------------------------------------------------------------
     # Extra methods not in mp.pool.AsyncResult
     #----------------------------------------------------------------
     
     def get_dict(self, timeout=-1):
         """Get the results as a dict, keyed by engine_id."""
         results = self.get(timeout)
         engine_ids = [md['engine_id'] for md in self._metadata ]
         bycount = sorted(engine_ids, key=lambda k: engine_ids.count(k))
         maxcount = bycount.count(bycount[-1])
         if maxcount > 1:
             raise ValueError("Cannot build dict, %i jobs ran on engine #%i"%(
                     maxcount, bycount[-1]))
         
         return dict(zip(engine_ids,results))
     
     @property
     @check_ready
     def result(self):
         """result property."""
         return self._result
     
     @property
     @check_ready
     def metadata(self):
         """metadata property."""
         return self._metadata
     
     @property
-    @check_ready
     def result_dict(self):
         """result property as a dict."""
         return self.get_dict(0)
     
+    def __dict__(self):
+        return self.get_dict(0)
+
     #-------------------------------------
     # dict-access
     #-------------------------------------
     
     @check_ready
     def __getitem__(self, key):
         """getitem returns result value(s) if keyed by int/slice, or metadata if key is str.
         """
         if isinstance(key, int):
             return error.collect_exceptions([self._result[key]], self._fname)[0]
         elif isinstance(key, slice):
             return error.collect_exceptions(self._result[key], self._fname)
         elif isinstance(key, basestring):
             return [ md[key] for md in self._metadata ]
         else:
             raise TypeError("Invalid key type %r, must be 'int','slice', or 'str'"%type(key))
     
     @check_ready
     def __getattr__(self, key):
         """getattr maps to getitem for convenient access to metadata."""
         if key not in self._metadata[0].keys():
             raise AttributeError("%r object has no attribute %r"%(
                     self.__class__.__name__, key))
         return self.__getitem__(key)
 
     
 class AsyncMapResult(AsyncResult):
     """Class for representing results of non-blocking gathers.
     
     This will properly reconstruct the gather.
     """
     
     def __init__(self, client, msg_ids, mapObject, fname=''):
         self._mapObject = mapObject
         AsyncResult.__init__(self, client, msg_ids, fname=fname)
     
     def _reconstruct_result(self, res):
         """Perform the gather on the actual results."""
         return self._mapObject.joinPartitions(res)
     
         
 __all__ = ['AsyncResult', 'AsyncMapResult']
  No newline at end of file

IPython/zmq/parallel/client.py

@@ -1,1105 +1,1177 b''
 """A semi-synchronous Client for the ZMQ controller"""
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2010  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import os
 import time
 from getpass import getpass
 from pprint import pprint
 from datetime import datetime
 
 import zmq
 from zmq.eventloop import ioloop, zmqstream
 
 from IPython.external.decorator import decorator
 from IPython.zmq import tunnel
 
 import streamsession as ss
 # from remotenamespace import RemoteNamespace
 from view import DirectView, LoadBalancedView
 from dependency import Dependency, depend, require
 import error
 import map as Map
 from asyncresult import AsyncResult, AsyncMapResult
 from remotefunction import remote,parallel,ParallelFunction,RemoteFunction
 from util import ReverseDict
 
 #--------------------------------------------------------------------------
 # helpers for implementing old MEC API via client.apply
 #--------------------------------------------------------------------------
 
 def _push(ns):
     """helper method for implementing `client.push` via `client.apply`"""
     globals().update(ns)
 
 def _pull(keys):
     """helper method for implementing `client.pull` via `client.apply`"""
     g = globals()
     if isinstance(keys, (list,tuple, set)):
         for key in keys:
             if not g.has_key(key):
                 raise NameError("name '%s' is not defined"%key)
         return map(g.get, keys)
     else:
         if not g.has_key(keys):
             raise NameError("name '%s' is not defined"%keys)
         return g.get(keys)
 
 def _clear():
     """helper method for implementing `client.clear` via `client.apply`"""
     globals().clear()
 
 def _execute(code):
     """helper method for implementing `client.execute` via `client.apply`"""
     exec code in globals()
     
 
 #--------------------------------------------------------------------------
 # Decorators for Client methods
 #--------------------------------------------------------------------------
 
 @decorator
 def spinfirst(f, self, *args, **kwargs):
     """Call spin() to sync state prior to calling the method."""
     self.spin()
     return f(self, *args, **kwargs)
 
 @decorator
 def defaultblock(f, self, *args, **kwargs):
     """Default to self.block; preserve self.block."""
     block = kwargs.get('block',None)
     block = self.block if block is None else block
     saveblock = self.block
     self.block = block
     try:
         ret = f(self, *args, **kwargs)
     finally:
         self.block = saveblock
     return ret
 
 
 #--------------------------------------------------------------------------
 # Classes
 #--------------------------------------------------------------------------
 
 class AbortedTask(object):
     """A basic wrapper object describing an aborted task."""
     def __init__(self, msg_id):
         self.msg_id = msg_id
 
 class ResultDict(dict):
     """A subclass of dict that raises errors if it has them."""
     def __getitem__(self, key):
         res = dict.__getitem__(self, key)
         if isinstance(res, error.KernelError):
             raise res
         return res
 
+class Metadata(dict):
+    """Subclass of dict for initializing metadata values."""
+    def __init__(self, *args, **kwargs):
+        dict.__init__(self)
+        md = {'msg_id' : None,
+              'submitted' : None,
+              'started' : None,
+              'completed' : None,
+              'received' : None,
+              'engine_uuid' : None,
+              'engine_id' : None,
+              'follow' : None,
+              'after' : None,
+              'status' : None,
+
+              'pyin' : None,
+              'pyout' : None,
+              'pyerr' : None,
+              'stdout' : '',
+              'stderr' : '',
+            }
+        self.update(md)
+        self.update(dict(*args, **kwargs))
+
+        
+
 class Client(object):
     """A semi-synchronous client to the IPython ZMQ controller
     
     Parameters
     ----------
     
     addr : bytes; zmq url, e.g. 'tcp://127.0.0.1:10101'
         The address of the controller's registration socket.
         [Default: 'tcp://127.0.0.1:10101']
     context : zmq.Context
         Pass an existing zmq.Context instance, otherwise the client will create its own
     username : bytes
         set username to be passed to the Session object
     debug : bool
         flag for lots of message printing for debug purposes
  
     #-------------- ssh related args ----------------
     # These are args for configuring the ssh tunnel to be used
     # credentials are used to forward connections over ssh to the Controller
     # Note that the ip given in `addr` needs to be relative to sshserver
     # The most basic case is to leave addr as pointing to localhost (127.0.0.1),
     # and set sshserver as the same machine the Controller is on. However, 
     # the only requirement is that sshserver is able to see the Controller
     # (i.e. is within the same trusted network).
     
     sshserver : str
         A string of the form passed to ssh, i.e. 'server.tld' or '[e-mail]:port'
         If keyfile or password is specified, and this is not, it will default to
         the ip given in addr.
     sshkey : str; path to public ssh key file
         This specifies a key to be used in ssh login, default None.
         Regular default ssh keys will be used without specifying this argument.
     password : str; 
         Your ssh password to sshserver. Note that if this is left None,
         you will be prompted for it if passwordless key based login is unavailable.
     
     #------- exec authentication args -------
     # If even localhost is untrusted, you can have some protection against
     # unauthorized execution by using a key.  Messages are still sent
     # as cleartext, so if someone can snoop your loopback traffic this will
     # not help anything.
     
     exec_key : str
         an authentication key or file containing a key
         default: None
     
     
     Attributes
     ----------
     ids : set of int engine IDs
         requesting the ids attribute always synchronizes
         the registration state. To request ids without synchronization,
         use semi-private _ids attributes.
     
     history : list of msg_ids
         a list of msg_ids, keeping track of all the execution
         messages you have submitted in order.
     
     outstanding : set of msg_ids
         a set of msg_ids that have been submitted, but whose
         results have not yet been received.
     
     results : dict
         a dict of all our results, keyed by msg_id
     
     block : bool
         determines default behavior when block not specified
         in execution methods
     
     Methods
     -------
     spin : flushes incoming results and registration state changes
             control methods spin, and requesting `ids` also ensures up to date
             
     barrier : wait on one or more msg_ids
     
     execution methods: apply/apply_bound/apply_to/apply_bound
         legacy: execute, run
     
     query methods: queue_status, get_result, purge
     
     control methods: abort, kill
     
     """
     
     
     _connected=False
     _ssh=False
     _engines=None
     _addr='tcp://127.0.0.1:10101'
     _registration_socket=None
     _query_socket=None
     _control_socket=None
+    _iopub_socket=None
     _notification_socket=None
     _mux_socket=None
     _task_socket=None
     block = False
     outstanding=None
     results = None
     history = None
     debug = False
     targets = None
     
     def __init__(self, addr='tcp://127.0.0.1:10101', context=None, username=None, debug=False, 
             sshserver=None, sshkey=None, password=None, paramiko=None,
             exec_key=None,):
         if context is None:
             context = zmq.Context()
         self.context = context
         self.targets = 'all'
         self._addr = addr
         self._ssh = bool(sshserver or sshkey or password)
         if self._ssh and sshserver is None:
             # default to the same
             sshserver = addr.split('://')[1].split(':')[0]
         if self._ssh and password is None:
             if tunnel.try_passwordless_ssh(sshserver, sshkey, paramiko):
                 password=False
             else:
                 password = getpass("SSH Password for %s: "%sshserver)
         ssh_kwargs = dict(keyfile=sshkey, password=password, paramiko=paramiko)
         
         if exec_key is not None and os.path.isfile(exec_key):
             arg = 'keyfile'
         else:
             arg = 'key'
         key_arg = {arg:exec_key}
         if username is None:
             self.session = ss.StreamSession(**key_arg)
         else:
             self.session = ss.StreamSession(username, **key_arg)
         self._registration_socket = self.context.socket(zmq.XREQ)
         self._registration_socket.setsockopt(zmq.IDENTITY, self.session.session)
         if self._ssh:
             tunnel.tunnel_connection(self._registration_socket, addr, sshserver, **ssh_kwargs)
         else:
             self._registration_socket.connect(addr)
         self._engines = ReverseDict()
         self._ids = set()
         self.outstanding=set()
         self.results = {}
         self.metadata = {}
         self.history = []
         self.debug = debug
         self.session.debug = debug
         
         self._notification_handlers = {'registration_notification' : self._register_engine,
                                     'unregistration_notification' : self._unregister_engine,
                                     }
         self._queue_handlers = {'execute_reply' : self._handle_execute_reply,
                                 'apply_reply' : self._handle_apply_reply}
         self._connect(sshserver, ssh_kwargs)
         
     
     @property
     def ids(self):
         """Always up to date ids property."""
         self._flush_notifications()
         return self._ids
     
     def _update_engines(self, engines):
         """Update our engines dict and _ids from a dict of the form: {id:uuid}."""
         for k,v in engines.iteritems():
             eid = int(k)
             self._engines[eid] = bytes(v) # force not unicode
             self._ids.add(eid)
     
     def _build_targets(self, targets):
         """Turn valid target IDs or 'all' into two lists:
         (int_ids, uuids).
         """
         if targets is None:
             targets = self._ids
         elif isinstance(targets, str):
             if targets.lower() == 'all':
                 targets = self._ids
             else:
                 raise TypeError("%r not valid str target, must be 'all'"%(targets))
         elif isinstance(targets, int):
             targets = [targets]
         return [self._engines[t] for t in targets], list(targets)
     
     def _connect(self, sshserver, ssh_kwargs):
         """setup all our socket connections to the controller. This is called from
         __init__."""
         if self._connected:
             return
         self._connected=True
 
         def connect_socket(s, addr):
             if self._ssh:
                 return tunnel.tunnel_connection(s, addr, sshserver, **ssh_kwargs)
             else:
                 return s.connect(addr)
             
         self.session.send(self._registration_socket, 'connection_request')
         idents,msg = self.session.recv(self._registration_socket,mode=0)
         if self.debug:
             pprint(msg)
         msg = ss.Message(msg)
         content = msg.content
         if content.status == 'ok':
             if content.queue:
                 self._mux_socket = self.context.socket(zmq.PAIR)
                 self._mux_socket.setsockopt(zmq.IDENTITY, self.session.session)
                 connect_socket(self._mux_socket, content.queue)
             if content.task:
                 self._task_socket = self.context.socket(zmq.PAIR)
                 self._task_socket.setsockopt(zmq.IDENTITY, self.session.session)
                 connect_socket(self._task_socket, content.task)
             if content.notification:
                 self._notification_socket = self.context.socket(zmq.SUB)
                 connect_socket(self._notification_socket, content.notification)
                 self._notification_socket.setsockopt(zmq.SUBSCRIBE, "")
             if content.query:
                 self._query_socket = self.context.socket(zmq.PAIR)
                 self._query_socket.setsockopt(zmq.IDENTITY, self.session.session)
                 connect_socket(self._query_socket, content.query)
             if content.control:
                 self._control_socket = self.context.socket(zmq.PAIR)
                 self._control_socket.setsockopt(zmq.IDENTITY, self.session.session)
                 connect_socket(self._control_socket, content.control)
+            if content.iopub:
+                self._iopub_socket = self.context.socket(zmq.SUB)
+                self._iopub_socket.setsockopt(zmq.SUBSCRIBE, '')
+                self._iopub_socket.setsockopt(zmq.IDENTITY, self.session.session)
+                connect_socket(self._iopub_socket, content.iopub)
             self._update_engines(dict(content.engines))
                 
         else:
             self._connected = False
             raise Exception("Failed to connect!")
     
     #--------------------------------------------------------------------------
     # handlers and callbacks for incoming messages
     #--------------------------------------------------------------------------
     
     def _register_engine(self, msg):
         """Register a new engine, and update our connection info."""
         content = msg['content']
         eid = content['id']
         d = {eid : content['queue']}
         self._update_engines(d)
         self._ids.add(int(eid))
 
     def _unregister_engine(self, msg):
         """Unregister an engine that has died."""
         content = msg['content']
         eid = int(content['id'])
         if eid in self._ids:
             self._ids.remove(eid)
             self._engines.pop(eid)
-    #
-    def _build_metadata(self, header, parent, content):
+    
+    def _extract_metadata(self, header, parent, content):
         md = {'msg_id' : parent['msg_id'],
               'submitted' : datetime.strptime(parent['date'], ss.ISO8601),
               'started' : datetime.strptime(header['started'], ss.ISO8601),
               'completed' : datetime.strptime(header['date'], ss.ISO8601),
               'received' : datetime.now(),
               'engine_uuid' : header['engine'],
               'engine_id' : self._engines.get(header['engine'], None),
               'follow' : parent['follow'],
               'after' : parent['after'],
-              'status' : content['status']
-             }
+              'status' : content['status'],
+            }
         return md
     
     def _handle_execute_reply(self, msg):
         """Save the reply to an execute_request into our results.
         
         execute messages are never actually used. apply is used instead.
         """
         
         parent = msg['parent_header']
         msg_id = parent['msg_id']
         if msg_id not in self.outstanding:
             print("got unknown result: %s"%msg_id)
         else:
             self.outstanding.remove(msg_id)
         self.results[msg_id] = ss.unwrap_exception(msg['content'])
     
     def _handle_apply_reply(self, msg):
         """Save the reply to an apply_request into our results."""
         parent = msg['parent_header']
         msg_id = parent['msg_id']
         if msg_id not in self.outstanding:
             print ("got unknown result: %s"%msg_id)
         else:
             self.outstanding.remove(msg_id)
         content = msg['content']
         header = msg['header']
         
-        self.metadata[msg_id] = self._build_metadata(header, parent, content)
+        # construct metadata:
+        md = self.metadata.setdefault(msg_id, Metadata())
+        md.update(self._extract_metadata(header, parent, content))
+        self.metadata[msg_id] = md
         
+        # construct result:
         if content['status'] == 'ok':
             self.results[msg_id] = ss.unserialize_object(msg['buffers'])[0]
         elif content['status'] == 'aborted':
             self.results[msg_id] = error.AbortedTask(msg_id)
         elif content['status'] == 'resubmitted':
             # TODO: handle resubmission
             pass
         else:
             e = ss.unwrap_exception(content)
             e_uuid = e.engine_info['engineid']
             eid = self._engines[e_uuid]
             e.engine_info['engineid'] = eid
             self.results[msg_id] = e
     
     def _flush_notifications(self):
         """Flush notifications of engine registrations waiting
         in ZMQ queue."""
         msg = self.session.recv(self._notification_socket, mode=zmq.NOBLOCK)
         while msg is not None:
             if self.debug:
                 pprint(msg)
             msg = msg[-1]
             msg_type = msg['msg_type']
             handler = self._notification_handlers.get(msg_type, None)
             if handler is None:
                 raise Exception("Unhandled message type: %s"%msg.msg_type)
             else:
                 handler(msg)
             msg = self.session.recv(self._notification_socket, mode=zmq.NOBLOCK)
     
     def _flush_results(self, sock):
         """Flush task or queue results waiting in ZMQ queue."""
         msg = self.session.recv(sock, mode=zmq.NOBLOCK)
         while msg is not None:
             if self.debug:
                 pprint(msg)
             msg = msg[-1]
             msg_type = msg['msg_type']
             handler = self._queue_handlers.get(msg_type, None)
             if handler is None:
                 raise Exception("Unhandled message type: %s"%msg.msg_type)
             else:
                 handler(msg)
             msg = self.session.recv(sock, mode=zmq.NOBLOCK)
     
     def _flush_control(self, sock):
         """Flush replies from the control channel waiting
         in the ZMQ queue.
         
         Currently: ignore them."""
         msg = self.session.recv(sock, mode=zmq.NOBLOCK)
         while msg is not None:
             if self.debug:
                 pprint(msg)
             msg = self.session.recv(sock, mode=zmq.NOBLOCK)
     
+    def _flush_iopub(self, sock):
+        """Flush replies from the iopub channel waiting
+        in the ZMQ queue.
+        """
+        msg = self.session.recv(sock, mode=zmq.NOBLOCK)
+        while msg is not None:
+            if self.debug:
+                pprint(msg)
+            msg = msg[-1]
+            parent = msg['parent_header']
+            msg_id = parent['msg_id']
+            content = msg['content']
+            header = msg['header']
+            msg_type = msg['msg_type']
+            
+            # init metadata:
+            md = self.metadata.setdefault(msg_id, Metadata())
+            
+            if msg_type == 'stream':
+                name = content['name']
+                s = md[name] or ''
+                md[name] = s + content['data']
+            elif msg_type == 'pyerr':
+                md.update({'pyerr' : ss.unwrap_exception(content)})
+            else:
+                md.update({msg_type : content['data']})
+            
+            self.metadata[msg_id] = md
+            
+            msg = self.session.recv(sock, mode=zmq.NOBLOCK)
+    
     #--------------------------------------------------------------------------
     # getitem
     #--------------------------------------------------------------------------
     
     def __getitem__(self, key):
         """Dict access returns DirectView multiplexer objects or,
         if key is None, a LoadBalancedView."""
         if key is None:
             return LoadBalancedView(self)
         if isinstance(key, int):
             if key not in self.ids:
                 raise IndexError("No such engine: %i"%key)
             return DirectView(self, key)
         
         if isinstance(key, slice):
             indices = range(len(self.ids))[key]
             ids = sorted(self._ids)
             key = [ ids[i] for i in indices ]
             # newkeys = sorted(self._ids)[thekeys[k]]
         
         if isinstance(key, (tuple, list, xrange)):
             _,targets = self._build_targets(list(key))
             return DirectView(self, targets)
         else:
             raise TypeError("key by int/iterable of ints only, not %s"%(type(key)))
     
     #--------------------------------------------------------------------------
     # Begin public methods
     #--------------------------------------------------------------------------
     
     @property
     def remote(self):
         """property for convenient RemoteFunction generation.
         
         >>> @client.remote
         ... def f():
                 import os
                 print (os.getpid())
         """
         return remote(self, block=self.block)
     
     def spin(self):
         """Flush any registration notifications and execution results
         waiting in the ZMQ queue.
         """
         if self._notification_socket:
             self._flush_notifications()
         if self._mux_socket:
             self._flush_results(self._mux_socket)
         if self._task_socket:
             self._flush_results(self._task_socket)
         if self._control_socket:
             self._flush_control(self._control_socket)
+        if self._iopub_socket:
+            self._flush_iopub(self._iopub_socket)
     
     def barrier(self, msg_ids=None, timeout=-1):
         """waits on one or more `msg_ids`, for up to `timeout` seconds.
         
         Parameters
         ----------
         msg_ids : int, str, or list of ints and/or strs, or one or more AsyncResult objects
                 ints are indices to self.history
                 strs are msg_ids
                 default: wait on all outstanding messages
         timeout : float
                 a time in seconds, after which to give up.
                 default is -1, which means no timeout
         
         Returns
         -------
         True : when all msg_ids are done
         False : timeout reached, some msg_ids still outstanding
         """
         tic = time.time()
         if msg_ids is None:
             theids = self.outstanding
         else:
             if isinstance(msg_ids, (int, str, AsyncResult)):
                 msg_ids = [msg_ids]
             theids = set()
             for msg_id in msg_ids:
                 if isinstance(msg_id, int):
                     msg_id = self.history[msg_id]
                 elif isinstance(msg_id, AsyncResult):
                     map(theids.add, msg_id.msg_ids)
                     continue
                 theids.add(msg_id)
         if not theids.intersection(self.outstanding):
             return True
         self.spin()
         while theids.intersection(self.outstanding):
             if timeout >= 0 and ( time.time()-tic ) > timeout:
                 break
             time.sleep(1e-3)
             self.spin()
         return len(theids.intersection(self.outstanding)) == 0
     
     #--------------------------------------------------------------------------
     # Control methods
     #--------------------------------------------------------------------------
     
     @spinfirst
     @defaultblock
     def clear(self, targets=None, block=None):
         """Clear the namespace in target(s)."""
         targets = self._build_targets(targets)[0]
         for t in targets:
             self.session.send(self._control_socket, 'clear_request', content={}, ident=t)
         error = False
         if self.block:
             for i in range(len(targets)):
                 idents,msg = self.session.recv(self._control_socket,0)
                 if self.debug:
                     pprint(msg)
                 if msg['content']['status'] != 'ok':
                     error = ss.unwrap_exception(msg['content'])
         if error:
             return error
         
     
     @spinfirst
     @defaultblock
     def abort(self, msg_ids = None, targets=None, block=None):
         """Abort the execution queues of target(s)."""
         targets = self._build_targets(targets)[0]
         if isinstance(msg_ids, basestring):
             msg_ids = [msg_ids]
         content = dict(msg_ids=msg_ids)
         for t in targets:
             self.session.send(self._control_socket, 'abort_request', 
                     content=content, ident=t)
         error = False
         if self.block:
             for i in range(len(targets)):
                 idents,msg = self.session.recv(self._control_socket,0)
                 if self.debug:
                     pprint(msg)
                 if msg['content']['status'] != 'ok':
                     error = ss.unwrap_exception(msg['content'])
         if error:
             return error
     
     @spinfirst
     @defaultblock
     def shutdown(self, targets=None, restart=False, controller=False, block=None):
         """Terminates one or more engine processes, optionally including the controller."""
         if controller:
             targets = 'all'
         targets = self._build_targets(targets)[0]
         for t in targets:
             self.session.send(self._control_socket, 'shutdown_request', 
                         content={'restart':restart},ident=t)
         error = False
         if block or controller:
             for i in range(len(targets)):
                 idents,msg = self.session.recv(self._control_socket,0)
                 if self.debug:
                     pprint(msg)
                 if msg['content']['status'] != 'ok':
                     error = ss.unwrap_exception(msg['content'])
         
         if controller:
             time.sleep(0.25)
             self.session.send(self._query_socket, 'shutdown_request')
             idents,msg = self.session.recv(self._query_socket, 0)
             if self.debug:
                 pprint(msg)
             if msg['content']['status'] != 'ok':
                 error = ss.unwrap_exception(msg['content'])
         
         if error:
             raise error
     
     #--------------------------------------------------------------------------
     # Execution methods
     #--------------------------------------------------------------------------
     
     @defaultblock
     def execute(self, code, targets='all', block=None):
         """Executes `code` on `targets` in blocking or nonblocking manner.
         
         ``execute`` is always `bound` (affects engine namespace)
         
         Parameters
         ----------
         code : str
                 the code string to be executed
         targets : int/str/list of ints/strs
                 the engines on which to execute
                 default : all
         block : bool
                 whether or not to wait until done to return
                 default: self.block
         """
         result = self.apply(_execute, (code,), targets=targets, block=self.block, bound=True)
         return result
     
     def run(self, code, block=None):
         """Runs `code` on an engine. 
         
         Calls to this are load-balanced.
         
         ``run`` is never `bound` (no effect on engine namespace)
         
         Parameters
         ----------
         code : str
                 the code string to be executed
         block : bool
                 whether or not to wait until done
         
         """
         result = self.apply(_execute, (code,), targets=None, block=block, bound=False)
         return result
     
     def _maybe_raise(self, result):
         """wrapper for maybe raising an exception if apply failed."""
         if isinstance(result, error.RemoteError):
             raise result
         
         return result
             
     def apply(self, f, args=None, kwargs=None, bound=True, block=None, targets=None,
                         after=None, follow=None):
         """Call `f(*args, **kwargs)` on a remote engine(s), returning the result.
         
         This is the central execution command for the client.
         
         Parameters
         ----------
         
         f : function
             The fuction to be called remotely
         args : tuple/list
             The positional arguments passed to `f`
         kwargs : dict
             The keyword arguments passed to `f`
         bound : bool (default: True)
             Whether to execute in the Engine(s) namespace, or in a clean
             namespace not affecting the engine.
         block : bool (default: self.block)
             Whether to wait for the result, or return immediately.
             False:
                 returns msg_id(s)
                 if multiple targets:
                     list of ids
             True:
                 returns actual result(s) of f(*args, **kwargs)
                 if multiple targets:
                     dict of results, by engine ID
         targets : int,list of ints, 'all', None
             Specify the destination of the job.
             if None:
                 Submit via Task queue for load-balancing.
             if 'all':
                 Run on all active engines
             if list:
                 Run on each specified engine
             if int:
                 Run on single engine
             
         after : Dependency or collection of msg_ids
             Only for load-balanced execution (targets=None)
             Specify a list of msg_ids as a time-based dependency.
             This job will only be run *after* the dependencies
             have been met.
             
         follow : Dependency or collection of msg_ids
             Only for load-balanced execution (targets=None)
             Specify a list of msg_ids as a location-based dependency.
             This job will only be run on an engine where this dependency
             is met.
         
         Returns
         -------
         if block is False:
             if single target:
                 return msg_id
             else:
                 return list of msg_ids
                 ? (should this be dict like block=True) ?
         else:
             if single target:
                 return result of f(*args, **kwargs)
             else:
                 return dict of results, keyed by engine
         """
         
         # defaults:
         block = block if block is not None else self.block
         args = args if args is not None else []
         kwargs = kwargs if kwargs is not None else {}
         
         # enforce types of f,args,kwrags
         if not callable(f):
             raise TypeError("f must be callable, not %s"%type(f))
         if not isinstance(args, (tuple, list)):
             raise TypeError("args must be tuple or list, not %s"%type(args))
         if not isinstance(kwargs, dict):
             raise TypeError("kwargs must be dict, not %s"%type(kwargs))
             
         if isinstance(after, Dependency):
             after = after.as_dict()
         elif isinstance(after, AsyncResult):
             after=after.msg_ids
         elif after is None:
             after = []
         if isinstance(follow, Dependency):
             follow = follow.as_dict()
         elif isinstance(follow, AsyncResult):
             follow=follow.msg_ids
         elif follow is None:
             follow = []
         options  = dict(bound=bound, block=block, after=after, follow=follow)
             
         if targets is None:
             return self._apply_balanced(f, args, kwargs, **options)
         else:
             return self._apply_direct(f, args, kwargs, targets=targets, **options)
     
     def _apply_balanced(self, f, args, kwargs, bound=True, block=None,
                             after=None, follow=None):
         """The underlying method for applying functions in a load balanced
         manner, via the task queue."""
         
         subheader = dict(after=after, follow=follow)
         bufs = ss.pack_apply_message(f,args,kwargs)
         content = dict(bound=bound)
         
         msg = self.session.send(self._task_socket, "apply_request", 
                 content=content, buffers=bufs, subheader=subheader)
         msg_id = msg['msg_id']
         self.outstanding.add(msg_id)
         self.history.append(msg_id)
         ar = AsyncResult(self, [msg_id], fname=f.__name__)
         if block:
             return ar.get()
         else:
             return ar
     
     def _apply_direct(self, f, args, kwargs, bound=True, block=None, targets=None,
                                 after=None, follow=None):
         """Then underlying method for applying functions to specific engines
         via the MUX queue."""
         
         queues,targets = self._build_targets(targets)
         
         subheader = dict(after=after, follow=follow)
         content = dict(bound=bound)
         bufs = ss.pack_apply_message(f,args,kwargs)
         
         msg_ids = []
         for queue in queues:
             msg = self.session.send(self._mux_socket, "apply_request", 
                     content=content, buffers=bufs,ident=queue, subheader=subheader)
             msg_id = msg['msg_id']
             self.outstanding.add(msg_id)
             self.history.append(msg_id)
             msg_ids.append(msg_id)
         ar = AsyncResult(self, msg_ids, fname=f.__name__)
         if block:
             return ar.get()
         else:
             return ar
     
     #--------------------------------------------------------------------------
     # Map and decorators
     #--------------------------------------------------------------------------
     
     def map(self, f, *sequences):
         """Parallel version of builtin `map`, using all our engines."""
         pf = ParallelFunction(self, f, block=self.block,
                         bound=True, targets='all')
         return pf.map(*sequences)
     
     def parallel(self, bound=True, targets='all', block=True):
         """Decorator for making a ParallelFunction."""
         return parallel(self, bound=bound, targets=targets, block=block)
     
     def remote(self, bound=True, targets='all', block=True):
         """Decorator for making a RemoteFunction."""
         return remote(self, bound=bound, targets=targets, block=block)
     
     #--------------------------------------------------------------------------
     # Data movement
     #--------------------------------------------------------------------------
     
     @defaultblock
     def push(self, ns, targets='all', block=None):
         """Push the contents of `ns` into the namespace on `target`"""
         if not isinstance(ns, dict):
             raise TypeError("Must be a dict, not %s"%type(ns))
         result = self.apply(_push, (ns,), targets=targets, block=block, bound=True)
         return result
     
     @defaultblock
     def pull(self, keys, targets='all', block=None):
         """Pull objects from `target`'s namespace by `keys`"""
         if isinstance(keys, str):
             pass
         elif isinstance(keys, (list,tuple,set)):
             for key in keys:
                 if not isinstance(key, str):
                     raise TypeError
         result = self.apply(_pull, (keys,), targets=targets, block=block, bound=True)
         return result
     
     def scatter(self, key, seq, dist='b', flatten=False, targets='all', block=None):
         """
         Partition a Python sequence and send the partitions to a set of engines.
         """
         block = block if block is not None else self.block
         targets = self._build_targets(targets)[-1]
         mapObject = Map.dists[dist]()
         nparts = len(targets)
         msg_ids = []
         for index, engineid in enumerate(targets):
             partition = mapObject.getPartition(seq, index, nparts)
             if flatten and len(partition) == 1:
                 r = self.push({key: partition[0]}, targets=engineid, block=False)
             else:
                 r = self.push({key: partition}, targets=engineid, block=False)
             msg_ids.extend(r.msg_ids)
         r = AsyncResult(self, msg_ids, fname='scatter')
         if block:
             return r.get()
         else:
             return r
     
     def gather(self, key, dist='b', targets='all', block=None):
         """
         Gather a partitioned sequence on a set of engines as a single local seq.
         """
         block = block if block is not None else self.block
         
         targets = self._build_targets(targets)[-1]
         mapObject = Map.dists[dist]()
         msg_ids = []
         for index, engineid in enumerate(targets):
             msg_ids.extend(self.pull(key, targets=engineid,block=False).msg_ids)
         
         r = AsyncMapResult(self, msg_ids, mapObject, fname='gather')
         if block:
             return r.get()
         else:
             return r
     
     #--------------------------------------------------------------------------
     # Query methods
     #--------------------------------------------------------------------------
     
     @spinfirst
     def get_results(self, msg_ids, status_only=False):
         """Returns the result of the execute or task request with `msg_ids`.
         
         Parameters
         ----------
         msg_ids : list of ints or msg_ids
             if int:
                 Passed as index to self.history for convenience.
         status_only : bool (default: False)
             if False:
                 return the actual results
         
         Returns
         -------
         
         results : dict
             There will always be the keys 'pending' and 'completed', which will
             be lists of msg_ids.
         """
         if not isinstance(msg_ids, (list,tuple)):
             msg_ids = [msg_ids]
         theids = []
         for msg_id in msg_ids:
             if isinstance(msg_id, int):
                 msg_id = self.history[msg_id]
             if not isinstance(msg_id, str):
                 raise TypeError("msg_ids must be str, not %r"%msg_id)
             theids.append(msg_id)
         
         completed = []
         local_results = {}
         # temporarily disable local shortcut
         # for msg_id in list(theids):
         #     if msg_id in self.results:
         #         completed.append(msg_id)
         #         local_results[msg_id] = self.results[msg_id]
         #         theids.remove(msg_id)
         
         if theids: # some not locally cached
             content = dict(msg_ids=theids, status_only=status_only)
             msg = self.session.send(self._query_socket, "result_request", content=content)
             zmq.select([self._query_socket], [], [])
             idents,msg = self.session.recv(self._query_socket, zmq.NOBLOCK)
             if self.debug:
                 pprint(msg)
             content = msg['content']
             if content['status'] != 'ok':
                 raise ss.unwrap_exception(content)
             buffers = msg['buffers']
         else:
             content = dict(completed=[],pending=[])
         
         content['completed'].extend(completed)
         
         if status_only:
             return content
         
         failures = []
         # load cached results into result:
         content.update(local_results)
         # update cache with results:
         for msg_id in sorted(theids):
             if msg_id in content['completed']:
                 rec = content[msg_id]
                 parent = rec['header']
                 header = rec['result_header']
                 rcontent = rec['result_content']
+                iodict = rec['io']
                 if isinstance(rcontent, str):
                     rcontent = self.session.unpack(rcontent)
                 
-                self.metadata[msg_id] = self._build_metadata(header, parent, rcontent)
+                md = self.metadata.setdefault(msg_id, Metadata())
+                md.update(self._extract_metadata(header, parent, rcontent))
+                md.update(iodict)
                 
                 if rcontent['status'] == 'ok':
                     res,buffers = ss.unserialize_object(buffers)
                 else:
                     res = ss.unwrap_exception(rcontent)
                     failures.append(res)
                 
                 self.results[msg_id] = res
                 content[msg_id] = res
         
         error.collect_exceptions(failures, "get_results")
         return content
 
     @spinfirst
     def queue_status(self, targets=None, verbose=False):
         """Fetch the status of engine queues.
         
         Parameters
         ----------
         targets : int/str/list of ints/strs
                 the engines on which to execute
                 default : all
         verbose : bool
                 Whether to return lengths only, or lists of ids for each element
         """
         targets = self._build_targets(targets)[1]
         content = dict(targets=targets, verbose=verbose)
         self.session.send(self._query_socket, "queue_request", content=content)
         idents,msg = self.session.recv(self._query_socket, 0)
         if self.debug:
             pprint(msg)
         content = msg['content']
         status = content.pop('status')
         if status != 'ok':
             raise ss.unwrap_exception(content)
         return ss.rekey(content)
         
     @spinfirst
     def purge_results(self, msg_ids=[], targets=[]):
         """Tell the controller to forget results.
         
         Individual results can be purged by msg_id, or the entire
         history of specific targets can be purged.
         
         Parameters
         ----------
         msg_ids : str or list of strs
                 the msg_ids whose results should be forgotten.
         targets : int/str/list of ints/strs
                 The targets, by uuid or int_id, whose entire history is to be purged.
                 Use `targets='all'` to scrub everything from the controller's memory.
                 
                 default : None
         """
         if not targets and not msg_ids:
             raise ValueError
         if targets:
             targets = self._build_targets(targets)[1]
         content = dict(targets=targets, msg_ids=msg_ids)
         self.session.send(self._query_socket, "purge_request", content=content)
         idents, msg = self.session.recv(self._query_socket, 0)
         if self.debug:
             pprint(msg)
         content = msg['content']
         if content['status'] != 'ok':
             raise ss.unwrap_exception(content)
 
     #----------------------------------------
     # activate for %px,%autopx magics
     #----------------------------------------
     def activate(self):
         """Make this `View` active for parallel magic commands.
         
         IPython has a magic command syntax to work with `MultiEngineClient` objects.
         In a given IPython session there is a single active one.  While
         there can be many `Views` created and used by the user, 
         there is only one active one.  The active `View` is used whenever 
         the magic commands %px and %autopx are used.
         
         The activate() method is called on a given `View` to make it 
         active.  Once this has been done, the magic commands can be used.
         """
         
         try:
             # This is injected into __builtins__.
             ip = get_ipython()
         except NameError:
             print "The IPython parallel magics (%result, %px, %autopx) only work within IPython."
         else:
             pmagic = ip.plugin_manager.get_plugin('parallelmagic')
             if pmagic is not None:
                 pmagic.active_multiengine_client = self
             else:
                 print "You must first load the parallelmagic extension " \
                       "by doing '%load_ext parallelmagic'"
 
 class AsynClient(Client):
     """An Asynchronous client, using the Tornado Event Loop.
     !!!unfinished!!!"""
     io_loop = None
     _queue_stream = None
     _notifier_stream = None
     _task_stream = None
     _control_stream = None
     
     def __init__(self, addr, context=None, username=None, debug=False, io_loop=None):
         Client.__init__(self, addr, context, username, debug)
         if io_loop is None:
             io_loop = ioloop.IOLoop.instance()
         self.io_loop = io_loop
         
         self._queue_stream = zmqstream.ZMQStream(self._mux_socket, io_loop)
         self._control_stream = zmqstream.ZMQStream(self._control_socket, io_loop)
         self._task_stream = zmqstream.ZMQStream(self._task_socket, io_loop)
         self._notification_stream = zmqstream.ZMQStream(self._notification_socket, io_loop)
     
     def spin(self):
         for stream in (self.queue_stream, self.notifier_stream, 
                         self.task_stream, self.control_stream):
             stream.flush()
 
 __all__ = [ 'Client', 
             'depend', 
             'require', 
             'remote',
             'parallel',
             'RemoteFunction',
             'ParallelFunction',
             'DirectView',
             'LoadBalancedView',
             'AsyncResult',
             'AsyncMapResult'
             ]

IPython/zmq/parallel/controller.py

@@ -1,227 +1,254 b''
 #!/usr/bin/env python
 """The IPython Controller with 0MQ
 This is the master object that handles connections from engines and clients,
 and monitors traffic through the various queues.
 """
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2010  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 from __future__ import print_function
 
 import os
 import time
 from multiprocessing import Process
 
 import zmq
 from zmq.eventloop import  ioloop
 from zmq.eventloop.zmqstream import ZMQStream
 from zmq.devices import ProcessMonitoredQueue
 
 # internal:
 from IPython.zmq.entry_point import bind_port
 
 from hub import Hub
 from entry_point import (make_base_argument_parser, select_random_ports, split_ports,
                         connect_logger, parse_url, signal_children, generate_exec_key)
 
 
 import streamsession as session
 import heartmonitor
 from scheduler import launch_scheduler
 
 from dictdb import DictDB
 try:
     import pymongo
 except ImportError:
     MongoDB=None
 else:
     from mongodb import MongoDB
     
 #-------------------------------------------------------------------------
 # Entry Point
 #-------------------------------------------------------------------------
 
 def make_argument_parser():
     """Make an argument parser"""
     parser = make_base_argument_parser()
     
     parser.add_argument('--client', type=int, metavar='PORT', default=0,
                         help='set the XREP port for clients [default: random]')
     parser.add_argument('--notice', type=int, metavar='PORT', default=0,
                         help='set the PUB socket for registration notification [default: random]')
     parser.add_argument('--hb', type=str, metavar='PORTS',
                         help='set the 2 ports for heartbeats [default: random]')
-    parser.add_argument('--ping', type=int, default=3000,
-                        help='set the heartbeat period in ms [default: 3000]')
+    parser.add_argument('--ping', type=int, default=100,
+                        help='set the heartbeat period in ms [default: 100]')
     parser.add_argument('--monitor', type=int, metavar='PORT', default=0,
                         help='set the SUB port for queue monitoring [default: random]')
     parser.add_argument('--mux', type=str, metavar='PORTS',
                         help='set the XREP ports for the MUX queue [default: random]')
     parser.add_argument('--task', type=str, metavar='PORTS',
                         help='set the XREP/XREQ ports for the task queue [default: random]')
     parser.add_argument('--control', type=str, metavar='PORTS',
                         help='set the XREP ports for the control queue [default: random]')
-    parser.add_argument('--scheduler', type=str, default='pure',
+    parser.add_argument('--iopub', type=str, metavar='PORTS',
+                        help='set the PUB/SUB ports for the iopub relay [default: random]')
+    parser.add_argument('--scheduler', type=str, default='lru',
                         choices = ['pure', 'lru', 'plainrandom', 'weighted', 'twobin','leastload'],
-                        help='select the task scheduler  [default: pure ZMQ]')
+                        help='select the task scheduler  [default: Python LRU]')
     parser.add_argument('--mongodb', action='store_true',
                         help='Use MongoDB task storage [default: in-memory]')
+    parser.add_argument('--session', type=str, default=None,
+                        help='Manually specify the session id.')
     
     return parser
     
 def main(argv=None):
     
     parser = make_argument_parser()
     
     args = parser.parse_args(argv)
     parse_url(args)
     
     iface="%s://%s"%(args.transport,args.ip)+':%i'
     
     random_ports = 0
     if args.hb:
         hb = split_ports(args.hb, 2)
     else:
         hb = select_random_ports(2)
     if args.mux:
         mux = split_ports(args.mux, 2)
     else:
         mux = None
         random_ports += 2
+    if args.iopub:
+        iopub = split_ports(args.iopub, 2)
+    else:
+        iopub = None
+        random_ports += 2
     if args.task:
         task = split_ports(args.task, 2)
     else:
         task = None
         random_ports += 2
     if args.control:
         control = split_ports(args.control, 2)
     else:
         control = None
         random_ports += 2
     
     ctx = zmq.Context()
     loop = ioloop.IOLoop.instance()
     
     # setup logging
     connect_logger(ctx, iface%args.logport, root="controller", loglevel=args.loglevel)
     
     # Registrar socket
     reg = ZMQStream(ctx.socket(zmq.XREP), loop)
     regport = bind_port(reg, args.ip, args.regport)
     
     ### Engine connections ###
     
     # heartbeat
     hpub = ctx.socket(zmq.PUB)
     bind_port(hpub, args.ip, hb[0])
     hrep = ctx.socket(zmq.XREP)
     bind_port(hrep, args.ip, hb[1])
     
     hmon = heartmonitor.HeartMonitor(loop, ZMQStream(hpub,loop), ZMQStream(hrep,loop),args.ping)
     hmon.start()
     
     ### Client connections ###
     # Clientele socket
     c = ZMQStream(ctx.socket(zmq.XREP), loop)
     cport = bind_port(c, args.ip, args.client)
     # Notifier socket
     n = ZMQStream(ctx.socket(zmq.PUB), loop)
     nport = bind_port(n, args.ip, args.notice)
     
     ### Key File ###
     if args.execkey and not os.path.isfile(args.execkey):
             generate_exec_key(args.execkey)
     
-    thesession = session.StreamSession(username=args.ident or "controller", keyfile=args.execkey)
+    thesession = session.StreamSession(username=args.ident or "controller", 
+                    keyfile=args.execkey, session=args.session)
     
     ### build and launch the queues ###
     
     # monitor socket
     sub = ctx.socket(zmq.SUB)
     sub.setsockopt(zmq.SUBSCRIBE, "")
     monport = bind_port(sub, args.ip, args.monitor)
     sub = ZMQStream(sub, loop)
     
     ports = select_random_ports(random_ports)
     children = []
+    
+    # IOPub relay (in a Process)
+    if not iopub:
+        iopub = (ports.pop(),ports.pop())
+    q = ProcessMonitoredQueue(zmq.SUB, zmq.PUB, zmq.PUB, 'iopub', 'N/A')
+    q.bind_in(iface%iopub[1])
+    q.bind_out(iface%iopub[0])
+    q.setsockopt_in(zmq.SUBSCRIBE, '')
+    q.connect_mon(iface%monport)
+    q.daemon=True
+    q.start()
+    children.append(q.launcher)
+    
     # Multiplexer Queue (in a Process)
     if not mux:
         mux = (ports.pop(),ports.pop())
     q = ProcessMonitoredQueue(zmq.XREP, zmq.XREP, zmq.PUB, 'in', 'out')
     q.bind_in(iface%mux[0])
     q.bind_out(iface%mux[1])
     q.connect_mon(iface%monport)
     q.daemon=True
     q.start()
     children.append(q.launcher)
     
     # Control Queue (in a Process)
     if not control:
         control = (ports.pop(),ports.pop())
     q = ProcessMonitoredQueue(zmq.XREP, zmq.XREP, zmq.PUB, 'incontrol', 'outcontrol')
     q.bind_in(iface%control[0])
     q.bind_out(iface%control[1])
     q.connect_mon(iface%monport)
     q.daemon=True
     q.start()
     children.append(q.launcher)
     # Task Queue (in a Process)
     if not task:
         task = (ports.pop(),ports.pop())
     if args.scheduler == 'pure':
         q = ProcessMonitoredQueue(zmq.XREP, zmq.XREQ, zmq.PUB, 'intask', 'outtask')
         q.bind_in(iface%task[0])
         q.bind_out(iface%task[1])
         q.connect_mon(iface%monport)
         q.daemon=True
         q.start()
         children.append(q.launcher)
     else:
         sargs = (iface%task[0],iface%task[1],iface%monport,iface%nport,args.scheduler)
         print (sargs)
         q = Process(target=launch_scheduler, args=sargs)
         q.daemon=True
         q.start()
         children.append(q)
     
     if args.mongodb:
         from mongodb import MongoDB
         db = MongoDB(thesession.session)
     else:
         db = DictDB()
     time.sleep(.25)
     
     # build connection dicts
     engine_addrs = {
         'control' : iface%control[1],
         'queue': iface%mux[1],
         'heartbeat': (iface%hb[0], iface%hb[1]),
         'task' : iface%task[1],
+        'iopub' : iface%iopub[1],
         'monitor' : iface%monport,
         }
     
     client_addrs = {
         'control' : iface%control[0],
         'query': iface%cport,
         'queue': iface%mux[0],
         'task' : iface%task[0],
+        'iopub' : iface%iopub[0],
         'notification': iface%nport
         }
+    
+    # register relay of signals to the children
     signal_children(children)
     hub = Hub(loop, thesession, sub, reg, hmon, c, n, db, engine_addrs, client_addrs)
     dc = ioloop.DelayedCallback(lambda : print("Controller started..."), 100, loop)
     dc.start()
     loop.start()
     
     
     
 
 if __name__ == '__main__':
     main()

IPython/zmq/parallel/engine.py

@@ -1,146 +1,139 b''
 #!/usr/bin/env python
 """A simple engine that talks to a controller over 0MQ.
 it handles registration, etc. and launches a kernel
 connected to the Controller's queue(s).
 """
 from __future__ import print_function
 import sys
 import time
 import traceback
 import uuid
 from pprint import pprint
 
 import zmq
 from zmq.eventloop import ioloop, zmqstream
 
 from IPython.utils.traitlets import HasTraits
 from IPython.utils.localinterfaces import LOCALHOST 
 
 from streamsession import Message, StreamSession
 from client import Client
 from streamkernel import Kernel, make_kernel
 import heartmonitor
 from entry_point import make_base_argument_parser, connect_logger, parse_url
 # import taskthread
 # from log import logger
 
 
 def printer(*msg):
-    pprint(msg)
+    pprint(msg, stream=sys.__stdout__)
 
 class Engine(object):
     """IPython engine"""
     
     id=None
     context=None
     loop=None
     session=None
     ident=None
     registrar=None
     heart=None
     kernel=None
     user_ns=None
     
     def __init__(self, context, loop, session, registrar, client=None, ident=None,
         heart_id=None, user_ns=None):
         self.context = context
         self.loop = loop
         self.session = session
         self.registrar = registrar
         self.client = client
         self.ident = ident if ident else str(uuid.uuid4())
         self.registrar.on_send(printer)
         self.user_ns = user_ns
         
     def register(self):
         
         content = dict(queue=self.ident, heartbeat=self.ident, control=self.ident)
         self.registrar.on_recv(self.complete_registration)
         # print (self.session.key)
         self.session.send(self.registrar, "registration_request",content=content)
     
     def complete_registration(self, msg):
         # print msg
         idents,msg = self.session.feed_identities(msg)
         msg = Message(self.session.unpack_message(msg))
         if msg.content.status == 'ok':
             self.session.username = str(msg.content.id)
             queue_addr = msg.content.queue
             shell_addrs = [str(queue_addr)]
             control_addr = str(msg.content.control)
             task_addr = msg.content.task
+            iopub_addr = msg.content.iopub
             if task_addr:
                 shell_addrs.append(str(task_addr))
             
             hb_addrs = msg.content.heartbeat
             # ioloop.DelayedCallback(self.heart.start, 1000, self.loop).start()
-            
-            # placeholder for no, since pub isn't hooked up:
-            sub = self.context.socket(zmq.SUB)
-            sub = zmqstream.ZMQStream(sub, self.loop)
-            sub.on_recv(lambda *a: None)
-            port = sub.bind_to_random_port("tcp://%s"%LOCALHOST)
-            iopub_addr = "tcp://%s:%i"%(LOCALHOST,12345)
-            
             k = make_kernel(self.ident, control_addr, shell_addrs, iopub_addr,
                             hb_addrs, client_addr=None, loop=self.loop,
                             context=self.context, key=self.session.key)[-1]
             self.kernel = k
             if self.user_ns is not None:
                 self.user_ns.update(self.kernel.user_ns)
                 self.kernel.user_ns = self.user_ns
             
         else:
             # logger.error("Registration Failed: %s"%msg)
             raise Exception("Registration Failed: %s"%msg)
         
         # logger.info("engine::completed registration with id %s"%self.session.username)
         
-        print (msg)
+        print (msg,file=sys.__stdout__)
     
     def unregister(self):
         self.session.send(self.registrar, "unregistration_request", content=dict(id=int(self.session.username)))
         time.sleep(1)
         sys.exit(0)
     
     def start(self):
-        print ("registering")
+        print ("registering",file=sys.__stdout__)
         self.register()
 
         
 
 def main(argv=None, user_ns=None):
     
     parser = make_base_argument_parser()
     
     args = parser.parse_args(argv)
     
     parse_url(args)
     
     iface="%s://%s"%(args.transport,args.ip)+':%i'
     
     loop = ioloop.IOLoop.instance()
     session = StreamSession(keyfile=args.execkey)
     # print (session.key)
     ctx = zmq.Context()
 
     # setup logging
     connect_logger(ctx, iface%args.logport, root="engine", loglevel=args.loglevel)
     
     reg_conn = iface % args.regport
-    print (reg_conn)
+    print (reg_conn, file=sys.__stdout__)
     print ("Starting the engine...", file=sys.__stderr__)
     
     reg = ctx.socket(zmq.PAIR)
     reg.connect(reg_conn)
     reg = zmqstream.ZMQStream(reg, loop)
     client = None
     
     e = Engine(ctx, loop, session, reg, client, args.ident, user_ns=user_ns)
     dc = ioloop.DelayedCallback(e.start, 100, loop)
     dc.start()
     loop.start()
 
 # Execution as a script
 if __name__ == '__main__':
     main()

IPython/zmq/parallel/entry_point.py

@@ -1,116 +1,121 b''
 """ Defines helper functions for creating kernel entry points and process
 launchers.
 """
 
 # Standard library imports.
 import logging
 import atexit
 import sys
 import os
 import stat
 import socket
 from subprocess import Popen, PIPE
 from signal import signal, SIGINT, SIGABRT, SIGTERM
 try:
     from signal import SIGKILL
 except ImportError:
     SIGKILL=None
 
 # System library imports.
 import zmq
 from zmq.log import handlers
 # Local imports.
 from IPython.core.ultratb import FormattedTB
 from IPython.external.argparse import ArgumentParser
 from IPython.zmq.log import logger
 
 def split_ports(s, n):
     """Parser helper for multiport strings"""
     if not s:
         return tuple([0]*n)
     ports = map(int, s.split(','))
     if len(ports) != n:
         raise ValueError
     return ports
 
 def select_random_ports(n):
     """Selects and return n random ports that are available."""
     ports = []
     for i in xrange(n):
         sock = socket.socket()
         sock.bind(('', 0))
         ports.append(sock)
     for i, sock in enumerate(ports):
         port = sock.getsockname()[1]
         sock.close()
         ports[i] = port
     return ports
 
 def parse_url(args):
     """Ensure args.url contains full transport://interface:port"""
     if args.url:
         iface = args.url.split('://',1)
         if len(args) == 2:
             args.transport,iface = iface
         iface = iface.split(':')
         args.ip = iface[0]
         if iface[1]:
             args.regport = iface[1]
     args.url = "%s://%s:%i"%(args.transport, args.ip,args.regport)
 
 def signal_children(children):
     """Relay interupt/term signals to children, for more solid process cleanup."""
     def terminate_children(sig, frame):
         for child in children:
             child.terminate()
         # sys.exit(sig)
     for sig in (SIGINT, SIGABRT, SIGTERM):
         signal(sig, terminate_children)
 
 def generate_exec_key(keyfile):
     import uuid
     newkey = str(uuid.uuid4())
     with open(keyfile, 'w') as f:
         # f.write('ipython-key ')
         f.write(newkey)
     # set user-only RW permissions (0600)
     # this will have no effect on Windows
     os.chmod(keyfile, stat.S_IRUSR|stat.S_IWUSR)
         
 
 def make_base_argument_parser():
     """ Creates an ArgumentParser for the generic arguments supported by all 
     ipcluster entry points.
     """
     parser = ArgumentParser()
     parser.add_argument('--ip', type=str, default='127.0.0.1',
                         help='set the controller\'s IP address [default: local]')
     parser.add_argument('--transport', type=str, default='tcp',
                         help='set the transport to use [default: tcp]')
     parser.add_argument('--regport', type=int, metavar='PORT', default=10101,
                         help='set the XREP port for registration [default: 10101]')
     parser.add_argument('--logport', type=int, metavar='PORT', default=20202,
                         help='set the PUB port for logging [default: 10201]')
-    parser.add_argument('--loglevel', type=int, metavar='LEVEL', default=logging.DEBUG,
+    parser.add_argument('--loglevel', type=str, metavar='LEVEL', default=logging.DEBUG,
                         help='set the log level [default: DEBUG]')
     parser.add_argument('--ident', type=str,
                         help='set the ZMQ identity [default: random]')
     parser.add_argument('--packer', type=str, default='json',
                         choices=['json','pickle'],
                         help='set the message format method [default: json]')
     parser.add_argument('--url', type=str,
                         help='set transport,ip,regport in one arg, e.g. tcp://127.0.0.1:10101')
     parser.add_argument('--execkey', type=str,
                         help="File containing key for authenticating requests.")
 
     return parser
 
 
 def connect_logger(context, iface, root="ip", loglevel=logging.DEBUG):
+    try:
+        loglevel = int(loglevel)
+    except ValueError:
+        if isinstance(loglevel, str):
+            loglevel = getattr(logging, loglevel)
     lsock = context.socket(zmq.PUB)
     lsock.connect(iface)
     handler = handlers.PUBHandler(lsock)
     handler.setLevel(loglevel)
     handler.root_topic = root
     logger.addHandler(handler)
     
  No newline at end of file

IPython/zmq/parallel/hub.py

@@ -1,837 +1,889 b''
 #!/usr/bin/env python
 """The IPython Controller Hub with 0MQ
 This is the master object that handles connections from engines and clients,
 and monitors traffic through the various queues.
 """
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2010  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 from __future__ import print_function
 
 import sys
 from datetime import datetime
 import time
 
 import zmq
 from zmq.eventloop import ioloop
 
 # internal:
 from IPython.zmq.log import logger # a Logger object
 
 from streamsession import Message, wrap_exception, ISO8601
 
 try:
     from pymongo.binary import Binary
 except ImportError:
     MongoDB=None
 else:
     from mongodb import MongoDB
     
 #-----------------------------------------------------------------------------
 # Code
 #-----------------------------------------------------------------------------
 
 def _passer(*args, **kwargs):
     return
 
+def _printer(*args, **kwargs):
+    print (args)
+    print (kwargs)
+
 def init_record(msg):
     """return an empty TaskRecord dict, with all keys initialized with None."""
     header = msg['header']
     return {
         'msg_id' : header['msg_id'],
         'header' : header,
         'content': msg['content'],
         'buffers': msg['buffers'],
         'submitted': datetime.strptime(header['date'], ISO8601),
         'client_uuid' : None,
         'engine_uuid' : None,
         'started': None,
         'completed': None,
         'resubmitted': None,
         'result_header' : None,
         'result_content' : None,
         'result_buffers' : None,
-        'queue' : None
+        'queue' : None,
+        'pyin' : None,
+        'pyout': None,
+        'pyerr': None,
+        'stdout': '',
+        'stderr': '',
     }
 
 
 class EngineConnector(object):
     """A simple object for accessing the various zmq connections of an object.
     Attributes are:
     id (int): engine ID
     uuid (str): uuid (unused?)
     queue (str): identity of queue's XREQ socket
     registration (str): identity of registration XREQ socket
     heartbeat (str): identity of heartbeat XREQ socket
     """
     id=0
     queue=None
     control=None
     registration=None
     heartbeat=None
     pending=None
     
     def __init__(self, id, queue, registration, control, heartbeat=None):
         logger.info("engine::Engine Connected: %i"%id)
         self.id = id
         self.queue = queue
         self.registration = registration
         self.control = control
         self.heartbeat = heartbeat
         
 class Hub(object):
     """The IPython Controller Hub with 0MQ connections
     
     Parameters
     ==========
     loop: zmq IOLoop instance
     session: StreamSession object
     <removed> context: zmq context for creating new connections (?)
     queue: ZMQStream for monitoring the command queue (SUB)
     registrar: ZMQStream for engine registration requests (XREP)
     heartbeat: HeartMonitor object checking the pulse of the engines
     clientele: ZMQStream for client connections (XREP)
                 not used for jobs, only query/control commands
     notifier: ZMQStream for broadcasting engine registration changes (PUB)
     db: connection to db for out of memory logging of commands
                 NotImplemented
     engine_addrs: dict of zmq connection information for engines to connect
                 to the queues.
     client_addrs: dict of zmq connection information for engines to connect
                 to the queues.
     """
     # internal data structures:
     ids=None # engine IDs
     keytable=None
     engines=None
     clients=None
     hearts=None
     pending=None
     results=None
     tasks=None
     completed=None
     mia=None
     incoming_registrations=None
     registration_timeout=None
     
     #objects from constructor:
     loop=None
     registrar=None
     clientelle=None
     queue=None
     heartbeat=None
     notifier=None
     db=None
     client_addr=None
     engine_addrs=None
     
     
     def __init__(self, loop, session, queue, registrar, heartbeat, clientele, notifier, db, engine_addrs, client_addrs):
         """
         # universal:
         loop: IOLoop for creating future connections
         session: streamsession for sending serialized data
         # engine:
         queue: ZMQStream for monitoring queue messages
         registrar: ZMQStream for engine registration
         heartbeat: HeartMonitor object for tracking engines
         # client:
         clientele: ZMQStream for client connections
         # extra:
         db: ZMQStream for db connection (NotImplemented)
         engine_addrs: zmq address/protocol dict for engine connections
         client_addrs: zmq address/protocol dict for client connections
         """
         self.ids = set()
         self.keytable={}
         self.incoming_registrations={}
         self.engines = {}
         self.by_ident = {}
         self.clients = {}
         self.hearts = {}
         # self.mia = set()
         
         # self.sockets = {}
         self.loop = loop
         self.session = session
         self.registrar = registrar
         self.clientele = clientele
         self.queue = queue
         self.heartbeat = heartbeat
         self.notifier = notifier
         self.db = db
         
         # validate connection dicts:
         self.client_addrs = client_addrs
         assert isinstance(client_addrs['queue'], str)
         assert isinstance(client_addrs['control'], str)
         # self.hb_addrs = hb_addrs
         self.engine_addrs = engine_addrs
         assert isinstance(engine_addrs['queue'], str)
         assert isinstance(client_addrs['control'], str)
         assert len(engine_addrs['heartbeat']) == 2
         
         # register our callbacks
         self.registrar.on_recv(self.dispatch_register_request)
         self.clientele.on_recv(self.dispatch_client_msg)
-        self.queue.on_recv(self.dispatch_queue_traffic)
+        self.queue.on_recv(self.dispatch_monitor_traffic)
         
         if heartbeat is not None:
             heartbeat.add_heart_failure_handler(self.handle_heart_failure)
             heartbeat.add_new_heart_handler(self.handle_new_heart)
         
-        self.queue_handlers = { 'in' : self.save_queue_request,
+        self.monitor_handlers = { 'in' : self.save_queue_request,
                                 'out': self.save_queue_result,
                                 'intask': self.save_task_request,
                                 'outtask': self.save_task_result,
                                 'tracktask': self.save_task_destination,
                                 'incontrol': _passer,
                                 'outcontrol': _passer,
+                                'iopub': self.save_iopub_message,
         }
         
         self.client_handlers = {'queue_request': self.queue_status,
                                 'result_request': self.get_results,
                                 'purge_request': self.purge_results,
                                 'load_request': self.check_load,
                                 'resubmit_request': self.resubmit_task,
                                 'shutdown_request': self.shutdown_request,
                                 }
         
         self.registrar_handlers = {'registration_request' : self.register_engine,
                                 'unregistration_request' : self.unregister_engine,
                                 'connection_request': self.connection_request,
         }
         self.registration_timeout = max(5000, 2*self.heartbeat.period)
         # this is the stuff that will move to DB:
         # self.results = {} # completed results
         self.pending = set() # pending messages, keyed by msg_id
         self.queues = {} # pending msg_ids keyed by engine_id
         self.tasks = {} # pending msg_ids submitted as tasks, keyed by client_id
         self.completed = {} # completed msg_ids keyed by engine_id
         self.all_completed = set()
         
         logger.info("controller::created controller")
     
     def _new_id(self):
         """gemerate a new ID"""
         newid = 0
         incoming = [id[0] for id in self.incoming_registrations.itervalues()]
         # print newid, self.ids, self.incoming_registrations
         while newid in self.ids or newid in incoming:
             newid += 1
         return newid
     
     #-----------------------------------------------------------------------------
     # message validation
     #-----------------------------------------------------------------------------
     
     def _validate_targets(self, targets):
         """turn any valid targets argument into a list of integer ids"""
         if targets is None:
             # default to all
             targets = self.ids
             
         if isinstance(targets, (int,str,unicode)):
             # only one target specified
             targets = [targets]
         _targets = []
         for t in targets:
             # map raw identities to ids
             if isinstance(t, (str,unicode)):
                 t = self.by_ident.get(t, t)
             _targets.append(t)
         targets = _targets
         bad_targets = [ t for t in targets if t not in self.ids ]
         if bad_targets:
             raise IndexError("No Such Engine: %r"%bad_targets)
         if not targets:
             raise IndexError("No Engines Registered")
         return targets
     
     def _validate_client_msg(self, msg):
         """validates and unpacks headers of a message. Returns False if invalid,
         (ident, header, parent, content)"""
         client_id = msg[0]
         try:
             msg = self.session.unpack_message(msg[1:], content=True)
         except:
-            logger.error("client::Invalid Message %s"%msg)
+            logger.error("client::Invalid Message %s"%msg, exc_info=True)
             return False
         
         msg_type = msg.get('msg_type', None)
         if msg_type is None:
             return False
         header = msg.get('header')
         # session doesn't handle split content for now:
         return client_id, msg
         
     
     #-----------------------------------------------------------------------------
     # dispatch methods (1 per stream)
     #-----------------------------------------------------------------------------
     
     def dispatch_register_request(self, msg):
         """"""
         logger.debug("registration::dispatch_register_request(%s)"%msg)
         idents,msg = self.session.feed_identities(msg)
         if not idents:
             logger.error("Bad Queue Message: %s"%msg, exc_info=True)
             return
         try:
             msg = self.session.unpack_message(msg,content=True)
         except:
             logger.error("registration::got bad registration message: %s"%msg, exc_info=True)
             return
         
         msg_type = msg['msg_type']
         content = msg['content']
         
         handler = self.registrar_handlers.get(msg_type, None)
         if handler is None:
             logger.error("registration::got bad registration message: %s"%msg)
         else:
             handler(idents, msg)
     
-    def dispatch_queue_traffic(self, msg):
-        """all ME and Task queue messages come through here"""
-        logger.debug("queue traffic: %s"%msg[:2])
+    def dispatch_monitor_traffic(self, msg):
+        """all ME and Task queue messages come through here, as well as
+        IOPub traffic."""
+        logger.debug("monitor traffic: %s"%msg[:2])
         switch = msg[0]
         idents, msg = self.session.feed_identities(msg[1:])
         if not idents:
-            logger.error("Bad Queue Message: %s"%msg)
+            logger.error("Bad Monitor Message: %s"%msg)
             return
-        handler = self.queue_handlers.get(switch, None)
+        handler = self.monitor_handlers.get(switch, None)
         if handler is not None:
             handler(idents, msg)
         else:
-            logger.error("Invalid message topic: %s"%switch)
+            logger.error("Invalid monitor topic: %s"%switch)
         
     
     def dispatch_client_msg(self, msg):
         """Route messages from clients"""
         idents, msg = self.session.feed_identities(msg)
         if not idents:
             logger.error("Bad Client Message: %s"%msg)
             return
         client_id = idents[0]
         try:
             msg = self.session.unpack_message(msg, content=True)
         except:
             content = wrap_exception()
             logger.error("Bad Client Message: %s"%msg, exc_info=True)
             self.session.send(self.clientele, "controller_error", ident=client_id, 
                     content=content)
             return
         
         # print client_id, header, parent, content
         #switch on message type:
         msg_type = msg['msg_type']
         logger.info("client:: client %s requested %s"%(client_id, msg_type))
         handler = self.client_handlers.get(msg_type, None)
         try:
             assert handler is not None, "Bad Message Type: %s"%msg_type
         except:
             content = wrap_exception()
             logger.error("Bad Message Type: %s"%msg_type, exc_info=True)
             self.session.send(self.clientele, "controller_error", ident=client_id, 
                     content=content)
             return
         else:
             handler(client_id, msg)
             
     def dispatch_db(self, msg):
         """"""
         raise NotImplementedError
     
     #---------------------------------------------------------------------------
     # handler methods (1 per event)
     #---------------------------------------------------------------------------
     
     #----------------------- Heartbeat --------------------------------------
     
     def handle_new_heart(self, heart):
         """handler to attach to heartbeater.
         Called when a new heart starts to beat.
         Triggers completion of registration."""
         logger.debug("heartbeat::handle_new_heart(%r)"%heart)
         if heart not in self.incoming_registrations:
             logger.info("heartbeat::ignoring new heart: %r"%heart)
         else:
             self.finish_registration(heart)
         
     
     def handle_heart_failure(self, heart):
         """handler to attach to heartbeater.
         called when a previously registered heart fails to respond to beat request.
         triggers unregistration"""
         logger.debug("heartbeat::handle_heart_failure(%r)"%heart)
         eid = self.hearts.get(heart, None)
         queue = self.engines[eid].queue
         if eid is None:
             logger.info("heartbeat::ignoring heart failure %r"%heart)
         else:
             self.unregister_engine(heart, dict(content=dict(id=eid, queue=queue)))
     
     #----------------------- MUX Queue Traffic ------------------------------
     
     def save_queue_request(self, idents, msg):
         if len(idents) < 2:
             logger.error("invalid identity prefix: %s"%idents)
             return
         queue_id, client_id = idents[:2]
         try:
             msg = self.session.unpack_message(msg, content=False)
         except:
             logger.error("queue::client %r sent invalid message to %r: %s"%(client_id, queue_id, msg), exc_info=True)
             return
         
         eid = self.by_ident.get(queue_id, None)
         if eid is None:
             logger.error("queue::target %r not registered"%queue_id)
             logger.debug("queue::    valid are: %s"%(self.by_ident.keys()))
             return
             
         header = msg['header']
         msg_id = header['msg_id']
         record = init_record(msg)
         record['engine_uuid'] = queue_id
         record['client_uuid'] = client_id
         record['queue'] = 'mux'
         if MongoDB is not None and isinstance(self.db, MongoDB):
             record['buffers'] = map(Binary, record['buffers'])
         self.pending.add(msg_id)
         self.queues[eid].append(msg_id)
         self.db.add_record(msg_id, record)
     
     def save_queue_result(self, idents, msg):
         if len(idents) < 2:
             logger.error("invalid identity prefix: %s"%idents)
             return
             
         client_id, queue_id = idents[:2]
         try:
             msg = self.session.unpack_message(msg, content=False)
         except:
             logger.error("queue::engine %r sent invalid message to %r: %s"%(
                     queue_id,client_id, msg), exc_info=True)
             return
         
         eid = self.by_ident.get(queue_id, None)
         if eid is None:
             logger.error("queue::unknown engine %r is sending a reply: "%queue_id)
             logger.debug("queue::       %s"%msg[2:])
             return
         
         parent = msg['parent_header']
         if not parent:
             return
         msg_id = parent['msg_id']
         if msg_id in self.pending:
             self.pending.remove(msg_id)
             self.all_completed.add(msg_id)
             self.queues[eid].remove(msg_id)
             self.completed[eid].append(msg_id)
             rheader = msg['header']
             completed = datetime.strptime(rheader['date'], ISO8601)
             started = rheader.get('started', None)
             if started is not None:
                 started = datetime.strptime(started, ISO8601)
             result = {
                 'result_header' : rheader,
                 'result_content': msg['content'],
                 'started' : started,
                 'completed' : completed
             }
             if MongoDB is not None and isinstance(self.db, MongoDB):
                 result['result_buffers'] = map(Binary, msg['buffers'])
             else:
                 result['result_buffers'] = msg['buffers']
             self.db.update_record(msg_id, result)
         else:
             logger.debug("queue:: unknown msg finished %s"%msg_id)
             
     #--------------------- Task Queue Traffic ------------------------------
     
     def save_task_request(self, idents, msg):
         """Save the submission of a task."""
         client_id = idents[0]
         
         try:
             msg = self.session.unpack_message(msg, content=False)
         except:
             logger.error("task::client %r sent invalid task message: %s"%(
                     client_id, msg), exc_info=True)
             return
         record = init_record(msg)
         if MongoDB is not None and isinstance(self.db, MongoDB):
             record['buffers'] = map(Binary, record['buffers'])
         record['client_uuid'] = client_id
         record['queue'] = 'task'
         header = msg['header']
         msg_id = header['msg_id']
         self.pending.add(msg_id)
         self.db.add_record(msg_id, record)
     
     def save_task_result(self, idents, msg):
         """save the result of a completed task."""
         client_id = idents[0]
         try:
             msg = self.session.unpack_message(msg, content=False)
         except:
             logger.error("task::invalid task result message send to %r: %s"%(
-                    client_id, msg))
+                    client_id, msg), exc_info=True)
             raise
             return
         
         parent = msg['parent_header']
         if not parent:
             # print msg
             logger.warn("Task %r had no parent!"%msg)
             return
         msg_id = parent['msg_id']
         
         header = msg['header']
         engine_uuid = header.get('engine', None)
         eid = self.by_ident.get(engine_uuid, None)
         
         if msg_id in self.pending:
             self.pending.remove(msg_id)
             self.all_completed.add(msg_id)
             if eid is not None:
                 self.completed[eid].append(msg_id)
                 if msg_id in self.tasks[eid]:
                     self.tasks[eid].remove(msg_id)
             completed = datetime.strptime(header['date'], ISO8601)
             started = header.get('started', None)
             if started is not None:
                 started = datetime.strptime(started, ISO8601)
             result = {
                 'result_header' : header,
                 'result_content': msg['content'],
                 'started' : started,
                 'completed' : completed,
                 'engine_uuid': engine_uuid
             }
             if MongoDB is not None and isinstance(self.db, MongoDB):
                 result['result_buffers'] = map(Binary, msg['buffers'])
             else:
                 result['result_buffers'] = msg['buffers']
             self.db.update_record(msg_id, result)
             
         else:
             logger.debug("task::unknown task %s finished"%msg_id)
     
     def save_task_destination(self, idents, msg):
         try:
             msg = self.session.unpack_message(msg, content=True)
         except:
-            logger.error("task::invalid task tracking message")
+            logger.error("task::invalid task tracking message", exc_info=True)
             return
         content = msg['content']
         print (content)
         msg_id = content['msg_id']
         engine_uuid = content['engine_id']
         eid = self.by_ident[engine_uuid]
         
         logger.info("task::task %s arrived on %s"%(msg_id, eid))
         # if msg_id in self.mia:
         #     self.mia.remove(msg_id)
         # else:
         #     logger.debug("task::task %s not listed as MIA?!"%(msg_id))
         
         self.tasks[eid].append(msg_id)
         # self.pending[msg_id][1].update(received=datetime.now(),engine=(eid,engine_uuid))
         self.db.update_record(msg_id, dict(engine_uuid=engine_uuid))
     
     def mia_task_request(self, idents, msg):
         raise NotImplementedError
         client_id = idents[0]
         # content = dict(mia=self.mia,status='ok')
         # self.session.send('mia_reply', content=content, idents=client_id)
         
         
+    #--------------------- IOPub Traffic ------------------------------
+    
+    def save_iopub_message(self, topics, msg):
+        """save an iopub message into the db"""
+        print (topics)
+        try:
+            msg = self.session.unpack_message(msg, content=True)
+        except:
+            logger.error("iopub::invalid IOPub message", exc_info=True)
+            return
+        
+        parent = msg['parent_header']
+        msg_id = parent['msg_id']
+        msg_type = msg['msg_type']
+        content = msg['content']
+        
+        # ensure msg_id is in db
+        try:
+            rec = self.db.get_record(msg_id)
+        except:
+            logger.error("iopub::IOPub message has invalid parent", exc_info=True)
+            return
+        # stream
+        d = {}
+        if msg_type == 'stream':
+            name = content['name']
+            s = rec[name] or ''
+            d[name] = s + content['data']
+            
+        elif msg_type == 'pyerr':
+            d['pyerr'] = content
+        else:
+            d[msg_type] = content['data']
+        
+        self.db.update_record(msg_id, d)
+        
+    
             
     #-------------------------------------------------------------------------
     # Registration requests
     #-------------------------------------------------------------------------
         
     def connection_request(self, client_id, msg):
         """Reply with connection addresses for clients."""
         logger.info("client::client %s connected"%client_id)
         content = dict(status='ok')
         content.update(self.client_addrs)
         jsonable = {}
         for k,v in self.keytable.iteritems():
             jsonable[str(k)] = v
         content['engines'] = jsonable
         self.session.send(self.registrar, 'connection_reply', content, parent=msg, ident=client_id)
     
     def register_engine(self, reg, msg):
         """Register a new engine."""
         content = msg['content']
         try:
             queue = content['queue']
         except KeyError:
-            logger.error("registration::queue not specified")
+            logger.error("registration::queue not specified", exc_info=True)
             return
         heart = content.get('heartbeat', None)
         """register a new engine, and create the socket(s) necessary"""
         eid = self._new_id()
         # print (eid, queue, reg, heart)
         
         logger.debug("registration::register_engine(%i, %r, %r, %r)"%(eid, queue, reg, heart))
         
         content = dict(id=eid,status='ok')
         content.update(self.engine_addrs)
         # check if requesting available IDs:
         if queue in self.by_ident:
             try:
                 raise KeyError("queue_id %r in use"%queue)
             except:
                 content = wrap_exception()
         elif heart in self.hearts: # need to check unique hearts?
             try:
                 raise KeyError("heart_id %r in use"%heart)
             except:
                 content = wrap_exception()
         else:
             for h, pack in self.incoming_registrations.iteritems():
                 if heart == h:
                     try:
                         raise KeyError("heart_id %r in use"%heart)
                     except:
                         content = wrap_exception()
                     break
                 elif queue == pack[1]:
                     try:
                         raise KeyError("queue_id %r in use"%queue)
                     except:
                         content = wrap_exception()
                     break
         
         msg = self.session.send(self.registrar, "registration_reply", 
                 content=content, 
                 ident=reg)
         
         if content['status'] == 'ok':
             if heart in self.heartbeat.hearts:
                 # already beating
                 self.incoming_registrations[heart] = (eid,queue,reg,None)
                 self.finish_registration(heart)
             else:
                 purge = lambda : self._purge_stalled_registration(heart)
                 dc = ioloop.DelayedCallback(purge, self.registration_timeout, self.loop)
                 dc.start()
                 self.incoming_registrations[heart] = (eid,queue,reg,dc)
         else:
             logger.error("registration::registration %i failed: %s"%(eid, content['evalue']))
         return eid
     
     def unregister_engine(self, ident, msg):
         """Unregister an engine that explicitly requested to leave."""
         try:
             eid = msg['content']['id']
         except:
-            logger.error("registration::bad engine id for unregistration: %s"%ident)
+            logger.error("registration::bad engine id for unregistration: %s"%ident, exc_info=True)
             return
         logger.info("registration::unregister_engine(%s)"%eid)
         content=dict(id=eid, queue=self.engines[eid].queue)
         self.ids.remove(eid)
         self.keytable.pop(eid)
         ec = self.engines.pop(eid)
         self.hearts.pop(ec.heartbeat)
         self.by_ident.pop(ec.queue)
         self.completed.pop(eid)
         for msg_id in self.queues.pop(eid):
             msg = self.pending.remove(msg_id)
             ############## TODO: HANDLE IT ################
         
         if self.notifier:
             self.session.send(self.notifier, "unregistration_notification", content=content)
     
     def finish_registration(self, heart):
         """Second half of engine registration, called after our HeartMonitor
         has received a beat from the Engine's Heart."""
         try: 
             (eid,queue,reg,purge) = self.incoming_registrations.pop(heart)
         except KeyError:
-            logger.error("registration::tried to finish nonexistant registration")
+            logger.error("registration::tried to finish nonexistant registration", exc_info=True)
             return
         logger.info("registration::finished registering engine %i:%r"%(eid,queue))
         if purge is not None:
             purge.stop()
         control = queue
         self.ids.add(eid)
         self.keytable[eid] = queue
         self.engines[eid] = EngineConnector(eid, queue, reg, control, heart)
         self.by_ident[queue] = eid
         self.queues[eid] = list()
         self.tasks[eid] = list()
         self.completed[eid] = list()
         self.hearts[heart] = eid
         content = dict(id=eid, queue=self.engines[eid].queue)
         if self.notifier:
             self.session.send(self.notifier, "registration_notification", content=content)
     
     def _purge_stalled_registration(self, heart):
         if heart in self.incoming_registrations:
             eid = self.incoming_registrations.pop(heart)[0]
             logger.info("registration::purging stalled registration: %i"%eid)
         else:
             pass
             
     #-------------------------------------------------------------------------
     # Client Requests
     #-------------------------------------------------------------------------
     
     def shutdown_request(self, client_id, msg):
         """handle shutdown request."""
         # s = self.context.socket(zmq.XREQ)
         # s.connect(self.client_connections['mux'])
         # time.sleep(0.1)
         # for eid,ec in self.engines.iteritems():
         #     self.session.send(s, 'shutdown_request', content=dict(restart=False), ident=ec.queue)
         # time.sleep(1)
         self.session.send(self.clientele, 'shutdown_reply', content={'status': 'ok'}, ident=client_id)
         dc = ioloop.DelayedCallback(lambda : self._shutdown(), 1000, self.loop)
         dc.start()
     
     def _shutdown(self):
         logger.info("controller::controller shutting down.")
         time.sleep(0.1)
         sys.exit(0)
         
     
     def check_load(self, client_id, msg):
         content = msg['content']
         try:
             targets = content['targets']
             targets = self._validate_targets(targets)
         except:
             content = wrap_exception()
             self.session.send(self.clientele, "controller_error", 
                     content=content, ident=client_id)
             return
         
         content = dict(status='ok')
         # loads = {}
         for t in targets:
             content[bytes(t)] = len(self.queues[t])+len(self.tasks[t])
         self.session.send(self.clientele, "load_reply", content=content, ident=client_id)
             
     
     def queue_status(self, client_id, msg):
         """Return the Queue status of one or more targets.
         if verbose: return the msg_ids
         else: return len of each type.
         keys: queue (pending MUX jobs)
             tasks (pending Task jobs)
             completed (finished jobs from both queues)"""
         content = msg['content']
         targets = content['targets']
         try:
             targets = self._validate_targets(targets)
         except:
             content = wrap_exception()
             self.session.send(self.clientele, "controller_error", 
                     content=content, ident=client_id)
             return
         verbose = content.get('verbose', False)
         content = dict(status='ok')
         for t in targets:
             queue = self.queues[t]
             completed = self.completed[t]
             tasks = self.tasks[t]
             if not verbose:
                 queue = len(queue)
                 completed = len(completed)
                 tasks = len(tasks)
             content[bytes(t)] = {'queue': queue, 'completed': completed , 'tasks': tasks}
             # pending
         self.session.send(self.clientele, "queue_reply", content=content, ident=client_id)
     
     def purge_results(self, client_id, msg):
         """Purge results from memory. This method is more valuable before we move
         to a DB based message storage mechanism."""
         content = msg['content']
         msg_ids = content.get('msg_ids', [])
         reply = dict(status='ok')
         if msg_ids == 'all':
             self.db.drop_matching_records(dict(completed={'$ne':None}))
         else:
             for msg_id in msg_ids:
                 if msg_id in self.all_completed:
                     self.db.drop_record(msg_id)
                 else:
                     if msg_id in self.pending:
                         try:
                             raise IndexError("msg pending: %r"%msg_id)
                         except:
                             reply = wrap_exception()
                     else:
                         try:
                             raise IndexError("No such msg: %r"%msg_id)
                         except:
                             reply = wrap_exception()
                     break
             eids = content.get('engine_ids', [])
             for eid in eids:
                 if eid not in self.engines:
                     try:
                         raise IndexError("No such engine: %i"%eid)
                     except:
                         reply = wrap_exception()
                     break
                 msg_ids = self.completed.pop(eid)
                 uid = self.engines[eid].queue
                 self.db.drop_matching_records(dict(engine_uuid=uid, completed={'$ne':None}))
         
         self.session.send(self.clientele, 'purge_reply', content=reply, ident=client_id)
     
     def resubmit_task(self, client_id, msg, buffers):
         """Resubmit a task."""
         raise NotImplementedError
     
     def get_results(self, client_id, msg):
         """Get the result of 1 or more messages."""
         content = msg['content']
         msg_ids = sorted(set(content['msg_ids']))
         statusonly = content.get('status_only', False)
         pending = []
         completed = []
         content = dict(status='ok')
         content['pending'] = pending
         content['completed'] = completed
         buffers = []
         if not statusonly:
             content['results'] = {}
             records = self.db.find_records(dict(msg_id={'$in':msg_ids}))
         for msg_id in msg_ids:
             if msg_id in self.pending:
                 pending.append(msg_id)
             elif msg_id in self.all_completed:
                 completed.append(msg_id)
                 if not statusonly:
                     rec = records[msg_id]
+                    io_dict = {}
+                    for key in 'pyin pyout pyerr stdout stderr'.split():
+                            io_dict[key] = rec[key]
                     content[msg_id] = { 'result_content': rec['result_content'],
                                         'header': rec['header'],
                                         'result_header' : rec['result_header'],
+                                        'io' : io_dict,
                                       }
                     buffers.extend(map(str, rec['result_buffers']))
             else:
                 try:
                     raise KeyError('No such message: '+msg_id)
                 except:
                     content = wrap_exception()
                 break
         self.session.send(self.clientele, "result_reply", content=content, 
                                             parent=msg, ident=client_id,
                                             buffers=buffers)
 

IPython/zmq/parallel/scheduler.py

@@ -1,424 +1,424 b''
 """The Python scheduler for rich scheduling.
 
 The Pure ZMQ scheduler does not allow routing schemes other than LRU,
 nor does it check msg_id DAG dependencies. For those, a slightly slower
 Python Scheduler exists.
 """
 
 #----------------------------------------------------------------------
 # Imports
 #----------------------------------------------------------------------
 
 from __future__ import print_function
 from random import randint,random
 
 try:
     import numpy
 except ImportError:
     numpy = None
 
 import zmq
 from zmq.eventloop import ioloop, zmqstream
 
 # local imports
 from IPython.zmq.log import logger # a Logger object
 from client import Client
 from dependency import Dependency
 import streamsession as ss
 
 from IPython.external.decorator import decorator
 
 @decorator
 def logged(f,self,*args,**kwargs):
-    print ("#--------------------")
-    print ("%s(*%s,**%s)"%(f.func_name, args, kwargs))
-    print ("#--")
+    # print ("#--------------------")
+    # print ("%s(*%s,**%s)"%(f.func_name, args, kwargs))
+    # print ("#--")
     return f(self,*args, **kwargs)
 
 #----------------------------------------------------------------------
 # Chooser functions
 #----------------------------------------------------------------------
 
 def plainrandom(loads):
     """Plain random pick."""
     n = len(loads)
     return randint(0,n-1)
 
 def lru(loads):
     """Always pick the front of the line.
     
     The content of `loads` is ignored.
     
     Assumes LRU ordering of loads, with oldest first.
     """
     return 0
 
 def twobin(loads):
     """Pick two at random, use the LRU of the two.
     
     The content of loads is ignored.
     
     Assumes LRU ordering of loads, with oldest first.
     """
     n = len(loads)
     a = randint(0,n-1)
     b = randint(0,n-1)
     return min(a,b)
 
 def weighted(loads):
     """Pick two at random using inverse load as weight.
     
     Return the less loaded of the two.
     """
     # weight 0 a million times more than 1:
     weights = 1./(1e-6+numpy.array(loads))
     sums = weights.cumsum()
     t = sums[-1]
     x = random()*t
     y = random()*t
     idx = 0
     idy = 0
     while sums[idx] < x:
         idx += 1
     while sums[idy] < y:
         idy += 1
     if weights[idy] > weights[idx]:
         return idy
     else:
         return idx
 
 def leastload(loads):
     """Always choose the lowest load.
     
     If the lowest load occurs more than once, the first
     occurance will be used.  If loads has LRU ordering, this means
     the LRU of those with the lowest load is chosen.
     """
     return loads.index(min(loads))
 
 #---------------------------------------------------------------------
 # Classes
 #---------------------------------------------------------------------
 class TaskScheduler(object):
     """Python TaskScheduler object.
     
     This is the simplest object that supports msg_id based
     DAG dependencies. *Only* task msg_ids are checked, not
     msg_ids of jobs submitted via the MUX queue.
     
     """
     
     scheme = leastload # function for determining the destination
     client_stream = None # client-facing stream
     engine_stream = None # engine-facing stream
     mon_stream = None # controller-facing stream
     dependencies = None # dict by msg_id of [ msg_ids that depend on key ]
     depending = None # dict by msg_id of (msg_id, raw_msg, after, follow)
     pending = None # dict by engine_uuid of submitted tasks
     completed = None # dict by engine_uuid of completed tasks
     clients = None # dict by msg_id for who submitted the task
     targets = None # list of target IDENTs
     loads = None # list of engine loads
     all_done = None # set of all completed tasks
     blacklist = None # dict by msg_id of locations where a job has encountered UnmetDependency
     
     
     def __init__(self, client_stream, engine_stream, mon_stream, 
                 notifier_stream, scheme=None, io_loop=None):
         if io_loop is None:
             io_loop = ioloop.IOLoop.instance()
         self.io_loop = io_loop
         self.client_stream = client_stream
         self.engine_stream = engine_stream
         self.mon_stream = mon_stream
         self.notifier_stream = notifier_stream
         
         if scheme is not None:
             self.scheme = scheme
         else:
             self.scheme = TaskScheduler.scheme
         
         self.session = ss.StreamSession(username="TaskScheduler")
         
         self.dependencies = {}
         self.depending = {}
         self.completed = {}
         self.pending = {}
         self.all_done = set()
         self.blacklist = {}
         
         self.targets = []
         self.loads = []
         
         engine_stream.on_recv(self.dispatch_result, copy=False)
         self._notification_handlers = dict(
             registration_notification = self._register_engine,
             unregistration_notification = self._unregister_engine
         )
         self.notifier_stream.on_recv(self.dispatch_notification)
     
     def resume_receiving(self):
         """Resume accepting jobs."""
         self.client_stream.on_recv(self.dispatch_submission, copy=False)
     
     def stop_receiving(self):
         """Stop accepting jobs while there are no engines.
         Leave them in the ZMQ queue."""
         self.client_stream.on_recv(None)
     
     #-----------------------------------------------------------------------
     # [Un]Registration Handling
     #-----------------------------------------------------------------------
     
     def dispatch_notification(self, msg):
         """dispatch register/unregister events."""
         idents,msg = self.session.feed_identities(msg)
         msg = self.session.unpack_message(msg)
         msg_type = msg['msg_type']
         handler = self._notification_handlers.get(msg_type, None)
         if handler is None:
             raise Exception("Unhandled message type: %s"%msg_type)
         else:
             try:
                 handler(str(msg['content']['queue']))
             except KeyError:
                 logger.error("task::Invalid notification msg: %s"%msg)
     @logged
     def _register_engine(self, uid):
         """New engine with ident `uid` became available."""
         # head of the line:
         self.targets.insert(0,uid)
         self.loads.insert(0,0)
         # initialize sets
         self.completed[uid] = set()
         self.pending[uid] = {}
         if len(self.targets) == 1:
             self.resume_receiving()
 
     def _unregister_engine(self, uid):
         """Existing engine with ident `uid` became unavailable."""
         if len(self.targets) == 1:
             # this was our only engine
             self.stop_receiving()
         
         # handle any potentially finished tasks:
         self.engine_stream.flush()
         
         self.completed.pop(uid)
         lost = self.pending.pop(uid)
         
         idx = self.targets.index(uid)
         self.targets.pop(idx)
         self.loads.pop(idx)
         
         self.handle_stranded_tasks(lost)
     
     def handle_stranded_tasks(self, lost):
         """Deal with jobs resident in an engine that died."""
         # TODO: resubmit the tasks?
         for msg_id in lost:
             pass
     
     
     #-----------------------------------------------------------------------
     # Job Submission
     #-----------------------------------------------------------------------
     @logged
     def dispatch_submission(self, raw_msg):
         """Dispatch job submission to appropriate handlers."""
         # ensure targets up to date:
         self.notifier_stream.flush()
         try:
             idents, msg = self.session.feed_identities(raw_msg, copy=False)
         except Exception as e:
             logger.error("task::Invaid msg: %s"%msg)
             return
         
         # send to monitor
         self.mon_stream.send_multipart(['intask']+raw_msg, copy=False)
         
         msg = self.session.unpack_message(msg, content=False, copy=False)
         header = msg['header']
         msg_id = header['msg_id']
         
         # time dependencies
         after = Dependency(header.get('after', []))
         if after.mode == 'all':
             after.difference_update(self.all_done)
         if after.check(self.all_done):
             # recast as empty set, if `after` already met,
             # to prevent unnecessary set comparisons
             after = Dependency([])
         
         # location dependencies
         follow = Dependency(header.get('follow', []))
         if len(after) == 0:
             # time deps already met, try to run
             if not self.maybe_run(msg_id, raw_msg, follow):
                 # can't run yet
                 self.save_unmet(msg_id, raw_msg, after, follow)
         else:
             self.save_unmet(msg_id, raw_msg, after, follow)
     
     @logged
     def maybe_run(self, msg_id, raw_msg, follow=None):
         """check location dependencies, and run if they are met."""
             
         if follow:
             def can_run(idx):
                 target = self.targets[idx]
                 return target not in self.blacklist.get(msg_id, []) and\
                         follow.check(self.completed[target])
             
             indices = filter(can_run, range(len(self.targets)))
             if not indices:
                 return False
         else:
             indices = None
             
         self.submit_task(msg_id, raw_msg, indices)
         return True
             
     @logged
     def save_unmet(self, msg_id, msg, after, follow):
         """Save a message for later submission when its dependencies are met."""
         self.depending[msg_id] = (msg_id,msg,after,follow)
         # track the ids in both follow/after, but not those already completed
         for dep_id in after.union(follow).difference(self.all_done):
             if dep_id not in self.dependencies:
                 self.dependencies[dep_id] = set()
             self.dependencies[dep_id].add(msg_id)
     
     @logged
     def submit_task(self, msg_id, msg, follow=None, indices=None):
         """Submit a task to any of a subset of our targets."""
         if indices:
             loads = [self.loads[i] for i in indices]
         else:
             loads = self.loads
         idx = self.scheme(loads)
         if indices:
             idx = indices[idx]
         target = self.targets[idx]
         # print (target, map(str, msg[:3]))
         self.engine_stream.send(target, flags=zmq.SNDMORE, copy=False)
         self.engine_stream.send_multipart(msg, copy=False)
         self.add_job(idx)
         self.pending[target][msg_id] = (msg, follow)
         content = dict(msg_id=msg_id, engine_id=target)
         self.session.send(self.mon_stream, 'task_destination', content=content, ident='tracktask')
     
     #-----------------------------------------------------------------------
     # Result Handling
     #-----------------------------------------------------------------------
     @logged
     def dispatch_result(self, raw_msg):
         try:
             idents,msg = self.session.feed_identities(raw_msg, copy=False)
         except Exception as e:
             logger.error("task::Invaid result: %s"%msg)
             return
         msg = self.session.unpack_message(msg, content=False, copy=False)
         header = msg['header']
         if header.get('dependencies_met', True):
             self.handle_result_success(idents, msg['parent_header'], raw_msg)
             # send to monitor
             self.mon_stream.send_multipart(['outtask']+raw_msg, copy=False)
         else:
             self.handle_unmet_dependency(idents, msg['parent_header'])
         
     @logged
     def handle_result_success(self, idents, parent, raw_msg):
         # first, relay result to client
         engine = idents[0]
         client = idents[1]
         # swap_ids for XREP-XREP mirror
         raw_msg[:2] = [client,engine]
         # print (map(str, raw_msg[:4]))
         self.client_stream.send_multipart(raw_msg, copy=False)
         # now, update our data structures
         msg_id = parent['msg_id']
         self.pending[engine].pop(msg_id)
         self.completed[engine].add(msg_id)
         self.all_done.add(msg_id)
         
         self.update_dependencies(msg_id)
         
     @logged
     def handle_unmet_dependency(self, idents, parent):
         engine = idents[0]
         msg_id = parent['msg_id']
         if msg_id not in self.blacklist:
             self.blacklist[msg_id] = set()
         self.blacklist[msg_id].add(engine)
         raw_msg,follow = self.pending[engine].pop(msg_id)
         if not self.maybe_run(msg_id, raw_msg, follow):
             # resubmit failed, put it back in our dependency tree
             self.save_unmet(msg_id, raw_msg, Dependency(), follow)
         pass
     @logged
     def update_dependencies(self, dep_id):
         """dep_id just finished. Update our dependency
         table and submit any jobs that just became runable."""
         
         if dep_id not in self.dependencies:
             return
         jobs = self.dependencies.pop(dep_id)
         for job in jobs:
             msg_id, raw_msg, after, follow = self.depending[job]
             if dep_id in after:
                 after.remove(dep_id)
             if not after: # time deps met, maybe run
                 if self.maybe_run(msg_id, raw_msg, follow):
                     self.depending.pop(job)
                     for mid in follow:
                         if mid in self.dependencies:
                             self.dependencies[mid].remove(msg_id)
     
     #----------------------------------------------------------------------
     # methods to be overridden by subclasses
     #----------------------------------------------------------------------
     
     def add_job(self, idx):
         """Called after self.targets[idx] just got the job with header.
         Override with subclasses.  The default ordering is simple LRU.
         The default loads are the number of outstanding jobs."""
         self.loads[idx] += 1
         for lis in (self.targets, self.loads):
             lis.append(lis.pop(idx))
             
     
     def finish_job(self, idx):
         """Called after self.targets[idx] just finished a job.
         Override with subclasses."""
         self.loads[idx] -= 1
     
 
 
 def launch_scheduler(in_addr, out_addr, mon_addr, not_addr, scheme='weighted'):
     from zmq.eventloop import ioloop
     from zmq.eventloop.zmqstream import ZMQStream
     
     ctx = zmq.Context()
     loop = ioloop.IOLoop()
     
     scheme = globals().get(scheme)
     
     ins = ZMQStream(ctx.socket(zmq.XREP),loop)
     ins.bind(in_addr)
     outs = ZMQStream(ctx.socket(zmq.XREP),loop)
     outs.bind(out_addr)
     mons = ZMQStream(ctx.socket(zmq.PUB),loop)
     mons.connect(mon_addr)
     nots = ZMQStream(ctx.socket(zmq.SUB),loop)
     nots.setsockopt(zmq.SUBSCRIBE, '')
     nots.connect(not_addr)
     
     scheduler = TaskScheduler(ins,outs,mons,nots,scheme,loop)
     
     loop.start()
 
 
 if __name__ == '__main__':
     iface = 'tcp://127.0.0.1:%i'
     launch_scheduler(iface%12345,iface%1236,iface%12347,iface%12348)

IPython/zmq/parallel/streamkernel.py

@@ -1,435 +1,459 b''
 #!/usr/bin/env python
 """
 Kernel adapted from kernel.py to use ZMQ Streams
 """
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 # Standard library imports.
 from __future__ import print_function
 import __builtin__
 from code import CommandCompiler
 import os
 import sys
 import time
 import traceback
 from datetime import datetime
 from signal import SIGTERM, SIGKILL
 from pprint import pprint
 
 # System library imports.
 import zmq
 from zmq.eventloop import ioloop, zmqstream
 
 # Local imports.
 from IPython.core import ultratb
 from IPython.utils.traitlets import HasTraits, Instance, List
 from IPython.zmq.completer import KernelCompleter
 from IPython.zmq.log import logger # a Logger object
+from IPython.zmq.iostream import OutStream
+from IPython.zmq.displayhook import DisplayHook
+
 
 from streamsession import StreamSession, Message, extract_header, serialize_object,\
                 unpack_apply_message, ISO8601, wrap_exception
 from dependency import UnmetDependency
 import heartmonitor
 from client import Client
 
 def printer(*args):
-    pprint(args)
+    pprint(args, stream=sys.__stdout__)
 
 #-----------------------------------------------------------------------------
 # Main kernel class
 #-----------------------------------------------------------------------------
 
 class Kernel(HasTraits):
 
     #---------------------------------------------------------------------------
     # Kernel interface
     #---------------------------------------------------------------------------
 
     session = Instance(StreamSession)
     shell_streams = Instance(list)
     control_stream = Instance(zmqstream.ZMQStream)
     task_stream = Instance(zmqstream.ZMQStream)
     iopub_stream = Instance(zmqstream.ZMQStream)
     client = Instance(Client)
     loop = Instance(ioloop.IOLoop)
     
     def __init__(self, **kwargs):
         super(Kernel, self).__init__(**kwargs)
         self.identity = self.shell_streams[0].getsockopt(zmq.IDENTITY)
+        self.prefix = 'engine.%s'%self.identity
         self.user_ns = {}
         self.history = []
         self.compiler = CommandCompiler()
         self.completer = KernelCompleter(self.user_ns)
         self.aborted = set()
         
         # Build dict of handlers for message types
         self.shell_handlers = {}
         self.control_handlers = {}
         for msg_type in ['execute_request', 'complete_request', 'apply_request', 
                 'clear_request']:
             self.shell_handlers[msg_type] = getattr(self, msg_type)
         
         for msg_type in ['shutdown_request', 'abort_request']+self.shell_handlers.keys():
             self.control_handlers[msg_type] = getattr(self, msg_type)
     
     
     def _wrap_exception(self, method=None):
         e_info = dict(engineid=self.identity, method=method)
         content=wrap_exception(e_info)
         return content
     
     #-------------------- control handlers -----------------------------
     def abort_queues(self):
         for stream in self.shell_streams:
             if stream:
                 self.abort_queue(stream)
     
     def abort_queue(self, stream):
         while True:
             try:
                 msg = self.session.recv(stream, zmq.NOBLOCK,content=True)
             except zmq.ZMQError as e:
                 if e.errno == zmq.EAGAIN:
                     break
                 else:
                     return
             else:
                 if msg is None:
                     return
                 else:
                     idents,msg = msg
                 
                 # assert self.reply_socketly_socket.rcvmore(), "Unexpected missing message part."
                 # msg = self.reply_socket.recv_json()
             print ("Aborting:", file=sys.__stdout__)
             print (Message(msg), file=sys.__stdout__)
             msg_type = msg['msg_type']
             reply_type = msg_type.split('_')[0] + '_reply'
             # reply_msg = self.session.msg(reply_type, {'status' : 'aborted'}, msg)
             # self.reply_socket.send(ident,zmq.SNDMORE)
             # self.reply_socket.send_json(reply_msg)
             reply_msg = self.session.send(stream, reply_type, 
                         content={'status' : 'aborted'}, parent=msg, ident=idents)[0]
             print(Message(reply_msg), file=sys.__stdout__)
             # We need to wait a bit for requests to come in. This can probably
             # be set shorter for true asynchronous clients.
             time.sleep(0.05)
     
     def abort_request(self, stream, ident, parent):
         """abort a specifig msg by id"""
         msg_ids = parent['content'].get('msg_ids', None)
         if isinstance(msg_ids, basestring):
             msg_ids = [msg_ids]
         if not msg_ids:
             self.abort_queues()
         for mid in msg_ids:
             self.aborted.add(str(mid))
         
         content = dict(status='ok')
         reply_msg = self.session.send(stream, 'abort_reply', content=content, 
                 parent=parent, ident=ident)[0]
         print(Message(reply_msg), file=sys.__stdout__)
     
     def shutdown_request(self, stream, ident, parent):
         """kill ourself.  This should really be handled in an external process"""
         try:
             self.abort_queues()
         except:
             content = self._wrap_exception('shutdown')
         else:
             content = dict(parent['content'])
             content['status'] = 'ok'
         msg = self.session.send(stream, 'shutdown_reply',
                                 content=content, parent=parent, ident=ident)
         # msg = self.session.send(self.pub_socket, 'shutdown_reply',
         #                         content, parent, ident)
         # print >> sys.__stdout__, msg
         # time.sleep(0.2)
         dc = ioloop.DelayedCallback(lambda : sys.exit(0), 1000, self.loop)
         dc.start()
     
     def dispatch_control(self, msg):
         idents,msg = self.session.feed_identities(msg, copy=False)
         try:
             msg = self.session.unpack_message(msg, content=True, copy=False)
         except:
             logger.error("Invalid Message", exc_info=True)
             return
         
         header = msg['header']
         msg_id = header['msg_id']
         
         handler = self.control_handlers.get(msg['msg_type'], None)
         if handler is None:
             print ("UNKNOWN CONTROL MESSAGE TYPE:", msg, file=sys.__stderr__)
         else:
             handler(self.control_stream, idents, msg)
     
 
     #-------------------- queue helpers ------------------------------
     
     def check_dependencies(self, dependencies):
         if not dependencies:
             return True
         if len(dependencies) == 2 and dependencies[0] in 'any all'.split():
             anyorall = dependencies[0]
             dependencies = dependencies[1]
         else:
             anyorall = 'all'
         results = self.client.get_results(dependencies,status_only=True)
         if results['status'] != 'ok':
             return False
         
         if anyorall == 'any':
             if not results['completed']:
                 return False
         else:
             if results['pending']:
                 return False
         
         return True
     
     def check_aborted(self, msg_id):
         return msg_id in self.aborted
     
     #-------------------- queue handlers -----------------------------
     
     def clear_request(self, stream, idents, parent):
         """Clear our namespace."""
         self.user_ns = {}
         msg = self.session.send(stream, 'clear_reply', ident=idents, parent=parent, 
                 content = dict(status='ok'))
     
     def execute_request(self, stream, ident, parent):
         try:
             code = parent[u'content'][u'code']
         except:
             print("Got bad msg: ", file=sys.__stderr__)
             print(Message(parent), file=sys.__stderr__)
             return
         # pyin_msg = self.session.msg(u'pyin',{u'code':code}, parent=parent)
         # self.iopub_stream.send(pyin_msg)
-        self.session.send(self.iopub_stream, u'pyin', {u'code':code},parent=parent)
+        self.session.send(self.iopub_stream, u'pyin', {u'code':code},parent=parent,
+                            ident=self.identity+'.pyin')
         started = datetime.now().strftime(ISO8601)
         try:
             comp_code = self.compiler(code, '<zmq-kernel>')
             # allow for not overriding displayhook
             if hasattr(sys.displayhook, 'set_parent'):
                 sys.displayhook.set_parent(parent)
+                sys.stdout.set_parent(parent)
+                sys.stderr.set_parent(parent)
             exec comp_code in self.user_ns, self.user_ns
         except:
             exc_content = self._wrap_exception('execute')
             # exc_msg = self.session.msg(u'pyerr', exc_content, parent)
-            self.session.send(self.iopub_stream, u'pyerr', exc_content, parent=parent)
+            self.session.send(self.iopub_stream, u'pyerr', exc_content, parent=parent,
+                            ident=self.identity+'.pyerr')
             reply_content = exc_content
         else:
             reply_content = {'status' : 'ok'}
         # reply_msg = self.session.msg(u'execute_reply', reply_content, parent)
         # self.reply_socket.send(ident, zmq.SNDMORE)
         # self.reply_socket.send_json(reply_msg)
         reply_msg = self.session.send(stream, u'execute_reply', reply_content, parent=parent, 
                     ident=ident, subheader = dict(started=started))
         print(Message(reply_msg), file=sys.__stdout__)
         if reply_msg['content']['status'] == u'error':
             self.abort_queues()
 
     def complete_request(self, stream, ident, parent):
         matches = {'matches' : self.complete(parent),
                    'status' : 'ok'}
         completion_msg = self.session.send(stream, 'complete_reply',
                                            matches, parent, ident)
         # print >> sys.__stdout__, completion_msg
 
     def complete(self, msg):
         return self.completer.complete(msg.content.line, msg.content.text)
     
     def apply_request(self, stream, ident, parent):
-        print (parent)
+        # print (parent)
         try:
             content = parent[u'content']
             bufs = parent[u'buffers']
             msg_id = parent['header']['msg_id']
             bound = content.get('bound', False)
         except:
             print("Got bad msg: ", file=sys.__stderr__)
             print(Message(parent), file=sys.__stderr__)
             return
         # pyin_msg = self.session.msg(u'pyin',{u'code':code}, parent=parent)
         # self.iopub_stream.send(pyin_msg)
         # self.session.send(self.iopub_stream, u'pyin', {u'code':code},parent=parent)
         sub = {'dependencies_met' : True, 'engine' : self.identity,
                 'started': datetime.now().strftime(ISO8601)}
         try:
             # allow for not overriding displayhook
             if hasattr(sys.displayhook, 'set_parent'):
                 sys.displayhook.set_parent(parent)
+                sys.stdout.set_parent(parent)
+                sys.stderr.set_parent(parent)
             # exec "f(*args,**kwargs)" in self.user_ns, self.user_ns
             if bound:
                 working = self.user_ns
                 suffix = str(msg_id).replace("-","")
                 prefix = "_"
                 
             else:
                 working = dict()
                 suffix = prefix = "_" # prevent keyword collisions with lambda
             f,args,kwargs = unpack_apply_message(bufs, working, copy=False)
             # if f.fun
             if hasattr(f, 'func_name'):
                 fname = f.func_name
             else:
                 fname = f.__name__
             
             fname = prefix+fname.strip('<>')+suffix
             argname = prefix+"args"+suffix
             kwargname = prefix+"kwargs"+suffix
             resultname = prefix+"result"+suffix
             
             ns = { fname : f, argname : args, kwargname : kwargs }
             # print ns
             working.update(ns)
             code = "%s=%s(*%s,**%s)"%(resultname, fname, argname, kwargname)
             exec code in working, working
             result = working.get(resultname)
             # clear the namespace
             if bound:
                 for key in ns.iterkeys():
                     self.user_ns.pop(key)
             else:
                 del working
             
             packed_result,buf = serialize_object(result)
             result_buf = [packed_result]+buf
         except:
             exc_content = self._wrap_exception('apply')
             # exc_msg = self.session.msg(u'pyerr', exc_content, parent)
-            self.session.send(self.iopub_stream, u'pyerr', exc_content, parent=parent)
+            self.session.send(self.iopub_stream, u'pyerr', exc_content, parent=parent,
+                                ident=self.identity+'.pyerr')
             reply_content = exc_content
             result_buf = []
             
             if exc_content['ename'] == UnmetDependency.__name__:
                 sub['dependencies_met'] = False
         else:
             reply_content = {'status' : 'ok'}
         # reply_msg = self.session.msg(u'execute_reply', reply_content, parent)
         # self.reply_socket.send(ident, zmq.SNDMORE)
         # self.reply_socket.send_json(reply_msg)
         reply_msg = self.session.send(stream, u'apply_reply', reply_content, 
                     parent=parent, ident=ident,buffers=result_buf, subheader=sub)
-        print(Message(reply_msg), file=sys.__stdout__)
+        # print(Message(reply_msg), file=sys.__stdout__)
         # if reply_msg['content']['status'] == u'error':
         #     self.abort_queues()
     
     def dispatch_queue(self, stream, msg):
         self.control_stream.flush()
         idents,msg = self.session.feed_identities(msg, copy=False)
         try:
             msg = self.session.unpack_message(msg, content=True, copy=False)
         except:
             logger.error("Invalid Message", exc_info=True)
             return
             
         
         header = msg['header']
         msg_id = header['msg_id']
         if self.check_aborted(msg_id):
             self.aborted.remove(msg_id)
             # is it safe to assume a msg_id will not be resubmitted?
             reply_type = msg['msg_type'].split('_')[0] + '_reply'
             reply_msg = self.session.send(stream, reply_type, 
                         content={'status' : 'aborted'}, parent=msg, ident=idents)
             return
         handler = self.shell_handlers.get(msg['msg_type'], None)
         if handler is None:
             print ("UNKNOWN MESSAGE TYPE:", msg, file=sys.__stderr__)
         else:
             handler(stream, idents, msg)
     
     def start(self):
         #### stream mode:
         if self.control_stream:
             self.control_stream.on_recv(self.dispatch_control, copy=False)
             self.control_stream.on_err(printer)
         
         def make_dispatcher(stream):
             def dispatcher(msg):
                 return self.dispatch_queue(stream, msg)
             return dispatcher
         
         for s in self.shell_streams:
             s.on_recv(make_dispatcher(s), copy=False)
             s.on_err(printer)
         
         if self.iopub_stream:
             self.iopub_stream.on_err(printer)
-            self.iopub_stream.on_send(printer)
+            # self.iopub_stream.on_send(printer)
         
         #### while True mode:
         # while True:
         #     idle = True
         #     try:
         #         msg = self.shell_stream.socket.recv_multipart(
         #                     zmq.NOBLOCK, copy=False)
         #     except zmq.ZMQError, e:
         #         if e.errno != zmq.EAGAIN:
         #             raise e
         #     else:
         #         idle=False
         #         self.dispatch_queue(self.shell_stream, msg)
         #             
         #     if not self.task_stream.empty():
         #         idle=False
         #         msg = self.task_stream.recv_multipart()
         #         self.dispatch_queue(self.task_stream, msg)
         #     if idle:
         #         # don't busywait
         #         time.sleep(1e-3)
 
 def make_kernel(identity, control_addr, shell_addrs, iopub_addr, hb_addrs, 
-                client_addr=None, loop=None, context=None, key=None):
+                client_addr=None, loop=None, context=None, key=None,
+                out_stream_factory=OutStream, display_hook_factory=DisplayHook):
+    
     # create loop, context, and session:
     if loop is None:
         loop = ioloop.IOLoop.instance()
     if context is None:
         context = zmq.Context()
     c = context
     session = StreamSession(key=key)
     # print (session.key)
     print (control_addr, shell_addrs, iopub_addr, hb_addrs)
     
     # create Control Stream
     control_stream = zmqstream.ZMQStream(c.socket(zmq.PAIR), loop)
     control_stream.setsockopt(zmq.IDENTITY, identity)
     control_stream.connect(control_addr)
     
     # create Shell Streams (MUX, Task, etc.):
     shell_streams = []
     for addr in shell_addrs:
         stream = zmqstream.ZMQStream(c.socket(zmq.PAIR), loop)
         stream.setsockopt(zmq.IDENTITY, identity)
         stream.connect(addr)
         shell_streams.append(stream)
     
     # create iopub stream:
     iopub_stream = zmqstream.ZMQStream(c.socket(zmq.PUB), loop)
     iopub_stream.setsockopt(zmq.IDENTITY, identity)
     iopub_stream.connect(iopub_addr)
     
+    # Redirect input streams and set a display hook.
+    if out_stream_factory:
+        sys.stdout = out_stream_factory(session, iopub_stream, u'stdout')
+        sys.stdout.topic = identity+'.stdout'
+        sys.stderr = out_stream_factory(session, iopub_stream, u'stderr')
+        sys.stderr.topic = identity+'.stderr'
+    if display_hook_factory:
+        sys.displayhook = display_hook_factory(session, iopub_stream)
+        sys.displayhook.topic = identity+'.pyout'
+    
+    
     # launch heartbeat
     heart = heartmonitor.Heart(*map(str, hb_addrs), heart_id=identity)
     heart.start()
     
     # create (optional) Client
     if client_addr:
         client = Client(client_addr, username=identity)
     else:
         client = None
     
     kernel = Kernel(session=session, control_stream=control_stream, 
             shell_streams=shell_streams, iopub_stream=iopub_stream, 
             client=client, loop=loop)
     kernel.start()
     return loop, c, kernel
 

IPython/zmq/parallel/streamsession.py

@@ -1,544 +1,543 b''
 #!/usr/bin/env python
 """edited session.py to work with streams, and move msg_type to the header
 """
 
 
 import os
 import sys
 import traceback
 import pprint
 import uuid
 from datetime import datetime
 
 import zmq
 from zmq.utils import jsonapi
 from zmq.eventloop.zmqstream import ZMQStream
 
 from IPython.utils.pickleutil import can, uncan, canSequence, uncanSequence
 from IPython.utils.newserialized import serialize, unserialize
 
 from IPython.zmq.parallel.error import RemoteError
 
 try:
     import cPickle
     pickle = cPickle
 except:
     cPickle = None
     import pickle
 
 # packer priority: jsonlib[2], cPickle, simplejson/json, pickle
 json_name = '' if not jsonapi.jsonmod else jsonapi.jsonmod.__name__
 if json_name in ('jsonlib', 'jsonlib2'):
     use_json = True
 elif json_name:
     if cPickle is None:
         use_json = True
     else:
         use_json = False
 else:
     use_json = False
 
 def squash_unicode(obj):
     if isinstance(obj,dict):
         for key in obj.keys():
             obj[key] = squash_unicode(obj[key])
             if isinstance(key, unicode):
                 obj[squash_unicode(key)] = obj.pop(key)
     elif isinstance(obj, list):
         for i,v in enumerate(obj):
             obj[i] = squash_unicode(v)
     elif isinstance(obj, unicode):
         obj = obj.encode('utf8')
     return obj
 
 if use_json:
     default_packer = jsonapi.dumps
     default_unpacker = lambda s: squash_unicode(jsonapi.loads(s))
 else:
     default_packer = lambda o: pickle.dumps(o,-1)
     default_unpacker = pickle.loads
 
 
 DELIM="<IDS|MSG>"
 ISO8601="%Y-%m-%dT%H:%M:%S.%f"
 
 def wrap_exception(engine_info={}):
     etype, evalue, tb = sys.exc_info()
     stb = traceback.format_exception(etype, evalue, tb)
     exc_content = {
         'status' : 'error',
         'traceback' : stb,
         'ename' : unicode(etype.__name__),
         'evalue' : unicode(evalue),
         'engine_info' : engine_info
     }
     return exc_content
 
 def unwrap_exception(content):
     err = RemoteError(content['ename'], content['evalue'], 
                 ''.join(content['traceback']),
                 content.get('engine_info', {}))
     return err
     
 
 class Message(object):
     """A simple message object that maps dict keys to attributes.
 
     A Message can be created from a dict and a dict from a Message instance
     simply by calling dict(msg_obj)."""
     
     def __init__(self, msg_dict):
         dct = self.__dict__
         for k, v in dict(msg_dict).iteritems():
             if isinstance(v, dict):
                 v = Message(v)
             dct[k] = v
 
     # Having this iterator lets dict(msg_obj) work out of the box.
     def __iter__(self):
         return iter(self.__dict__.iteritems())
     
     def __repr__(self):
         return repr(self.__dict__)
 
     def __str__(self):
         return pprint.pformat(self.__dict__)
 
     def __contains__(self, k):
         return k in self.__dict__
 
     def __getitem__(self, k):
         return self.__dict__[k]
 
 
 def msg_header(msg_id, msg_type, username, session):
     date=datetime.now().strftime(ISO8601)
     return locals()
 
 def extract_header(msg_or_header):
     """Given a message or header, return the header."""
     if not msg_or_header:
         return {}
     try:
         # See if msg_or_header is the entire message.
         h = msg_or_header['header']
     except KeyError:
         try:
             # See if msg_or_header is just the header
             h = msg_or_header['msg_id']
         except KeyError:
             raise
         else:
             h = msg_or_header
     if not isinstance(h, dict):
         h = dict(h)
     return h
 
 def rekey(dikt):
     """Rekey a dict that has been forced to use str keys where there should be
     ints by json.  This belongs in the jsonutil added by fperez."""
     for k in dikt.iterkeys():
         if isinstance(k, str):
             ik=fk=None
             try:
                 ik = int(k)
             except ValueError:
                 try:
                     fk = float(k)
                 except ValueError:
                     continue
             if ik is not None:
                 nk = ik
             else:
                 nk = fk
             if nk in dikt:
                 raise KeyError("already have key %r"%nk)
             dikt[nk] = dikt.pop(k)
     return dikt
 
 def serialize_object(obj, threshold=64e-6):
     """Serialize an object into a list of sendable buffers.
     
     Parameters
     ----------
     
     obj : object
         The object to be serialized
     threshold : float
         The threshold for not double-pickling the content.
         
     
     Returns
     -------
     ('pmd', [bufs]) :
         where pmd is the pickled metadata wrapper,
         bufs is a list of data buffers
     """
     databuffers = []
     if isinstance(obj, (list, tuple)):
         clist = canSequence(obj)
         slist = map(serialize, clist)
         for s in slist:
             if s.typeDescriptor in ('buffer', 'ndarray') or s.getDataSize() > threshold:
                 databuffers.append(s.getData())
                 s.data = None
         return pickle.dumps(slist,-1), databuffers
     elif isinstance(obj, dict):
         sobj = {}
         for k in sorted(obj.iterkeys()):
             s = serialize(can(obj[k]))
             if s.typeDescriptor in ('buffer', 'ndarray') or s.getDataSize() > threshold:
                 databuffers.append(s.getData())
                 s.data = None
             sobj[k] = s
         return pickle.dumps(sobj,-1),databuffers
     else:
         s = serialize(can(obj))
         if s.typeDescriptor in ('buffer', 'ndarray') or s.getDataSize() > threshold:
             databuffers.append(s.getData())
             s.data = None
         return pickle.dumps(s,-1),databuffers
             
         
 def unserialize_object(bufs):
     """reconstruct an object serialized by serialize_object from data buffers."""
     bufs = list(bufs)
     sobj = pickle.loads(bufs.pop(0))
     if isinstance(sobj, (list, tuple)):
         for s in sobj:
             if s.data is None:
                 s.data = bufs.pop(0)
         return uncanSequence(map(unserialize, sobj)), bufs
     elif isinstance(sobj, dict):
         newobj = {}
         for k in sorted(sobj.iterkeys()):
             s = sobj[k]
             if s.data is None:
                 s.data = bufs.pop(0)
             newobj[k] = uncan(unserialize(s))
         return newobj, bufs
     else:
         if sobj.data is None:
             sobj.data = bufs.pop(0)
         return uncan(unserialize(sobj)), bufs
 
 def pack_apply_message(f, args, kwargs, threshold=64e-6):
     """pack up a function, args, and kwargs to be sent over the wire
     as a series of buffers. Any object whose data is larger than `threshold`
     will not have their data copied (currently only numpy arrays support zero-copy)"""
     msg = [pickle.dumps(can(f),-1)]
     databuffers = [] # for large objects
     sargs, bufs = serialize_object(args,threshold)
     msg.append(sargs)
     databuffers.extend(bufs)
     skwargs, bufs = serialize_object(kwargs,threshold)
     msg.append(skwargs)
     databuffers.extend(bufs)
     msg.extend(databuffers)
     return msg
 
 def unpack_apply_message(bufs, g=None, copy=True):
     """unpack f,args,kwargs from buffers packed by pack_apply_message()
     Returns: original f,args,kwargs"""
     bufs = list(bufs) # allow us to pop
     assert len(bufs) >= 3, "not enough buffers!"
     if not copy:
         for i in range(3):
             bufs[i] = bufs[i].bytes
     cf = pickle.loads(bufs.pop(0))
     sargs = list(pickle.loads(bufs.pop(0)))
     skwargs = dict(pickle.loads(bufs.pop(0)))
     # print sargs, skwargs
     f = uncan(cf, g)
     for sa in sargs:
         if sa.data is None:
             m = bufs.pop(0)
             if sa.getTypeDescriptor() in ('buffer', 'ndarray'):
                 if copy:
                     sa.data = buffer(m)
                 else:
                     sa.data = m.buffer
             else:
                 if copy:
                     sa.data = m
                 else:
                     sa.data = m.bytes
     
     args = uncanSequence(map(unserialize, sargs), g)
     kwargs = {}
     for k in sorted(skwargs.iterkeys()):
         sa = skwargs[k]
         if sa.data is None:
             sa.data = bufs.pop(0)
         kwargs[k] = uncan(unserialize(sa), g)
     
     return f,args,kwargs
 
 class StreamSession(object):
     """tweaked version of IPython.zmq.session.Session, for development in Parallel"""
     debug=False
     key=None
     
     def __init__(self, username=None, session=None, packer=None, unpacker=None, key=None, keyfile=None):
         if username is None:
             username = os.environ.get('USER','username')
         self.username = username
         if session is None:
             self.session = str(uuid.uuid4())
         else:
             self.session = session
         self.msg_id = str(uuid.uuid4())
         if packer is None:
             self.pack = default_packer
         else:
             if not callable(packer):
                 raise TypeError("packer must be callable, not %s"%type(packer))
             self.pack = packer
         
         if unpacker is None:
             self.unpack = default_unpacker
         else:
             if not callable(unpacker):
                 raise TypeError("unpacker must be callable, not %s"%type(unpacker))
             self.unpack = unpacker
         
         if key is not None and keyfile is not None:
             raise TypeError("Must specify key OR keyfile, not both")
         if keyfile is not None:
             with open(keyfile) as f:
                 self.key = f.read().strip()
         else:
             self.key = key
         # print key, keyfile, self.key
         self.none = self.pack({})
             
     def msg_header(self, msg_type):
         h = msg_header(self.msg_id, msg_type, self.username, self.session)
         self.msg_id = str(uuid.uuid4())
         return h
 
     def msg(self, msg_type, content=None, parent=None, subheader=None):
         msg = {}
         msg['header'] = self.msg_header(msg_type)
         msg['msg_id'] = msg['header']['msg_id']
         msg['parent_header'] = {} if parent is None else extract_header(parent)
         msg['msg_type'] = msg_type
         msg['content'] = {} if content is None else content
         sub = {} if subheader is None else subheader
         msg['header'].update(sub)
         return msg
 
     def check_key(self, msg_or_header):
         """Check that a message's header has the right key"""
         if self.key is None:
             return True
         header = extract_header(msg_or_header)
         return header.get('key', None) == self.key
             
         
-    def send(self, stream, msg_type, content=None, buffers=None, parent=None, subheader=None, ident=None):
+    def send(self, stream, msg_or_type, content=None, buffers=None, parent=None, subheader=None, ident=None):
         """Build and send a message via stream or socket.
         
         Parameters
         ----------
         
         stream : zmq.Socket or ZMQStream
             the socket-like object used to send the data
-        msg_type : str or Message/dict
-            Normally, msg_type will be 
-            
-            
+        msg_or_type : str or Message/dict
+            Normally, msg_or_type will be a msg_type unless a message is being sent more
+            than once.
         
         Returns
         -------
         (msg,sent) : tuple
             msg : Message
                 the nice wrapped dict-like object containing the headers
             
         """
-        if isinstance(msg_type, (Message, dict)):
+        if isinstance(msg_or_type, (Message, dict)):
             # we got a Message, not a msg_type
             # don't build a new Message
-            msg = msg_type
+            msg = msg_or_type
             content = msg['content']
         else:
-            msg = self.msg(msg_type, content, parent, subheader)
+            msg = self.msg(msg_or_type, content, parent, subheader)
         buffers = [] if buffers is None else buffers
         to_send = []
         if isinstance(ident, list):
             # accept list of idents
             to_send.extend(ident)
         elif ident is not None:
             to_send.append(ident)
         to_send.append(DELIM)
         if self.key is not None:
             to_send.append(self.key)
         to_send.append(self.pack(msg['header']))
         to_send.append(self.pack(msg['parent_header']))
         
         if content is None:
             content = self.none
         elif isinstance(content, dict):
             content = self.pack(content)
         elif isinstance(content, str):
             # content is already packed, as in a relayed message
             pass
         else:
             raise TypeError("Content incorrect type: %s"%type(content))
         to_send.append(content)
         flag = 0
         if buffers:
             flag = zmq.SNDMORE
         stream.send_multipart(to_send, flag, copy=False)
         for b in buffers[:-1]:
             stream.send(b, flag, copy=False)
         if buffers:
             stream.send(buffers[-1], copy=False)
         omsg = Message(msg)
         if self.debug:
             pprint.pprint(omsg)
             pprint.pprint(to_send)
             pprint.pprint(buffers)
         return omsg
     
     def send_raw(self, stream, msg, flags=0, copy=True, ident=None):
         """Send a raw message via ident path.
         
         Parameters
         ----------
         msg : list of sendable buffers"""
         to_send = []
         if isinstance(ident, str):
             ident = [ident]
         if ident is not None:
             to_send.extend(ident)
         to_send.append(DELIM)
         if self.key is not None:
             to_send.append(self.key)
         to_send.extend(msg)
         stream.send_multipart(msg, flags, copy=copy)
     
     def recv(self, socket, mode=zmq.NOBLOCK, content=True, copy=True):
         """receives and unpacks a message
         returns [idents], msg"""
         if isinstance(socket, ZMQStream):
             socket = socket.socket
         try:
             msg = socket.recv_multipart(mode)
         except zmq.ZMQError as e:
             if e.errno == zmq.EAGAIN:
                 # We can convert EAGAIN to None as we know in this case
                 # recv_json won't return None.
                 return None
             else:
                 raise
         # return an actual Message object
         # determine the number of idents by trying to unpack them.
         # this is terrible:
         idents, msg = self.feed_identities(msg, copy)
         try:
             return idents, self.unpack_message(msg, content=content, copy=copy)
         except Exception as e:
             print (idents, msg)
             # TODO: handle it
             raise e
     
     def feed_identities(self, msg, copy=True):
         """feed until DELIM is reached, then return the prefix as idents and remainder as
         msg. This is easily broken by setting an IDENT to DELIM, but that would be silly.
         
         Parameters
         ----------
         msg : a list of Message or bytes objects
             the message to be split
         copy : bool
             flag determining whether the arguments are bytes or Messages
         
         Returns
         -------
         (idents,msg) : two lists
             idents will always be a list of bytes - the indentity prefix
             msg will be a list of bytes or Messages, unchanged from input
             msg should be unpackable via self.unpack_message at this point.
         """
         msg = list(msg)
         idents = []
         while len(msg) > 3:
             if copy:
                 s = msg[0]
             else:
                 s = msg[0].bytes
             if s == DELIM:
                 msg.pop(0)
                 break
             else:
                 idents.append(s)
                 msg.pop(0)
                 
         return idents, msg
     
     def unpack_message(self, msg, content=True, copy=True):
         """Return a message object from the format
         sent by self.send.
         
         Parameters:
         -----------
         
         content : bool (True)
             whether to unpack the content dict (True), 
             or leave it serialized (False)
         
         copy : bool (True)
             whether to return the bytes (True), 
             or the non-copying Message object in each place (False)
         
         """
         ikey = int(self.key is not None)
         minlen = 3 + ikey
         if not len(msg) >= minlen:
             raise TypeError("malformed message, must have at least %i elements"%minlen)
         message = {}
         if not copy:
             for i in range(minlen):
                 msg[i] = msg[i].bytes
         if ikey:
             if not self.key == msg[0]:
                 raise KeyError("Invalid Session Key: %s"%msg[0])
         message['header'] = self.unpack(msg[ikey+0])
         message['msg_type'] = message['header']['msg_type']
         message['parent_header'] = self.unpack(msg[ikey+1])
         if content:
             message['content'] = self.unpack(msg[ikey+2])
         else:
             message['content'] = msg[ikey+2]
     
         # message['buffers'] = msg[3:]
         # else:
         #     message['header'] = self.unpack(msg[0].bytes)
         #     message['msg_type'] = message['header']['msg_type']
         #     message['parent_header'] = self.unpack(msg[1].bytes)
         #     if content:
         #         message['content'] = self.unpack(msg[2].bytes)
         #     else:
         #         message['content'] = msg[2].bytes
         
         message['buffers'] = msg[ikey+3:]# [ m.buffer for m in msg[3:] ]
         return message
             
         
 
 def test_msg2obj():
     am = dict(x=1)
     ao = Message(am)
     assert ao.x == am['x']
 
     am['y'] = dict(z=1)
     ao = Message(am)
     assert ao.y.z == am['y']['z']
     
     k1, k2 = 'y', 'z'
     assert ao[k1][k2] == am[k1][k2]
     
     am2 = dict(ao)
     assert am['x'] == am2['x']
     assert am['y']['z'] == am2['y']['z']