Merge pull request #1768 from minrk/parallelmagics

Update parallel magics

They now display all output, so you can do parallel plotting or other actions with complex display.  The `px` magic has now both line and cell modes, and in cell mode finer control has been added about how to collate output from multiple engines.

Tests, docs and example notebook added.

IPython/parallel/tests/test_magics.py

@@ -0,0 +1,342 b''
+# -*- coding: utf-8 -*-
+"""Test Parallel magics
+
+Authors:
+
+* Min RK
+"""
+#-------------------------------------------------------------------------------
+#  Copyright (C) 2011  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-------------------------------------------------------------------------------
+
+#-------------------------------------------------------------------------------
+# Imports
+#-------------------------------------------------------------------------------
+
+import sys
+import time
+
+import zmq
+from nose import SkipTest
+
+from IPython.testing import decorators as dec
+from IPython.testing.ipunittest import ParametricTestCase
+
+from IPython import parallel  as pmod
+from IPython.parallel import error
+from IPython.parallel import AsyncResult
+from IPython.parallel.util import interactive
+
+from IPython.parallel.tests import add_engines
+
+from .clienttest import ClusterTestCase, capture_output, generate_output
+
+def setup():
+    add_engines(3, total=True)
+
+class TestParallelMagics(ClusterTestCase, ParametricTestCase):
+    
+    def test_px_blocking(self):
+        ip = get_ipython()
+        v = self.client[-1:]
+        v.activate()
+        v.block=True
+
+        ip.magic('px a=5')
+        self.assertEquals(v['a'], [5])
+        ip.magic('px a=10')
+        self.assertEquals(v['a'], [10])
+        # just 'print a' works ~99% of the time, but this ensures that
+        # the stdout message has arrived when the result is finished:
+        with capture_output() as io:
+            ip.magic(
+                'px import sys,time;print(a);sys.stdout.flush();time.sleep(0.2)'
+            )
+        out = io.stdout
+        self.assertTrue('[stdout:' in out, out)
+        self.assertTrue(out.rstrip().endswith('10'))
+        self.assertRaisesRemote(ZeroDivisionError, ip.magic, 'px 1/0')
+    
+    def test_cellpx_block_args(self):
+        """%%px --[no]block flags work"""
+        ip = get_ipython()
+        v = self.client[-1:]
+        v.activate()
+        v.block=False
+        
+        for block in (True, False):
+            v.block = block
+            
+            with capture_output() as io:
+                ip.run_cell_magic("px", "", "1")
+            if block:
+                self.assertTrue(io.stdout.startswith("Parallel"), io.stdout)
+            else:
+                self.assertTrue(io.stdout.startswith("Async"), io.stdout)
+            
+            with capture_output() as io:
+                ip.run_cell_magic("px", "--block", "1")
+            self.assertTrue(io.stdout.startswith("Parallel"), io.stdout)
+
+            with capture_output() as io:
+                ip.run_cell_magic("px", "--noblock", "1")
+            self.assertTrue(io.stdout.startswith("Async"), io.stdout)
+    
+    def test_cellpx_groupby_engine(self):
+        """%%px --group-outputs=engine"""
+        ip = get_ipython()
+        v = self.client[:]
+        v.block = True
+        v.activate()
+        
+        v['generate_output'] = generate_output
+        
+        with capture_output() as io:
+            ip.run_cell_magic('px', '--group-outputs=engine', 'generate_output()')
+        
+        lines = io.stdout.strip().splitlines()[1:]
+        expected = [
+            ('[stdout:', '] stdout'),
+            'stdout2',
+            'IPython.core.display.HTML',
+            'IPython.core.display.Math',
+            ('] Out[', 'IPython.core.display.Math')
+            ] * len(v)
+        
+        self.assertEquals(len(lines), len(expected), io.stdout)
+        for line,expect in zip(lines, expected):
+            if isinstance(expect, str):
+                expect = [expect]
+            for ex in expect:
+                self.assertTrue(ex in line, "Expected %r in %r" % (ex, line))
+        
+        expected = [
+            ('[stderr:', '] stderr'),
+            'stderr2',
+        ] * len(v)
+        
+        lines = io.stderr.strip().splitlines()
+        self.assertEquals(len(lines), len(expected), io.stderr)
+        for line,expect in zip(lines, expected):
+            if isinstance(expect, str):
+                expect = [expect]
+            for ex in expect:
+                self.assertTrue(ex in line, "Expected %r in %r" % (ex, line))
+
+
+    def test_cellpx_groupby_order(self):
+        """%%px --group-outputs=order"""
+        ip = get_ipython()
+        v = self.client[:]
+        v.block = True
+        v.activate()
+        
+        v['generate_output'] = generate_output
+        
+        with capture_output() as io:
+            ip.run_cell_magic('px', '--group-outputs=order', 'generate_output()')
+        
+        lines = io.stdout.strip().splitlines()[1:]
+        expected = []
+        expected.extend([
+            ('[stdout:', '] stdout'),
+            'stdout2',
+        ] * len(v))
+        expected.extend([
+            'IPython.core.display.HTML',
+        ] * len(v))
+        expected.extend([
+            'IPython.core.display.Math',
+        ] * len(v))
+        expected.extend([
+            ('] Out[', 'IPython.core.display.Math')
+        ] * len(v))
+        
+        self.assertEquals(len(lines), len(expected), io.stdout)
+        for line,expect in zip(lines, expected):
+            if isinstance(expect, str):
+                expect = [expect]
+            for ex in expect:
+                self.assertTrue(ex in line, "Expected %r in %r" % (ex, line))
+        
+        expected = [
+            ('[stderr:', '] stderr'),
+            'stderr2',
+        ] * len(v)
+        
+        lines = io.stderr.strip().splitlines()
+        self.assertEquals(len(lines), len(expected), io.stderr)
+        for line,expect in zip(lines, expected):
+            if isinstance(expect, str):
+                expect = [expect]
+            for ex in expect:
+                self.assertTrue(ex in line, "Expected %r in %r" % (ex, line))
+
+    def test_cellpx_groupby_atype(self):
+        """%%px --group-outputs=type"""
+        ip = get_ipython()
+        v = self.client[:]
+        v.block = True
+        v.activate()
+        
+        v['generate_output'] = generate_output
+        
+        with capture_output() as io:
+            ip.run_cell_magic('px', '--group-outputs=type', 'generate_output()')
+        
+        lines = io.stdout.strip().splitlines()[1:]
+        
+        expected = []
+        expected.extend([
+            ('[stdout:', '] stdout'),
+            'stdout2',
+        ] * len(v))
+        expected.extend([
+            'IPython.core.display.HTML',
+            'IPython.core.display.Math',
+        ] * len(v))
+        expected.extend([
+            ('] Out[', 'IPython.core.display.Math')
+        ] * len(v))
+        
+        self.assertEquals(len(lines), len(expected), io.stdout)
+        for line,expect in zip(lines, expected):
+            if isinstance(expect, str):
+                expect = [expect]
+            for ex in expect:
+                self.assertTrue(ex in line, "Expected %r in %r" % (ex, line))
+        
+        expected = [
+            ('[stderr:', '] stderr'),
+            'stderr2',
+        ] * len(v)
+        
+        lines = io.stderr.strip().splitlines()
+        self.assertEquals(len(lines), len(expected), io.stderr)
+        for line,expect in zip(lines, expected):
+            if isinstance(expect, str):
+                expect = [expect]
+            for ex in expect:
+                self.assertTrue(ex in line, "Expected %r in %r" % (ex, line))
+
+
+    def test_px_nonblocking(self):
+        ip = get_ipython()
+        v = self.client[-1:]
+        v.activate()
+        v.block=False
+
+        ip.magic('px a=5')
+        self.assertEquals(v['a'], [5])
+        ip.magic('px a=10')
+        self.assertEquals(v['a'], [10])
+        with capture_output() as io:
+            ar = ip.magic('px print (a)')
+        self.assertTrue(isinstance(ar, AsyncResult))
+        self.assertTrue('Async' in io.stdout)
+        self.assertFalse('[stdout:' in io.stdout)
+        ar = ip.magic('px 1/0')
+        self.assertRaisesRemote(ZeroDivisionError, ar.get)
+    
+    def test_autopx_blocking(self):
+        ip = get_ipython()
+        v = self.client[-1]
+        v.activate()
+        v.block=True
+
+        with capture_output() as io:
+            ip.magic('autopx')
+            ip.run_cell('\n'.join(('a=5','b=12345','c=0')))
+            ip.run_cell('b*=2')
+            ip.run_cell('print (b)')
+            ip.run_cell('b')
+            ip.run_cell("b/c")
+            ip.magic('autopx')
+        
+        output = io.stdout.strip()
+        
+        self.assertTrue(output.startswith('%autopx enabled'), output)
+        self.assertTrue(output.endswith('%autopx disabled'), output)
+        self.assertTrue('RemoteError: ZeroDivisionError' in output, output)
+        self.assertTrue('] Out[' in output, output)
+        self.assertTrue(': 24690' in output, output)
+        ar = v.get_result(-1)
+        self.assertEquals(v['a'], 5)
+        self.assertEquals(v['b'], 24690)
+        self.assertRaisesRemote(ZeroDivisionError, ar.get)
+
+    def test_autopx_nonblocking(self):
+        ip = get_ipython()
+        v = self.client[-1]
+        v.activate()
+        v.block=False
+
+        with capture_output() as io:
+            ip.magic('autopx')
+            ip.run_cell('\n'.join(('a=5','b=10','c=0')))
+            ip.run_cell('print (b)')
+            ip.run_cell('import time; time.sleep(0.1)')
+            ip.run_cell("b/c")
+            ip.run_cell('b*=2')
+            ip.magic('autopx')
+        
+        output = io.stdout.strip()
+        
+        self.assertTrue(output.startswith('%autopx enabled'))
+        self.assertTrue(output.endswith('%autopx disabled'))
+        self.assertFalse('ZeroDivisionError' in output)
+        ar = v.get_result(-2)
+        self.assertRaisesRemote(ZeroDivisionError, ar.get)
+        # prevent TaskAborted on pulls, due to ZeroDivisionError
+        time.sleep(0.5)
+        self.assertEquals(v['a'], 5)
+        # b*=2 will not fire, due to abort
+        self.assertEquals(v['b'], 10)
+    
+    def test_result(self):
+        ip = get_ipython()
+        v = self.client[-1]
+        v.activate()
+        data = dict(a=111,b=222)
+        v.push(data, block=True)
+
+        ip.magic('px a')
+        ip.magic('px b')
+        for idx, name in [
+                    ('', 'b'),
+                    ('-1', 'b'),
+                    ('2', 'b'),
+                    ('1', 'a'),
+                    ('-2', 'a'),
+                ]:
+            with capture_output() as io:
+                ip.magic('result ' + idx)
+            output = io.stdout.strip()
+            msg = "expected %s output to include %s, but got: %s" % \
+                ('%result '+idx, str(data[name]), output)
+            self.assertTrue(str(data[name]) in output, msg)
+        
+    @dec.skipif_not_matplotlib
+    def test_px_pylab(self):
+        """%pylab works on engines"""
+        ip = get_ipython()
+        v = self.client[-1]
+        v.block = True
+        v.activate()
+        
+        with capture_output() as io:
+            ip.magic("px %pylab inline")
+        
+        self.assertTrue("Welcome to pylab" in io.stdout, io.stdout)
+        self.assertTrue("backend_inline" in io.stdout, io.stdout)
+        
+        with capture_output() as io:
+            ip.magic("px plot(rand(100))")
+        
+        self.assertTrue('] Out[' in io.stdout, io.stdout)
+        self.assertTrue('matplotlib.lines' in io.stdout, io.stdout)
+        
+

docs/examples/parallel/Parallel Magics.ipynb

@@ -0,0 +1,228 b''
+{
+ "metadata": {
+  "name": "Parallel Magics"
+ },
+ "nbformat": 3,
+ "worksheets": [
+  {
+   "cells": [
+    {
+     "cell_type": "heading",
+     "level": 1,
+     "source": [
+      "Using Parallel Magics"
+     ]
+    },
+    {
+     "cell_type": "markdown",
+     "source": [
+      "IPython has a few magics for working with your engines.",
+      "",
+      "This assumes you have started an IPython cluster, either with the notebook interface,",
+      "or the `ipcluster/controller/engine` commands."
+     ]
+    },
+    {
+     "cell_type": "code",
+     "collapsed": false,
+     "input": [
+      "from IPython import parallel",
+      "rc = parallel.Client()",
+      "dv = rc[:]",
+      "dv.block = True",
+      "dv"
+     ],
+     "language": "python",
+     "outputs": []
+    },
+    {
+     "cell_type": "markdown",
+     "source": [
+      "The parallel magics come from the `parallelmagics` IPython extension.",
+      "The magics are set to work with a particular View object,",
+      "so to activate them, you call the `activate()` method on a particular view:"
+     ]
+    },
+    {
+     "cell_type": "code",
+     "collapsed": true,
+     "input": [
+      "dv.activate()"
+     ],
+     "language": "python",
+     "outputs": []
+    },
+    {
+     "cell_type": "markdown",
+     "source": [
+      "Now we can execute code remotely with `%px`:"
+     ]
+    },
+    {
+     "cell_type": "code",
+     "collapsed": false,
+     "input": [
+      "%px a=5"
+     ],
+     "language": "python",
+     "outputs": []
+    },
+    {
+     "cell_type": "code",
+     "collapsed": false,
+     "input": [
+      "%px print a"
+     ],
+     "language": "python",
+     "outputs": []
+    },
+    {
+     "cell_type": "code",
+     "collapsed": false,
+     "input": [
+      "%px a"
+     ],
+     "language": "python",
+     "outputs": []
+    },
+    {
+     "cell_type": "markdown",
+     "source": [
+      "You don't have to wait for results:"
+     ]
+    },
+    {
+     "cell_type": "code",
+     "collapsed": true,
+     "input": [
+      "dv.block = False"
+     ],
+     "language": "python",
+     "outputs": []
+    },
+    {
+     "cell_type": "code",
+     "collapsed": false,
+     "input": [
+      "%px import time",
+      "%px time.sleep(5)",
+      "%px time.time()"
+     ],
+     "language": "python",
+     "outputs": []
+    },
+    {
+     "cell_type": "markdown",
+     "source": [
+      "But you will notice that this didn't output the result of the last command.",
+      "For this, we have `%result`, which displays the output of the latest request:"
+     ]
+    },
+    {
+     "cell_type": "code",
+     "collapsed": false,
+     "input": [
+      "%result"
+     ],
+     "language": "python",
+     "outputs": []
+    },
+    {
+     "cell_type": "markdown",
+     "source": [
+      "Remember, an IPython engine is IPython, so you can do magics remotely as well!"
+     ]
+    },
+    {
+     "cell_type": "code",
+     "collapsed": false,
+     "input": [
+      "dv.block = True",
+      "%px %pylab inline"
+     ],
+     "language": "python",
+     "outputs": []
+    },
+    {
+     "cell_type": "markdown",
+     "source": [
+      "`%%px` can also be used as a cell magic, for submitting whole blocks.",
+      "This one acceps `--block` and `--noblock` flags to specify",
+      "the blocking behavior, though the default is unchanged.",
+      ""
+     ]
+    },
+    {
+     "cell_type": "code",
+     "collapsed": true,
+     "input": [
+      "dv.scatter('id', dv.targets, flatten=True)",
+      "dv['stride'] = len(dv)"
+     ],
+     "language": "python",
+     "outputs": []
+    },
+    {
+     "cell_type": "code",
+     "collapsed": false,
+     "input": [
+      "%%px --noblock",
+      "x = linspace(0,pi,1000)",
+      "for n in range(id,12, stride):",
+      "    print n",
+      "    plt.plot(x,sin(n*x))",
+      "plt.title(\"Plot %i\" % id)"
+     ],
+     "language": "python",
+     "outputs": []
+    },
+    {
+     "cell_type": "code",
+     "collapsed": false,
+     "input": [
+      "%result"
+     ],
+     "language": "python",
+     "outputs": []
+    },
+    {
+     "cell_type": "markdown",
+     "source": [
+      "It also lets you choose some amount of the grouping of the outputs with `--group-outputs`:",
+      "",
+      "The choices are:",
+      "",
+      "* `engine` - all of an engine's output is collected together",
+      "* `type` - where stdout of each engine is grouped, etc. (the default)",
+      "* `order` - same as `type`, but individual displaypub outputs are interleaved.",
+      "  That is, it will output the first plot from each engine, then the second from each,",
+      "  etc."
+     ]
+    },
+    {
+     "cell_type": "code",
+     "collapsed": false,
+     "input": [
+      "%%px --group-outputs=engine",
+      "x = linspace(0,pi,1000)",
+      "for n in range(id,12, stride):",
+      "    print n",
+      "    plt.plot(x,sin(n*x))",
+      "plt.title(\"Plot %i\" % id)"
+     ],
+     "language": "python",
+     "outputs": []
+    },
+    {
+     "cell_type": "code",
+     "collapsed": true,
+     "input": [
+      ""
+     ],
+     "language": "python",
+     "outputs": []
+    }
+   ]
+  }
+ ]
+}
  No newline at end of file

IPython/core/magic.py

@@ -1,576 +1,576 b''
 # encoding: utf-8
 """Magic functions for InteractiveShell.
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2001 Janko Hauser <[e-mail]> and
 #  Copyright (C) 2001 Fernando Perez <[e-mail]>
 #  Copyright (C) 2008 The IPython Development Team
 
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 # Stdlib
 import os
 import re
 import sys
 import types
 from getopt import getopt, GetoptError
 
 # Our own
 from IPython.config.configurable import Configurable
 from IPython.core import oinspect
 from IPython.core.error import UsageError
 from IPython.core.prefilter import ESC_MAGIC
 from IPython.external.decorator import decorator
 from IPython.utils.ipstruct import Struct
 from IPython.utils.process import arg_split
 from IPython.utils.text import dedent
 from IPython.utils.traitlets import Bool, Dict, Instance
 from IPython.utils.warn import error, warn
 
 #-----------------------------------------------------------------------------
 # Globals
 #-----------------------------------------------------------------------------
 
 # A dict we'll use for each class that has magics, used as temporary storage to
 # pass information between the @line/cell_magic method decorators and the
 # @magics_class class decorator, because the method decorators have no
 # access to the class when they run.  See for more details:
 # http://stackoverflow.com/questions/2366713/can-a-python-decorator-of-an-instance-method-access-the-class
 
 magics = dict(line={}, cell={})
 
 magic_kinds = ('line', 'cell')
 magic_spec = ('line', 'cell', 'line_cell')
 
 #-----------------------------------------------------------------------------
 # Utility classes and functions
 #-----------------------------------------------------------------------------
 
 class Bunch: pass
 
 
 def on_off(tag):
     """Return an ON/OFF string for a 1/0 input. Simple utility function."""
     return ['OFF','ON'][tag]
 
 
 def compress_dhist(dh):
     """Compress a directory history into a new one with at most 20 entries.
 
     Return a new list made from the first and last 10 elements of dhist after
     removal of duplicates.
     """
     head, tail = dh[:-10], dh[-10:]
 
     newhead = []
     done = set()
     for h in head:
         if h in done:
             continue
         newhead.append(h)
         done.add(h)
 
     return newhead + tail
 
 
 def needs_local_scope(func):
     """Decorator to mark magic functions which need to local scope to run."""
     func.needs_local_scope = True
     return func
 
 #-----------------------------------------------------------------------------
 # Class and method decorators for registering magics
 #-----------------------------------------------------------------------------
 
 def magics_class(cls):
     """Class decorator for all subclasses of the main Magics class.
 
     Any class that subclasses Magics *must* also apply this decorator, to
     ensure that all the methods that have been decorated as line/cell magics
     get correctly registered in the class instance.  This is necessary because
     when method decorators run, the class does not exist yet, so they
     temporarily store their information into a module global.  Application of
     this class decorator copies that global data to the class instance and
     clears the global.
 
     Obviously, this mechanism is not thread-safe, which means that the
     *creation* of subclasses of Magic should only be done in a single-thread
     context.  Instantiation of the classes has no restrictions.  Given that
     these classes are typically created at IPython startup time and before user
     application code becomes active, in practice this should not pose any
     problems.
     """
     cls.registered = True
     cls.magics = dict(line = magics['line'],
                       cell = magics['cell'])
     magics['line'] = {}
     magics['cell'] = {}
     return cls
 
 
 def record_magic(dct, magic_kind, magic_name, func):
     """Utility function to store a function as a magic of a specific kind.
 
     Parameters
     ----------
     dct : dict
       A dictionary with 'line' and 'cell' subdicts.
 
     magic_kind : str
       Kind of magic to be stored.
 
     magic_name : str
       Key to store the magic as.
 
     func : function
       Callable object to store.
     """
     if magic_kind == 'line_cell':
         dct['line'][magic_name] = dct['cell'][magic_name] = func
     else:
         dct[magic_kind][magic_name] = func
 
 
 def validate_type(magic_kind):
     """Ensure that the given magic_kind is valid.
 
     Check that the given magic_kind is one of the accepted spec types (stored
     in the global `magic_spec`), raise ValueError otherwise.
     """
     if magic_kind not in magic_spec:
         raise ValueError('magic_kind must be one of %s, %s given' %
                          magic_kinds, magic_kind)
 
 
 # The docstrings for the decorator below will be fairly similar for the two
 # types (method and function), so we generate them here once and reuse the
 # templates below.
 _docstring_template = \
 """Decorate the given {0} as {1} magic.
 
 The decorator can be used with or without arguments, as follows.
 
 i) without arguments: it will create a {1} magic named as the {0} being
 decorated::
 
     @deco
     def foo(...)
 
 will create a {1} magic named `foo`.
 
 ii) with one string argument: which will be used as the actual name of the
 resulting magic::
 
     @deco('bar')
     def foo(...)
 
 will create a {1} magic named `bar`.
 """
 
 # These two are decorator factories.  While they are conceptually very similar,
 # there are enough differences in the details that it's simpler to have them
 # written as completely standalone functions rather than trying to share code
 # and make a single one with convoluted logic.
 
 def _method_magic_marker(magic_kind):
     """Decorator factory for methods in Magics subclasses.
     """
 
     validate_type(magic_kind)
 
     # This is a closure to capture the magic_kind.  We could also use a class,
     # but it's overkill for just that one bit of state.
     def magic_deco(arg):
         call = lambda f, *a, **k: f(*a, **k)
 
         if callable(arg):
             # "Naked" decorator call (just @foo, no args)
             func = arg
             name = func.func_name
             retval = decorator(call, func)
             record_magic(magics, magic_kind, name, name)
         elif isinstance(arg, basestring):
             # Decorator called with arguments (@foo('bar'))
             name = arg
             def mark(func, *a, **kw):
                 record_magic(magics, magic_kind, name, func.func_name)
                 return decorator(call, func)
             retval = mark
         else:
             raise TypeError("Decorator can only be called with "
                             "string or function")
         return retval
 
     # Ensure the resulting decorator has a usable docstring
     magic_deco.__doc__ = _docstring_template.format('method', magic_kind)
     return magic_deco
 
 
 def _function_magic_marker(magic_kind):
     """Decorator factory for standalone functions.
     """
     validate_type(magic_kind)
     
     # This is a closure to capture the magic_kind.  We could also use a class,
     # but it's overkill for just that one bit of state.
     def magic_deco(arg):
         call = lambda f, *a, **k: f(*a, **k)
 
         # Find get_ipython() in the caller's namespace
         caller = sys._getframe(1)
         for ns in ['f_locals', 'f_globals', 'f_builtins']:
             get_ipython = getattr(caller, ns).get('get_ipython')
             if get_ipython is not None:
                 break
         else:
             raise NameError('Decorator can only run in context where '
                             '`get_ipython` exists')
 
         ip = get_ipython()
 
         if callable(arg):
             # "Naked" decorator call (just @foo, no args)
             func = arg
             name = func.func_name
             ip.register_magic_function(func, magic_kind, name)
             retval = decorator(call, func)
         elif isinstance(arg, basestring):
             # Decorator called with arguments (@foo('bar'))
             name = arg
             def mark(func, *a, **kw):
                 ip.register_magic_function(func, magic_kind, name)
                 return decorator(call, func)
             retval = mark
         else:
             raise TypeError("Decorator can only be called with "
                              "string or function")
         return retval
 
     # Ensure the resulting decorator has a usable docstring
     ds = _docstring_template.format('function', magic_kind)
 
     ds += dedent("""
     Note: this decorator can only be used in a context where IPython is already
     active, so that the `get_ipython()` call succeeds.  You can therefore use
     it in your startup files loaded after IPython initializes, but *not* in the
     IPython configuration file itself, which is executed before IPython is
     fully up and running.  Any file located in the `startup` subdirectory of
     your configuration profile will be OK in this sense.
     """)
     
     magic_deco.__doc__ = ds
     return magic_deco
 
 
 # Create the actual decorators for public use
 
 # These three are used to decorate methods in class definitions
 line_magic = _method_magic_marker('line')
 cell_magic = _method_magic_marker('cell')
 line_cell_magic = _method_magic_marker('line_cell')
 
 # These three decorate standalone functions and perform the decoration
 # immediately.  They can only run where get_ipython() works
 register_line_magic = _function_magic_marker('line')
 register_cell_magic = _function_magic_marker('cell')
 register_line_cell_magic = _function_magic_marker('line_cell')
 
 #-----------------------------------------------------------------------------
 # Core Magic classes
 #-----------------------------------------------------------------------------
 
 class MagicsManager(Configurable):
     """Object that handles all magic-related functionality for IPython.
     """
     # Non-configurable class attributes
 
     # A two-level dict, first keyed by magic type, then by magic function, and
     # holding the actual callable object as value.  This is the dict used for
     # magic function dispatch
     magics = Dict
 
     # A registry of the original objects that we've been given holding magics.
     registry = Dict
 
     shell = Instance('IPython.core.interactiveshell.InteractiveShellABC')
 
     auto_magic = Bool(True, config=True, help=
         "Automatically call line magics without requiring explicit % prefix")
     
     _auto_status = [
         'Automagic is OFF, % prefix IS needed for line magics.',
         'Automagic is ON, % prefix IS NOT needed for line magics.']
 
     user_magics = Instance('IPython.core.magics.UserMagics')
 
     def __init__(self, shell=None, config=None, user_magics=None, **traits):
 
         super(MagicsManager, self).__init__(shell=shell, config=config,
                                            user_magics=user_magics, **traits)
         self.magics = dict(line={}, cell={})
         # Let's add the user_magics to the registry for uniformity, so *all*
         # registered magic containers can be found there.
         self.registry[user_magics.__class__.__name__] = user_magics
 
     def auto_status(self):
         """Return descriptive string with automagic status."""
         return self._auto_status[self.auto_magic]
 
     def lsmagic(self):
         """Return a dict of currently available magic functions.
 
         The return dict has the keys 'line' and 'cell', corresponding to the
         two types of magics we support.  Each value is a list of names.
         """
         return self.magics
 
     def register(self, *magic_objects):
         """Register one or more instances of Magics.
 
         Take one or more classes or instances of classes that subclass the main 
         `core.Magic` class, and register them with IPython to use the magic
         functions they provide.  The registration process will then ensure that
         any methods that have decorated to provide line and/or cell magics will
         be recognized with the `%x`/`%%x` syntax as a line/cell magic
         respectively.
 
         If classes are given, they will be instantiated with the default
         constructor.  If your classes need a custom constructor, you should
         instanitate them first and pass the instance.
 
         The provided arguments can be an arbitrary mix of classes and instances.
 
         Parameters
         ----------
         magic_objects : one or more classes or instances
         """
         # Start by validating them to ensure they have all had their magic
         # methods registered at the instance level
         for m in magic_objects:
             if not m.registered:
                 raise ValueError("Class of magics %r was constructed without "
                                  "the @register_macics class decorator")
             if type(m) is type:
                 # If we're given an uninstantiated class
                 m = m(self.shell)
 
             # Now that we have an instance, we can register it and update the
             # table of callables
             self.registry[m.__class__.__name__] = m
             for mtype in magic_kinds:
                 self.magics[mtype].update(m.magics[mtype])
 
     def register_function(self, func, magic_kind='line', magic_name=None):
         """Expose a standalone function as magic function for IPython.
 
         This will create an IPython magic (line, cell or both) from a
         standalone function.  The functions should have the following
         signatures: 
 
         * For line magics: `def f(line)`
         * For cell magics: `def f(line, cell)`
         * For a function that does both: `def f(line, cell=None)`
 
         In the latter case, the function will be called with `cell==None` when
         invoked as `%f`, and with cell as a string when invoked as `%%f`.
 
         Parameters
         ----------
         func : callable
           Function to be registered as a magic.
 
         magic_kind : str
           Kind of magic, one of 'line', 'cell' or 'line_cell'
 
         magic_name : optional str
           If given, the name the magic will have in the IPython namespace.  By
           default, the name of the function itself is used.
         """
 
         # Create the new method in the user_magics and register it in the
         # global table
         validate_type(magic_kind)
         magic_name = func.func_name if magic_name is None else magic_name
         setattr(self.user_magics, magic_name, func)
         record_magic(self.magics, magic_kind, magic_name, func)
 
     def define_magic(self, name, func):
         """[Deprecated] Expose own function as magic function for IPython.
 
         Example::
 
             def foo_impl(self, parameter_s=''):
                 'My very own magic!. (Use docstrings, IPython reads them).'
                 print 'Magic function. Passed parameter is between < >:'
                 print '<%s>' % parameter_s
                 print 'The self object is:', self
 
             ip.define_magic('foo',foo_impl)
         """
         meth = types.MethodType(func, self.user_magics)
         setattr(self.user_magics, name, meth)
         record_magic(self.magics, 'line', name, meth)
 
 # Key base class that provides the central functionality for magics.
 
 class Magics(object):
     """Base class for implementing magic functions.
 
     Shell functions which can be reached as %function_name. All magic
     functions should accept a string, which they can parse for their own
     needs. This can make some functions easier to type, eg `%cd ../`
     vs. `%cd("../")`
 
     Classes providing magic functions need to subclass this class, and they
     MUST:
 
     - Use the method decorators `@line_magic` and `@cell_magic` to decorate
     individual methods as magic functions, AND
 
     - Use the class decorator `@magics_class` to ensure that the magic
     methods are properly registered at the instance level upon instance
     initialization.
 
     See :mod:`magic_functions` for examples of actual implementation classes.
     """
     # Dict holding all command-line options for each magic.
     options_table = None
     # Dict for the mapping of magic names to methods, set by class decorator
     magics = None
     # Flag to check that the class decorator was properly applied
     registered = False
     # Instance of IPython shell
     shell = None
 
     def __init__(self, shell):
         if not(self.__class__.registered):
             raise ValueError('Magics subclass without registration - '
                              'did you forget to apply @magics_class?')
         self.shell = shell
         self.options_table = {}
         # The method decorators are run when the instance doesn't exist yet, so
         # they can only record the names of the methods they are supposed to
         # grab.  Only now, that the instance exists, can we create the proper
         # mapping to bound methods.  So we read the info off the original names
         # table and replace each method name by the actual bound method.
         for mtype in magic_kinds:
             tab = self.magics[mtype]
             # must explicitly use keys, as we're mutating this puppy
             for magic_name in tab.keys():
                 meth_name = tab[magic_name]
                 if isinstance(meth_name, basestring):
                     tab[magic_name] = getattr(self, meth_name)
 
     def arg_err(self,func):
         """Print docstring if incorrect arguments were passed"""
         print 'Error in arguments:'
         print oinspect.getdoc(func)
 
     def format_latex(self, strng):
         """Format a string for latex inclusion."""
 
         # Characters that need to be escaped for latex:
         escape_re = re.compile(r'(%|_|\$|#|&)',re.MULTILINE)
         # Magic command names as headers:
         cmd_name_re = re.compile(r'^(%s.*?):' % ESC_MAGIC,
                                  re.MULTILINE)
         # Magic commands
         cmd_re = re.compile(r'(?P<cmd>%s.+?\b)(?!\}\}:)' % ESC_MAGIC,
                             re.MULTILINE)
         # Paragraph continue
         par_re = re.compile(r'\\$',re.MULTILINE)
 
         # The "\n" symbol
         newline_re = re.compile(r'\\n')
 
         # Now build the string for output:
         #strng = cmd_name_re.sub(r'\n\\texttt{\\textsl{\\large \1}}:',strng)
         strng = cmd_name_re.sub(r'\n\\bigskip\n\\texttt{\\textbf{ \1}}:',
                                 strng)
         strng = cmd_re.sub(r'\\texttt{\g<cmd>}',strng)
         strng = par_re.sub(r'\\\\',strng)
         strng = escape_re.sub(r'\\\1',strng)
         strng = newline_re.sub(r'\\textbackslash{}n',strng)
         return strng
 
     def parse_options(self, arg_str, opt_str, *long_opts, **kw):
         """Parse options passed to an argument string.
 
         The interface is similar to that of getopt(), but it returns back a
         Struct with the options as keys and the stripped argument string still
         as a string.
 
         arg_str is quoted as a true sys.argv vector by using shlex.split.
         This allows us to easily expand variables, glob files, quote
         arguments, etc.
 
         Options:
           -mode: default 'string'. If given as 'list', the argument string is
           returned as a list (split on whitespace) instead of a string.
 
           -list_all: put all option values in lists. Normally only options
           appearing more than once are put in a list.
 
           -posix (True): whether to split the input line in POSIX mode or not,
           as per the conventions outlined in the shlex module from the
           standard library."""
 
         # inject default options at the beginning of the input line
         caller = sys._getframe(1).f_code.co_name
         arg_str = '%s %s' % (self.options_table.get(caller,''),arg_str)
 
         mode = kw.get('mode','string')
         if mode not in ['string','list']:
             raise ValueError,'incorrect mode given: %s' % mode
         # Get options
         list_all = kw.get('list_all',0)
         posix = kw.get('posix', os.name == 'posix')
         strict = kw.get('strict', True)
 
         # Check if we have more than one argument to warrant extra processing:
         odict = {}  # Dictionary with options
         args = arg_str.split()
         if len(args) >= 1:
             # If the list of inputs only has 0 or 1 thing in it, there's no
             # need to look for options
             argv = arg_split(arg_str, posix, strict)
             # Do regular option processing
             try:
-                opts,args = getopt(argv,opt_str,*long_opts)
+                opts,args = getopt(argv, opt_str, long_opts)
             except GetoptError,e:
                 raise UsageError('%s ( allowed: "%s" %s)' % (e.msg,opt_str,
                                         " ".join(long_opts)))
             for o,a in opts:
                 if o.startswith('--'):
                     o = o[2:]
                 else:
                     o = o[1:]
                 try:
                     odict[o].append(a)
                 except AttributeError:
                     odict[o] = [odict[o],a]
                 except KeyError:
                     if list_all:
                         odict[o] = [a]
                     else:
                         odict[o] = a
 
         # Prepare opts,args for return
         opts = Struct(odict)
         if mode == 'string':
             args = ' '.join(args)
 
         return opts,args
 
     def default_option(self, fn, optstr):
         """Make an entry in the options_table for fn, with value optstr"""
 
         if fn not in self.lsmagic():
             error("%s is not a magic function" % fn)
         self.options_table[fn] = optstr

IPython/core/tests/test_magic.py

@@ -1,547 +1,556 b''
 # -*- coding: utf-8 -*-
 """Tests for various magic functions.
 
 Needs to be run by nose (to make ipython session available).
 """
 from __future__ import absolute_import
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import io
 import os
 import sys
 from StringIO import StringIO
 from unittest import TestCase
 
 import nose.tools as nt
 
 from IPython.core import magic
 from IPython.core.magic import (Magics, magics_class, line_magic,
                                 cell_magic, line_cell_magic,
                                 register_line_magic, register_cell_magic,
                                 register_line_cell_magic)
 from IPython.core.magics import execution
 from IPython.nbformat.v3.tests.nbexamples import nb0
 from IPython.nbformat import current
 from IPython.testing import decorators as dec
 from IPython.testing import tools as tt
 from IPython.utils import py3compat
 from IPython.utils.tempdir import TemporaryDirectory
 
 #-----------------------------------------------------------------------------
 # Test functions begin
 #-----------------------------------------------------------------------------
 
 @magic.magics_class
 class DummyMagics(magic.Magics): pass
 
 def test_rehashx():
     # clear up everything
     _ip = get_ipython()
     _ip.alias_manager.alias_table.clear()
     del _ip.db['syscmdlist']
     
     _ip.magic('rehashx')
     # Practically ALL ipython development systems will have more than 10 aliases
 
     yield (nt.assert_true, len(_ip.alias_manager.alias_table) > 10)
     for key, val in _ip.alias_manager.alias_table.iteritems():
         # we must strip dots from alias names
         nt.assert_true('.' not in key)
 
     # rehashx must fill up syscmdlist
     scoms = _ip.db['syscmdlist']
     yield (nt.assert_true, len(scoms) > 10)
 
 
 def test_magic_parse_options():
     """Test that we don't mangle paths when parsing magic options."""
     ip = get_ipython()
     path = 'c:\\x'
     m = DummyMagics(ip)
     opts = m.parse_options('-f %s' % path,'f:')[0]
     # argv splitting is os-dependent
     if os.name == 'posix':
         expected = 'c:x'
     else:
         expected = path
     nt.assert_equals(opts['f'], expected)
 
+def test_magic_parse_long_options():
+    """Magic.parse_options can handle --foo=bar long options"""
+    ip = get_ipython()
+    m = DummyMagics(ip)
+    opts, _ = m.parse_options('--foo --bar=bubble', 'a', 'foo', 'bar=')
+    nt.assert_true('foo' in opts)
+    nt.assert_true('bar' in opts)
+    nt.assert_true(opts['bar'], "bubble")
+
 
 @dec.skip_without('sqlite3')
 def doctest_hist_f():
     """Test %hist -f with temporary filename.
 
     In [9]: import tempfile
 
     In [10]: tfile = tempfile.mktemp('.py','tmp-ipython-')
 
     In [11]: %hist -nl -f $tfile 3
 
     In [13]: import os; os.unlink(tfile)
     """
 
 
 @dec.skip_without('sqlite3')
 def doctest_hist_r():
     """Test %hist -r
 
     XXX - This test is not recording the output correctly.  For some reason, in
     testing mode the raw history isn't getting populated.  No idea why.
     Disabling the output checking for now, though at least we do run it.
 
     In [1]: 'hist' in _ip.lsmagic()
     Out[1]: True
 
     In [2]: x=1
 
     In [3]: %hist -rl 2
     x=1 # random
     %hist -r 2
     """
 
 
 @dec.skip_without('sqlite3')
 def doctest_hist_op():
     """Test %hist -op
 
     In [1]: class b(float):
        ...:     pass
        ...: 
 
     In [2]: class s(object):
        ...:     def __str__(self):
        ...:         return 's'
        ...: 
 
     In [3]: 
 
     In [4]: class r(b):
        ...:     def __repr__(self):
        ...:         return 'r'
        ...: 
 
     In [5]: class sr(s,r): pass
        ...: 
 
     In [6]: 
 
     In [7]: bb=b()
 
     In [8]: ss=s()
 
     In [9]: rr=r()
 
     In [10]: ssrr=sr()
 
     In [11]: 4.5
     Out[11]: 4.5
 
     In [12]: str(ss)
     Out[12]: 's'
 
     In [13]: 
 
     In [14]: %hist -op
     >>> class b:
     ...     pass
     ... 
     >>> class s(b):
     ...     def __str__(self):
     ...         return 's'
     ... 
     >>> 
     >>> class r(b):
     ...     def __repr__(self):
     ...         return 'r'
     ... 
     >>> class sr(s,r): pass
     >>> 
     >>> bb=b()
     >>> ss=s()
     >>> rr=r()
     >>> ssrr=sr()
     >>> 4.5
     4.5
     >>> str(ss)
     's'
     >>> 
     """
 
 
 @dec.skip_without('sqlite3')
 def test_macro():
     ip = get_ipython()
     ip.history_manager.reset()   # Clear any existing history.
     cmds = ["a=1", "def b():\n  return a**2", "print(a,b())"]
     for i, cmd in enumerate(cmds, start=1):
         ip.history_manager.store_inputs(i, cmd)
     ip.magic("macro test 1-3")
     nt.assert_equal(ip.user_ns["test"].value, "\n".join(cmds)+"\n")
     
     # List macros.
     assert "test" in ip.magic("macro")
 
 
 @dec.skip_without('sqlite3')
 def test_macro_run():
     """Test that we can run a multi-line macro successfully."""
     ip = get_ipython()
     ip.history_manager.reset()
     cmds = ["a=10", "a+=1", py3compat.doctest_refactor_print("print a"),
                                                             "%macro test 2-3"]
     for cmd in cmds:
         ip.run_cell(cmd, store_history=True)
     nt.assert_equal(ip.user_ns["test"].value,
                             py3compat.doctest_refactor_print("a+=1\nprint a\n"))
     with tt.AssertPrints("12"):
         ip.run_cell("test")
     with tt.AssertPrints("13"):
         ip.run_cell("test")
 
 
 @dec.skipif_not_numpy
 def test_numpy_reset_array_undec():
     "Test '%reset array' functionality"
     _ip.ex('import numpy as np')
     _ip.ex('a = np.empty(2)')
     yield (nt.assert_true, 'a' in _ip.user_ns)
     _ip.magic('reset -f array')
     yield (nt.assert_false, 'a' in _ip.user_ns)
 
 def test_reset_out():
     "Test '%reset out' magic"
     _ip.run_cell("parrot = 'dead'", store_history=True)
     # test '%reset -f out', make an Out prompt
     _ip.run_cell("parrot", store_history=True)
     nt.assert_true('dead' in [_ip.user_ns[x] for x in '_','__','___'])
     _ip.magic('reset -f out')
     nt.assert_false('dead' in [_ip.user_ns[x] for x in '_','__','___'])
     nt.assert_true(len(_ip.user_ns['Out']) == 0)
 
 def test_reset_in():
     "Test '%reset in' magic"
     # test '%reset -f in'
     _ip.run_cell("parrot", store_history=True)
     nt.assert_true('parrot' in [_ip.user_ns[x] for x in '_i','_ii','_iii'])
     _ip.magic('%reset -f in')
     nt.assert_false('parrot' in [_ip.user_ns[x] for x in '_i','_ii','_iii'])
     nt.assert_true(len(set(_ip.user_ns['In'])) == 1)
 
 def test_reset_dhist():
     "Test '%reset dhist' magic"
     _ip.run_cell("tmp = [d for d in _dh]") # copy before clearing
     _ip.magic('cd ' + os.path.dirname(nt.__file__))
     _ip.magic('cd -')
     nt.assert_true(len(_ip.user_ns['_dh']) > 0)
     _ip.magic('reset -f dhist')
     nt.assert_true(len(_ip.user_ns['_dh']) == 0)
     _ip.run_cell("_dh = [d for d in tmp]") #restore
 
 def test_reset_in_length():
     "Test that '%reset in' preserves In[] length"
     _ip.run_cell("print 'foo'")
     _ip.run_cell("reset -f in")
     nt.assert_true(len(_ip.user_ns['In']) == _ip.displayhook.prompt_count+1)
 
 def test_time():
     _ip.magic('time None')
 
 def test_tb_syntaxerror():
     """test %tb after a SyntaxError"""
     ip = get_ipython()
     ip.run_cell("for")
     
     # trap and validate stdout
     save_stdout = sys.stdout
     try:
         sys.stdout = StringIO()
         ip.run_cell("%tb")
         out = sys.stdout.getvalue()
     finally:
         sys.stdout = save_stdout
     # trim output, and only check the last line
     last_line = out.rstrip().splitlines()[-1].strip()
     nt.assert_equals(last_line, "SyntaxError: invalid syntax")
 
 
 @py3compat.doctest_refactor_print
 def doctest_time():
     """
     In [10]: %time None
     CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
     Wall time: 0.00 s
     
     In [11]: def f(kmjy):
        ....:    %time print 2*kmjy
        
     In [12]: f(3)
     6
     CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
     Wall time: 0.00 s
     """
 
 
 def test_doctest_mode():
     "Toggle doctest_mode twice, it should be a no-op and run without error"
     _ip.magic('doctest_mode')
     _ip.magic('doctest_mode')
 
 
 def test_parse_options():
     """Tests for basic options parsing in magics."""
     # These are only the most minimal of tests, more should be added later.  At
     # the very least we check that basic text/unicode calls work OK.
     m = DummyMagics(_ip)
     nt.assert_equal(m.parse_options('foo', '')[1], 'foo')
     nt.assert_equal(m.parse_options(u'foo', '')[1], u'foo')
 
     
 def test_dirops():
     """Test various directory handling operations."""
     # curpath = lambda :os.path.splitdrive(os.getcwdu())[1].replace('\\','/')
     curpath = os.getcwdu
     startdir = os.getcwdu()
     ipdir = os.path.realpath(_ip.ipython_dir)
     try:
         _ip.magic('cd "%s"' % ipdir)
         nt.assert_equal(curpath(), ipdir)
         _ip.magic('cd -')
         nt.assert_equal(curpath(), startdir)
         _ip.magic('pushd "%s"' % ipdir)
         nt.assert_equal(curpath(), ipdir)
         _ip.magic('popd')
         nt.assert_equal(curpath(), startdir)
     finally:
         os.chdir(startdir)
 
 
 def test_xmode():
     # Calling xmode three times should be a no-op
     xmode = _ip.InteractiveTB.mode
     for i in range(3):
         _ip.magic("xmode")
     nt.assert_equal(_ip.InteractiveTB.mode, xmode)
     
 def test_reset_hard():
     monitor = []
     class A(object):
         def __del__(self):
             monitor.append(1)
         def __repr__(self):
             return "<A instance>"
             
     _ip.user_ns["a"] = A()
     _ip.run_cell("a")
     
     nt.assert_equal(monitor, [])
     _ip.magic("reset -f")
     nt.assert_equal(monitor, [1])
     
 class TestXdel(tt.TempFileMixin):
     def test_xdel(self):
         """Test that references from %run are cleared by xdel."""
         src = ("class A(object):\n"
                "    monitor = []\n"
                "    def __del__(self):\n"
                "        self.monitor.append(1)\n"
                "a = A()\n")
         self.mktmp(src)
         # %run creates some hidden references...
         _ip.magic("run %s" % self.fname)
         # ... as does the displayhook.
         _ip.run_cell("a")
         
         monitor = _ip.user_ns["A"].monitor
         nt.assert_equal(monitor, [])
         
         _ip.magic("xdel a")
         
         # Check that a's __del__ method has been called.
         nt.assert_equal(monitor, [1])
 
 def doctest_who():
     """doctest for %who
     
     In [1]: %reset -f
     
     In [2]: alpha = 123
     
     In [3]: beta = 'beta'
     
     In [4]: %who int
     alpha
     
     In [5]: %who str
     beta
     
     In [6]: %whos
     Variable   Type    Data/Info
     ----------------------------
     alpha      int     123
     beta       str     beta
     
     In [7]: %who_ls
     Out[7]: ['alpha', 'beta']
     """
 
 def test_whos():
     """Check that whos is protected against objects where repr() fails."""
     class A(object):
         def __repr__(self):
             raise Exception()
     _ip.user_ns['a'] = A()
     _ip.magic("whos")
 
 @py3compat.u_format
 def doctest_precision():
     """doctest for %precision
     
     In [1]: f = get_ipython().display_formatter.formatters['text/plain']
     
     In [2]: %precision 5
     Out[2]: {u}'%.5f'
     
     In [3]: f.float_format
     Out[3]: {u}'%.5f'
     
     In [4]: %precision %e
     Out[4]: {u}'%e'
     
     In [5]: f(3.1415927)
     Out[5]: {u}'3.141593e+00'
     """
 
 def test_psearch():
     with tt.AssertPrints("dict.fromkeys"):
         _ip.run_cell("dict.fr*?")
 
 def test_timeit_shlex():
     """test shlex issues with timeit (#1109)"""
     _ip.ex("def f(*a,**kw): pass")
     _ip.magic('timeit -n1 "this is a bug".count(" ")')
     _ip.magic('timeit -r1 -n1 f(" ", 1)')
     _ip.magic('timeit -r1 -n1 f(" ", 1, " ", 2, " ")')
     _ip.magic('timeit -r1 -n1 ("a " + "b")')
     _ip.magic('timeit -r1 -n1 f("a " + "b")')
     _ip.magic('timeit -r1 -n1 f("a " + "b ")')
 
 
 def test_timeit_arguments():
     "Test valid timeit arguments, should not cause SyntaxError (GH #1269)"
     _ip.magic("timeit ('#')")
 
 
 @dec.skipif(execution.profile is None)
 def test_prun_quotes():
     "Test that prun does not clobber string escapes (GH #1302)"
     _ip.magic("prun -q x = '\t'")
     nt.assert_equal(_ip.user_ns['x'], '\t')
 
 def test_extension():
     tmpdir = TemporaryDirectory()
     orig_ipython_dir = _ip.ipython_dir
     try:
         _ip.ipython_dir = tmpdir.name
         nt.assert_raises(ImportError, _ip.magic, "load_ext daft_extension")
         url = os.path.join(os.path.dirname(__file__), "daft_extension.py")
         _ip.magic("install_ext %s" % url)
         _ip.user_ns.pop('arq', None)
         _ip.magic("load_ext daft_extension")
         tt.assert_equal(_ip.user_ns['arq'], 185)
         _ip.magic("unload_ext daft_extension")
         assert 'arq' not in _ip.user_ns
     finally:
         _ip.ipython_dir = orig_ipython_dir
         
 def test_notebook_export_json():
     with TemporaryDirectory() as td:
         outfile = os.path.join(td, "nb.ipynb")
         _ip.ex(py3compat.u_format(u"u = {u}'héllo'"))
         _ip.magic("notebook -e %s" % outfile)
 
 def test_notebook_export_py():
     with TemporaryDirectory() as td:
         outfile = os.path.join(td, "nb.py")
         _ip.ex(py3compat.u_format(u"u = {u}'héllo'"))
         _ip.magic("notebook -e %s" % outfile)
 
 def test_notebook_reformat_py():
     with TemporaryDirectory() as td:
         infile = os.path.join(td, "nb.ipynb")
         with io.open(infile, 'w', encoding='utf-8') as f:
             current.write(nb0, f, 'json')
             
         _ip.ex(py3compat.u_format(u"u = {u}'héllo'"))
         _ip.magic("notebook -f py %s" % infile)
 
 def test_notebook_reformat_json():
     with TemporaryDirectory() as td:
         infile = os.path.join(td, "nb.py")
         with io.open(infile, 'w', encoding='utf-8') as f:
             current.write(nb0, f, 'py')
             
         _ip.ex(py3compat.u_format(u"u = {u}'héllo'"))
         _ip.magic("notebook -f ipynb %s" % infile)
         _ip.magic("notebook -f json %s" % infile)
 
 def test_env():
     env = _ip.magic("env")
     assert isinstance(env, dict), type(env)
 
 
 class CellMagicTestCase(TestCase):
 
     def check_ident(self, magic):
         # Manually called, we get the result
         out = _ip.run_cell_magic(magic, 'a', 'b')
         nt.assert_equals(out, ('a','b'))
         # Via run_cell, it goes into the user's namespace via displayhook
         _ip.run_cell('%%' + magic +' c\nd')
         nt.assert_equals(_ip.user_ns['_'], ('c','d'))
 
     def test_cell_magic_func_deco(self):
         "Cell magic using simple decorator"
         @register_cell_magic
         def cellm(line, cell):
             return line, cell
 
         self.check_ident('cellm')
 
     def test_cell_magic_reg(self):
         "Cell magic manually registered"
         def cellm(line, cell):
             return line, cell
 
         _ip.register_magic_function(cellm, 'cell', 'cellm2')
         self.check_ident('cellm2')
 
     def test_cell_magic_class(self):
         "Cell magics declared via a class"
         @magics_class
         class MyMagics(Magics):
 
             @cell_magic
             def cellm3(self, line, cell):
                 return line, cell
 
         _ip.register_magics(MyMagics)
         self.check_ident('cellm3')
 
     def test_cell_magic_class2(self):
         "Cell magics declared via a class, #2"
         @magics_class
         class MyMagics2(Magics):
 
             @cell_magic('cellm4')
             def cellm33(self, line, cell):
                 return line, cell
             
         _ip.register_magics(MyMagics2)
         self.check_ident('cellm4')
         # Check that nothing is registered as 'cellm33'
         c33 = _ip.find_cell_magic('cellm33')
         nt.assert_equals(c33, None)
     

IPython/extensions/parallelmagic.py

@@ -1,316 +1,343 b''
 # encoding: utf-8
 """
 =============
 parallelmagic
 =============
 
 Magic command interface for interactive parallel work.
 
 Usage
 =====
 
 ``%autopx``
 
-@AUTOPX_DOC@
+{AUTOPX_DOC}
 
 ``%px``
 
-@PX_DOC@
+{PX_DOC}
 
 ``%result``
 
-@RESULT_DOC@
+{RESULT_DOC}
 
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008 The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import ast
 import re
 
-from IPython.core.magic import Magics, magics_class, line_magic
+from IPython.core.error import UsageError
+from IPython.core.magic import Magics, magics_class, line_magic, cell_magic
 from IPython.testing.skipdoctest import skip_doctest
 
 #-----------------------------------------------------------------------------
 # Definitions of magic functions for use with IPython
 #-----------------------------------------------------------------------------
 
-NO_ACTIVE_VIEW = """
-Use activate() on a DirectView object to activate it for magics.
-"""
+
+NO_ACTIVE_VIEW = "Use activate() on a DirectView object to use it with magics."
 
 
 @magics_class
 class ParallelMagics(Magics):
     """A set of magics useful when controlling a parallel IPython cluster.
     """
 
-    def __init__(self, shell):
-        super(ParallelMagics, self).__init__(shell)
         # A flag showing if autopx is activated or not
-        self.autopx = False
+    _autopx = False
+    # the current view used by the magics:
+    active_view = None
 
     @skip_doctest
     @line_magic
     def result(self, parameter_s=''):
-        """Print the result of command i on all engines..
+        """Print the result of command i on all engines.
 
         To use this a :class:`DirectView` instance must be created
         and then activated by calling its :meth:`activate` method.
 
+        This lets you recall the results of %px computations after
+        asynchronous submission (view.block=False).
+
         Then you can do the following::
 
-            In [23]: %result
-            Out[23]:
-            <Results List>
-            [0] In [6]: a = 10
-            [1] In [6]: a = 10
-
-            In [22]: %result 6
-            Out[22]:
-            <Results List>
-            [0] In [6]: a = 10
-            [1] In [6]: a = 10
+            In [23]: %px os.getpid()
+            Async parallel execution on engine(s): all
+
+            In [24]: %result
+            [ 8] Out[10]: 60920
+            [ 9] Out[10]: 60921
+            [10] Out[10]: 60922
+            [11] Out[10]: 60923
         """
+        
         if self.active_view is None:
-            print NO_ACTIVE_VIEW
-            return
+            raise UsageError(NO_ACTIVE_VIEW)
 
+        stride = len(self.active_view)
         try:
             index = int(parameter_s)
         except:
-            index = None
-        result = self.active_view.get_result(index)
-        return result
+            index = -1
+        msg_ids = self.active_view.history[stride * index:(stride * (index + 1)) or None]
+        
+        result = self.active_view.get_result(msg_ids)
+        
+        result.get()
+        result.display_outputs()
 
     @skip_doctest
     @line_magic
     def px(self, parameter_s=''):
         """Executes the given python command in parallel.
 
         To use this a :class:`DirectView` instance must be created
         and then activated by calling its :meth:`activate` method.
 
         Then you can do the following::
 
-            In [24]: %px a = 5
+            In [24]: %px a = os.getpid()
             Parallel execution on engine(s): all
-            Out[24]:
-            <Results List>
-            [0] In [7]: a = 5
-            [1] In [7]: a = 5
+            
+            In [25]: %px print a
+            [stdout:0] 1234
+            [stdout:1] 1235
+            [stdout:2] 1236
+            [stdout:3] 1237
         """
+        return self.parallel_execute(parameter_s)
+        
+    def parallel_execute(self, cell, block=None, groupby='type'):
+        """implementation used by %px and %%parallel"""
 
         if self.active_view is None:
-            print NO_ACTIVE_VIEW
-            return
-        print "Parallel execution on engine(s): %s" % self.active_view.targets
-        result = self.active_view.execute(parameter_s, block=False)
-        if self.active_view.block:
+            raise UsageError(NO_ACTIVE_VIEW)
+        
+        # defaults:
+        block = self.active_view.block if block is None else block
+        
+        base = "Parallel" if block else "Async parallel"
+        print base + " execution on engine(s): %s" % self.active_view.targets
+        
+        result = self.active_view.execute(cell, silent=False, block=False)
+        if block:
             result.get()
-            self._maybe_display_output(result)
+            result.display_outputs(groupby)
+        else:
+            # return AsyncResult only on non-blocking submission
+            return result
+
+    @skip_doctest
+    @cell_magic('px')
+    def cell_px(self, line='', cell=None):
+        """Executes the given python command in parallel.
+        
+        Cell magic usage:
+        
+        %%px [-o] [-e] [--group-options=type|engine|order] [--[no]block]
+        
+        Options (%%px cell magic only):
+        
+        -o: collate outputs in oder (same as group-outputs=order)
+        
+        -e: group outputs by engine (same as group-outputs=engine)
+        
+        --group-outputs=type [default behavior]:
+            each output type (stdout, stderr, displaypub) for all engines
+            displayed together.
+        
+        --group-outputs=order:
+            The same as 'type', but individual displaypub outputs (e.g. plots)
+            will be interleaved, so it will display all of the first plots,
+            then all of the second plots, etc.
+        
+        --group-outputs=engine:
+            All of an engine's output is displayed before moving on to the next.
+        
+        --[no]block:
+            Whether or not to block for the execution to complete
+            (and display the results).  If unspecified, the active view's
+        
+        
+        To use this a :class:`DirectView` instance must be created
+        and then activated by calling its :meth:`activate` method.
+
+        Then you can do the following::
+
+            In [24]: %%parallel --noblock a = os.getpid()
+            Async parallel execution on engine(s): all
+            
+            In [25]: %px print a
+            [stdout:0] 1234
+            [stdout:1] 1235
+            [stdout:2] 1236
+            [stdout:3] 1237
+        """
+        
+        block = None
+        groupby = 'type'
+        # as a cell magic, we accept args
+        opts, _ = self.parse_options(line, 'oe', 'group-outputs=', 'block', 'noblock')
+
+        if 'group-outputs' in opts:
+            groupby = opts['group-outputs']
+        elif 'o' in opts:
+            groupby = 'order'
+        elif 'e' in opts:
+            groupby = 'engine'
+            
+        if 'block' in opts:
+            block = True
+        elif 'noblock' in opts:
+            block = False
+        
+        return self.parallel_execute(cell, block=block, groupby=groupby)
 
     @skip_doctest
     @line_magic
     def autopx(self, parameter_s=''):
         """Toggles auto parallel mode.
 
         To use this a :class:`DirectView` instance must be created
         and then activated by calling its :meth:`activate` method. Once this
         is called, all commands typed at the command line are send to
         the engines to be executed in parallel. To control which engine
         are used, set the ``targets`` attributed of the multiengine client
         before entering ``%autopx`` mode.
 
         Then you can do the following::
 
             In [25]: %autopx
             %autopx to enabled
 
             In [26]: a = 10
             Parallel execution on engine(s): [0,1,2,3]
             In [27]: print a
             Parallel execution on engine(s): [0,1,2,3]
             [stdout:0] 10
             [stdout:1] 10
             [stdout:2] 10
             [stdout:3] 10
 
 
             In [27]: %autopx
             %autopx disabled
         """
-        if self.autopx:
+        if self._autopx:
             self._disable_autopx()
         else:
             self._enable_autopx()
 
     def _enable_autopx(self):
         """Enable %autopx mode by saving the original run_cell and installing
         pxrun_cell.
         """
         if self.active_view is None:
-            print NO_ACTIVE_VIEW
-            return
+            raise UsageError(NO_ACTIVE_VIEW)
 
-        # override run_cell and run_code
+        # override run_cell
         self._original_run_cell = self.shell.run_cell
         self.shell.run_cell = self.pxrun_cell
-        self._original_run_code = self.shell.run_code
-        self.shell.run_code = self.pxrun_code
 
-        self.autopx = True
+        self._autopx = True
         print "%autopx enabled"
 
     def _disable_autopx(self):
         """Disable %autopx by restoring the original InteractiveShell.run_cell.
         """
-        if self.autopx:
+        if self._autopx:
             self.shell.run_cell = self._original_run_cell
-            self.shell.run_code = self._original_run_code
-            self.autopx = False
+            self._autopx = False
             print "%autopx disabled"
 
-    def _maybe_display_output(self, result):
-        """Maybe display the output of a parallel result.
-
-        If self.active_view.block is True, wait for the result
-        and display the result.  Otherwise, this is a noop.
-        """
-        if isinstance(result.stdout, basestring):
-            # single result
-            stdouts = [result.stdout.rstrip()]
-        else:
-            stdouts = [s.rstrip() for s in result.stdout]
-
-        targets = self.active_view.targets
-        if isinstance(targets, int):
-            targets = [targets]
-        elif targets == 'all':
-            targets = self.active_view.client.ids
-
-        if any(stdouts):
-            for eid,stdout in zip(targets, stdouts):
-                print '[stdout:%i]'%eid, stdout
-
-
     def pxrun_cell(self, raw_cell, store_history=False, silent=False):
         """drop-in replacement for InteractiveShell.run_cell.
 
         This executes code remotely, instead of in the local namespace.
 
         See InteractiveShell.run_cell for details.
         """
 
         if (not raw_cell) or raw_cell.isspace():
             return
 
         ipself = self.shell
 
         with ipself.builtin_trap:
             cell = ipself.prefilter_manager.prefilter_lines(raw_cell)
 
             # Store raw and processed history
             if store_history:
                 ipself.history_manager.store_inputs(ipself.execution_count,
                                                   cell, raw_cell)
 
             # ipself.logger.log(cell, raw_cell)
 
             cell_name = ipself.compile.cache(cell, ipself.execution_count)
 
             try:
                 ast.parse(cell, filename=cell_name)
             except (OverflowError, SyntaxError, ValueError, TypeError,
                     MemoryError):
                 # Case 1
                 ipself.showsyntaxerror()
                 ipself.execution_count += 1
                 return None
             except NameError:
                 # ignore name errors, because we don't know the remote keys
                 pass
 
         if store_history:
             # Write output to the database. Does nothing unless
             # history output logging is enabled.
             ipself.history_manager.store_output(ipself.execution_count)
             # Each cell is a *single* input, regardless of how many lines it has
             ipself.execution_count += 1
         if re.search(r'get_ipython\(\)\.magic\(u?["\']%?autopx', cell):
             self._disable_autopx()
             return False
         else:
             try:
                 result = self.active_view.execute(cell, silent=False, block=False)
             except:
                 ipself.showtraceback()
                 return True
             else:
                 if self.active_view.block:
                     try:
                         result.get()
                     except:
                         self.shell.showtraceback()
                         return True
                     else:
-                        self._maybe_display_output(result)
-                return False
-
-    def pxrun_code(self, code_obj):
-        """drop-in replacement for InteractiveShell.run_code.
-
-        This executes code remotely, instead of in the local namespace.
-
-        See InteractiveShell.run_code for details.
-        """
-        ipself = self.shell
-        # check code object for the autopx magic
-        if 'get_ipython' in code_obj.co_names and 'magic' in code_obj.co_names \
-          and any( [ isinstance(c, basestring) and 'autopx' in c
-                     for c in code_obj.co_consts ]):
-            self._disable_autopx()
-            return False
-        else:
-            try:
-                result = self.active_view.execute(code_obj, block=False)
-            except:
-                ipself.showtraceback()
-                return True
-            else:
-                if self.active_view.block:
-                    try:
-                        result.get()
-                    except:
-                        self.shell.showtraceback()
-                        return True
-                    else:
-                        self._maybe_display_output(result)
+                        with ipself.builtin_trap:
+                            result.display_outputs()
                 return False
 
 
-__doc__ = __doc__.replace('@AUTOPX_DOC@',
-                          "        " + ParallelMagics.autopx.__doc__)
-__doc__ = __doc__.replace('@PX_DOC@',
-                          "        " + ParallelMagics.px.__doc__)
-__doc__ = __doc__.replace('@RESULT_DOC@',
-                          "        " + ParallelMagics.result.__doc__)
+__doc__ = __doc__.format(
+                AUTOPX_DOC = ' '*8 + ParallelMagics.autopx.__doc__,
+                PX_DOC = ' '*8 + ParallelMagics.px.__doc__,
+                RESULT_DOC = ' '*8 + ParallelMagics.result.__doc__
+)
 
 _loaded = False
 
 
 def load_ipython_extension(ip):
     """Load the extension in IPython."""
     global _loaded
     if not _loaded:
         ip.register_magics(ParallelMagics)
         _loaded = True

IPython/parallel/client/asyncresult.py

@@ -1,517 +1,651 b''
 """AsyncResult objects for the client
 
 Authors:
 
 * MinRK
 """
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2010-2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import sys
 import time
 from datetime import datetime
 
 from zmq import MessageTracker
 
-from IPython.core.display import clear_output
+from IPython.core.display import clear_output, display
 from IPython.external.decorator import decorator
 from IPython.parallel import error
 
 #-----------------------------------------------------------------------------
 # Functions
 #-----------------------------------------------------------------------------
 
 def _total_seconds(td):
     """timedelta.total_seconds was added in 2.7"""
     try:
         # Python >= 2.7
         return td.total_seconds()
     except AttributeError:
         # Python 2.6
         return 1e-6 * (td.microseconds + (td.seconds + td.days * 24 * 3600) * 10**6)
 
 #-----------------------------------------------------------------------------
 # Classes
 #-----------------------------------------------------------------------------
 
 # global empty tracker that's always done:
 finished_tracker = MessageTracker()
 
 @decorator
 def check_ready(f, self, *args, **kwargs):
     """Call spin() to sync state prior to calling the method."""
     self.wait(0)
     if not self._ready:
         raise error.TimeoutError("result not ready")
     return f(self, *args, **kwargs)
 
 class AsyncResult(object):
     """Class for representing results of non-blocking calls.
 
     Provides the same interface as :py:class:`multiprocessing.pool.AsyncResult`.
     """
 
     msg_ids = None
     _targets = None
     _tracker = None
     _single_result = False
 
     def __init__(self, client, msg_ids, fname='unknown', targets=None, tracker=None):
         if isinstance(msg_ids, basestring):
             # always a list
             msg_ids = [msg_ids]
         if tracker is None:
             # default to always done
             tracker = finished_tracker
         self._client = client
         self.msg_ids = msg_ids
         self._fname=fname
         self._targets = targets
         self._tracker = tracker
         self._ready = False
         self._success = None
         self._metadata = None
         if len(msg_ids) == 1:
             self._single_result = not isinstance(targets, (list, tuple))
         else:
             self._single_result = False
 
     def __repr__(self):
         if self._ready:
             return "<%s: finished>"%(self.__class__.__name__)
         else:
             return "<%s: %s>"%(self.__class__.__name__,self._fname)
 
 
     def _reconstruct_result(self, res):
         """Reconstruct our result from actual result list (always a list)
 
         Override me in subclasses for turning a list of results
         into the expected form.
         """
         if self._single_result:
             return res[0]
         else:
             return res
 
     def get(self, timeout=-1):
         """Return the result when it arrives.
 
         If `timeout` is not ``None`` and the result does not arrive within
         `timeout` seconds then ``TimeoutError`` is raised. If the
         remote call raised an exception then that exception will be reraised
         by get() inside a `RemoteError`.
         """
         if not self.ready():
             self.wait(timeout)
 
         if self._ready:
             if self._success:
                 return self._result
             else:
                 raise self._exception
         else:
             raise error.TimeoutError("Result not ready.")
 
     def ready(self):
         """Return whether the call has completed."""
         if not self._ready:
             self.wait(0)
         return self._ready
 
     def wait(self, timeout=-1):
         """Wait until the result is available or until `timeout` seconds pass.
 
         This method always returns None.
         """
         if self._ready:
             return
         self._ready = self._client.wait(self.msg_ids, timeout)
         if self._ready:
             try:
                 results = map(self._client.results.get, self.msg_ids)
                 self._result = results
                 if self._single_result:
                     r = results[0]
                     if isinstance(r, Exception):
                         raise r
                 else:
                     results = error.collect_exceptions(results, self._fname)
                 self._result = self._reconstruct_result(results)
             except Exception, e:
                 self._exception = e
                 self._success = False
             else:
                 self._success = True
             finally:
                 self._metadata = map(self._client.metadata.get, self.msg_ids)
 
 
     def successful(self):
         """Return whether the call completed without raising an exception.
 
         Will raise ``AssertionError`` if the result is not ready.
         """
         assert self.ready()
         return self._success
 
     #----------------------------------------------------------------
     # Extra methods not in mp.pool.AsyncResult
     #----------------------------------------------------------------
 
     def get_dict(self, timeout=-1):
         """Get the results as a dict, keyed by engine_id.
 
         timeout behavior is described in `get()`.
         """
 
         results = self.get(timeout)
         engine_ids = [ md['engine_id'] for md in self._metadata ]
         bycount = sorted(engine_ids, key=lambda k: engine_ids.count(k))
         maxcount = bycount.count(bycount[-1])
         if maxcount > 1:
             raise ValueError("Cannot build dict, %i jobs ran on engine #%i"%(
                     maxcount, bycount[-1]))
 
         return dict(zip(engine_ids,results))
 
     @property
     def result(self):
         """result property wrapper for `get(timeout=0)`."""
         return self.get()
 
     # abbreviated alias:
     r = result
 
     @property
     @check_ready
     def metadata(self):
         """property for accessing execution metadata."""
         if self._single_result:
             return self._metadata[0]
         else:
             return self._metadata
 
     @property
     def result_dict(self):
         """result property as a dict."""
         return self.get_dict()
 
     def __dict__(self):
         return self.get_dict(0)
 
     def abort(self):
         """abort my tasks."""
         assert not self.ready(), "Can't abort, I am already done!"
         return self._client.abort(self.msg_ids, targets=self._targets, block=True)
 
     @property
     def sent(self):
         """check whether my messages have been sent."""
         return self._tracker.done
 
     def wait_for_send(self, timeout=-1):
         """wait for pyzmq send to complete.
 
         This is necessary when sending arrays that you intend to edit in-place.
         `timeout` is in seconds, and will raise TimeoutError if it is reached
         before the send completes.
         """
         return self._tracker.wait(timeout)
 
     #-------------------------------------
     # dict-access
     #-------------------------------------
 
     @check_ready
     def __getitem__(self, key):
         """getitem returns result value(s) if keyed by int/slice, or metadata if key is str.
         """
         if isinstance(key, int):
             return error.collect_exceptions([self._result[key]], self._fname)[0]
         elif isinstance(key, slice):
             return error.collect_exceptions(self._result[key], self._fname)
         elif isinstance(key, basestring):
             values = [ md[key] for md in self._metadata ]
             if self._single_result:
                 return values[0]
             else:
                 return values
         else:
             raise TypeError("Invalid key type %r, must be 'int','slice', or 'str'"%type(key))
 
     def __getattr__(self, key):
         """getattr maps to getitem for convenient attr access to metadata."""
         try:
             return self.__getitem__(key)
         except (error.TimeoutError, KeyError):
             raise AttributeError("%r object has no attribute %r"%(
                     self.__class__.__name__, key))
 
     # asynchronous iterator:
     def __iter__(self):
         if self._single_result:
             raise TypeError("AsyncResults with a single result are not iterable.")
         try:
             rlist = self.get(0)
         except error.TimeoutError:
             # wait for each result individually
             for msg_id in self.msg_ids:
                 ar = AsyncResult(self._client, msg_id, self._fname)
                 yield ar.get()
         else:
             # already done
             for r in rlist:
                 yield r
     
     def __len__(self):
         return len(self.msg_ids)
     
     #-------------------------------------
     # Sugar methods and attributes
     #-------------------------------------
     
     def timedelta(self, start, end, start_key=min, end_key=max):
         """compute the difference between two sets of timestamps
         
         The default behavior is to use the earliest of the first
         and the latest of the second list, but this can be changed
         by passing a different
         
         Parameters
         ----------
         
         start : one or more datetime objects (e.g. ar.submitted)
         end : one or more datetime objects (e.g. ar.received)
         start_key : callable
             Function to call on `start` to extract the relevant
             entry [defalt: min]
         end_key : callable
             Function to call on `end` to extract the relevant
             entry [default: max]
         
         Returns
         -------
         
         dt : float
             The time elapsed (in seconds) between the two selected timestamps.
         """
         if not isinstance(start, datetime):
             # handle single_result AsyncResults, where ar.stamp is single object,
             # not a list
             start = start_key(start)
         if not isinstance(end, datetime):
             # handle single_result AsyncResults, where ar.stamp is single object,
             # not a list
             end = end_key(end)
         return _total_seconds(end - start)
         
     @property
     def progress(self):
         """the number of tasks which have been completed at this point.
         
         Fractional progress would be given by 1.0 * ar.progress / len(ar)
         """
         self.wait(0)
         return len(self) - len(set(self.msg_ids).intersection(self._client.outstanding))
     
     @property
     def elapsed(self):
         """elapsed time since initial submission"""
         if self.ready():
             return self.wall_time
         
         now = submitted = datetime.now()
         for msg_id in self.msg_ids:
             if msg_id in self._client.metadata:
                 stamp = self._client.metadata[msg_id]['submitted']
                 if stamp and stamp < submitted:
                     submitted = stamp
         return _total_seconds(now-submitted)
     
     @property
     @check_ready
     def serial_time(self):
         """serial computation time of a parallel calculation
         
         Computed as the sum of (completed-started) of each task
         """
         t = 0
         for md in self._metadata:
             t += _total_seconds(md['completed'] - md['started'])
         return t
     
     @property
     @check_ready
     def wall_time(self):
         """actual computation time of a parallel calculation
         
         Computed as the time between the latest `received` stamp
         and the earliest `submitted`.
         
         Only reliable if Client was spinning/waiting when the task finished, because
         the `received` timestamp is created when a result is pulled off of the zmq queue,
         which happens as a result of `client.spin()`.
         
         For similar comparison of other timestamp pairs, check out AsyncResult.timedelta.
         
         """
         return self.timedelta(self.submitted, self.received)
     
     def wait_interactive(self, interval=1., timeout=None):
         """interactive wait, printing progress at regular intervals"""
         N = len(self)
         tic = time.time()
         while not self.ready() and (timeout is None or time.time() - tic <= timeout):
             self.wait(interval)
             clear_output()
             print "%4i/%i tasks finished after %4i s" % (self.progress, N, self.elapsed),
             sys.stdout.flush()
         print
         print "done"
 
+    def _republish_displaypub(self, content, eid):
+        """republish individual displaypub content dicts"""
+        try:
+            ip = get_ipython()
+        except NameError:
+            # displaypub is meaningless outside IPython
+            return
+        md = content['metadata'] or {}
+        md['engine'] = eid
+        ip.display_pub.publish(content['source'], content['data'], md)
+        
+        
+    def _display_single_result(self):
+        
+        print self.stdout
+        print >> sys.stderr, self.stderr
+        
+        try:
+            get_ipython()
+        except NameError:
+            # displaypub is meaningless outside IPython
+            return
+            
+        for output in self.outputs:
+            self._republish_displaypub(output, self.engine_id)
+        
+        if self.pyout is not None:
+            display(self.get())
+    
+    @check_ready
+    def display_outputs(self, groupby="type"):
+        """republish the outputs of the computation
+        
+        Parameters
+        ----------
+        
+        groupby : str [default: type]
+            if 'type':
+                Group outputs by type (show all stdout, then all stderr, etc.):
+                
+                [stdout:1] foo
+                [stdout:2] foo
+                [stderr:1] bar
+                [stderr:2] bar
+            if 'engine':
+                Display outputs for each engine before moving on to the next:
+                
+                [stdout:1] foo
+                [stderr:1] bar
+                [stdout:2] foo
+                [stderr:2] bar
+                
+            if 'order':
+                Like 'type', but further collate individual displaypub
+                outputs.  This is meant for cases of each command producing
+                several plots, and you would like to see all of the first
+                plots together, then all of the second plots, and so on.
+        """
+        # flush iopub, just in case
+        self._client._flush_iopub(self._client._iopub_socket)
+        if self._single_result:
+            self._display_single_result()
+            return
+        
+        stdouts = [s.rstrip() for s in self.stdout]
+        stderrs = [s.rstrip() for s in self.stderr]
+        pyouts  = [p for p in self.pyout]
+        output_lists = self.outputs
+        results = self.get()
+        
+        targets = self.engine_id
+        
+        if groupby == "engine":
+            for eid,stdout,stderr,outputs,r,pyout in zip(
+                    targets, stdouts, stderrs, output_lists, results, pyouts
+                ):
+                if stdout:
+                    print '[stdout:%i]' % eid, stdout
+                if stderr:
+                    print >> sys.stderr, '[stderr:%i]' % eid, stderr
+                
+                try:
+                    get_ipython()
+                except NameError:
+                    # displaypub is meaningless outside IPython
+                    return 
+                
+                for output in outputs:
+                    self._republish_displaypub(output, eid)
+                
+                if pyout is not None:
+                    display(r)
+        
+        elif groupby in ('type', 'order'):
+            # republish stdout:
+            if any(stdouts):
+                for eid,stdout in zip(targets, stdouts):
+                    print '[stdout:%i]' % eid, stdout
+        
+            # republish stderr:
+            if any(stderrs):
+                for eid,stderr in zip(targets, stderrs):
+                    print >> sys.stderr, '[stderr:%i]' % eid, stderr
+        
+            try:
+                get_ipython()
+            except NameError:
+                # displaypub is meaningless outside IPython
+                return
+            
+            if groupby == 'order':
+                output_dict = dict((eid, outputs) for eid,outputs in zip(targets, output_lists))
+                N = max(len(outputs) for outputs in output_lists)
+                for i in range(N):
+                    for eid in targets:
+                        outputs = output_dict[eid]
+                        if len(outputs) >= N:
+                            self._republish_displaypub(outputs[i], eid)
+            else:
+                # republish displaypub output
+                for eid,outputs in zip(targets, output_lists):
+                    for output in outputs:
+                        self._republish_displaypub(output, eid)
+        
+            # finally, add pyout:
+            for eid,r,pyout in zip(targets, results, pyouts):
+                if pyout is not None:
+                    display(r)
+        
+        else:
+            raise ValueError("groupby must be one of 'type', 'engine', 'collate', not %r" % groupby)
+        
+        
+
 
 class AsyncMapResult(AsyncResult):
     """Class for representing results of non-blocking gathers.
 
     This will properly reconstruct the gather.
     
     This class is iterable at any time, and will wait on results as they come.
     
     If ordered=False, then the first results to arrive will come first, otherwise
     results will be yielded in the order they were submitted.
     
     """
 
     def __init__(self, client, msg_ids, mapObject, fname='', ordered=True):
         AsyncResult.__init__(self, client, msg_ids, fname=fname)
         self._mapObject = mapObject
         self._single_result = False
         self.ordered = ordered
 
     def _reconstruct_result(self, res):
         """Perform the gather on the actual results."""
         return self._mapObject.joinPartitions(res)
 
     # asynchronous iterator:
     def __iter__(self):
         it = self._ordered_iter if self.ordered else self._unordered_iter
         for r in it():
             yield r
 
     # asynchronous ordered iterator:
     def _ordered_iter(self):
         """iterator for results *as they arrive*, preserving submission order."""
         try:
             rlist = self.get(0)
         except error.TimeoutError:
             # wait for each result individually
             for msg_id in self.msg_ids:
                 ar = AsyncResult(self._client, msg_id, self._fname)
                 rlist = ar.get()
                 try:
                     for r in rlist:
                         yield r
                 except TypeError:
                     # flattened, not a list
                     # this could get broken by flattened data that returns iterables
                     # but most calls to map do not expose the `flatten` argument
                     yield rlist
         else:
             # already done
             for r in rlist:
                 yield r
 
     # asynchronous unordered iterator:
     def _unordered_iter(self):
         """iterator for results *as they arrive*, on FCFS basis, ignoring submission order."""
         try:
             rlist = self.get(0)
         except error.TimeoutError:
             pending = set(self.msg_ids)
             while pending:
                 try:
                     self._client.wait(pending, 1e-3)
                 except error.TimeoutError:
                     # ignore timeout error, because that only means
                     # *some* jobs are outstanding
                     pass
                 # update ready set with those no longer outstanding:
                 ready = pending.difference(self._client.outstanding)
                 # update pending to exclude those that are finished
                 pending = pending.difference(ready)
                 while ready:
                     msg_id = ready.pop()
                     ar = AsyncResult(self._client, msg_id, self._fname)
                     rlist = ar.get()
                     try:
                         for r in rlist:
                             yield r
                     except TypeError:
                         # flattened, not a list
                         # this could get broken by flattened data that returns iterables
                         # but most calls to map do not expose the `flatten` argument
                         yield rlist
         else:
             # already done
             for r in rlist:
                 yield r
 
 
 
 class AsyncHubResult(AsyncResult):
     """Class to wrap pending results that must be requested from the Hub.
 
     Note that waiting/polling on these objects requires polling the Hubover the network,
     so use `AsyncHubResult.wait()` sparingly.
     """
 
     def wait(self, timeout=-1):
         """wait for result to complete."""
         start = time.time()
         if self._ready:
             return
         local_ids = filter(lambda msg_id: msg_id in self._client.outstanding, self.msg_ids)
         local_ready = self._client.wait(local_ids, timeout)
         if local_ready:
             remote_ids = filter(lambda msg_id: msg_id not in self._client.results, self.msg_ids)
             if not remote_ids:
                 self._ready = True
             else:
                 rdict = self._client.result_status(remote_ids, status_only=False)
                 pending = rdict['pending']
                 while pending and (timeout < 0 or time.time() < start+timeout):
                     rdict = self._client.result_status(remote_ids, status_only=False)
                     pending = rdict['pending']
                     if pending:
                         time.sleep(0.1)
                 if not pending:
                     self._ready = True
         if self._ready:
             try:
                 results = map(self._client.results.get, self.msg_ids)
                 self._result = results
                 if self._single_result:
                     r = results[0]
                     if isinstance(r, Exception):
                         raise r
                 else:
                     results = error.collect_exceptions(results, self._fname)
                 self._result = self._reconstruct_result(results)
             except Exception, e:
                 self._exception = e
                 self._success = False
             else:
                 self._success = True
             finally:
                 self._metadata = map(self._client.metadata.get, self.msg_ids)
 
 __all__ = ['AsyncResult', 'AsyncMapResult', 'AsyncHubResult']
  No newline at end of file

IPython/parallel/client/client.py

@@ -1,1628 +1,1655 b''
 """A semi-synchronous Client for the ZMQ cluster
 
 Authors:
 
 * MinRK
 """
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2010-2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import os
 import json
 import sys
 from threading import Thread, Event
 import time
 import warnings
 from datetime import datetime
 from getpass import getpass
 from pprint import pprint
 
 pjoin = os.path.join
 
 import zmq
 # from zmq.eventloop import ioloop, zmqstream
 
 from IPython.config.configurable import MultipleInstanceError
 from IPython.core.application import BaseIPythonApplication
 
+from IPython.utils.coloransi import TermColors
 from IPython.utils.jsonutil import rekey
 from IPython.utils.localinterfaces import LOCAL_IPS
 from IPython.utils.path import get_ipython_dir
 from IPython.utils.py3compat import cast_bytes
 from IPython.utils.traitlets import (HasTraits, Integer, Instance, Unicode,
                                     Dict, List, Bool, Set, Any)
 from IPython.external.decorator import decorator
 from IPython.external.ssh import tunnel
 
 from IPython.parallel import Reference
 from IPython.parallel import error
 from IPython.parallel import util
 
 from IPython.zmq.session import Session, Message
 
 from .asyncresult import AsyncResult, AsyncHubResult
 from IPython.core.profiledir import ProfileDir, ProfileDirError
 from .view import DirectView, LoadBalancedView
 
 if sys.version_info[0] >= 3:
     # xrange is used in a couple 'isinstance' tests in py2
     # should be just 'range' in 3k
     xrange = range
 
 #--------------------------------------------------------------------------
 # Decorators for Client methods
 #--------------------------------------------------------------------------
 
 @decorator
 def spin_first(f, self, *args, **kwargs):
     """Call spin() to sync state prior to calling the method."""
     self.spin()
     return f(self, *args, **kwargs)
 
 
 #--------------------------------------------------------------------------
 # Classes
 #--------------------------------------------------------------------------
 
 
 class ExecuteReply(object):
     """wrapper for finished Execute results"""
     def __init__(self, msg_id, content, metadata):
         self.msg_id = msg_id
         self._content = content
         self.execution_count = content['execution_count']
         self.metadata = metadata
     
     def __getitem__(self, key):
         return self.metadata[key]
     
     def __getattr__(self, key):
         if key not in self.metadata:
             raise AttributeError(key)
         return self.metadata[key]
     
     def __repr__(self):
-        pyout = self.metadata['pyout'] or {}
-        text_out = pyout.get('data', {}).get('text/plain', '')
+        pyout = self.metadata['pyout'] or {'data':{}}
+        text_out = pyout['data'].get('text/plain', '')
         if len(text_out) > 32:
             text_out = text_out[:29] + '...'
         
         return "<ExecuteReply[%i]: %s>" % (self.execution_count, text_out)
     
+    def _repr_pretty_(self, p, cycle):
+        pyout = self.metadata['pyout'] or {'data':{}}
+        text_out = pyout['data'].get('text/plain', '')
+        
+        if not text_out:
+            return
+        
+        try:
+            ip = get_ipython()
+        except NameError:
+            colors = "NoColor"
+        else:
+            colors = ip.colors
+        
+        if colors == "NoColor":
+            out = normal = ""
+        else:
+            out = TermColors.Red
+            normal = TermColors.Normal
+        
+        p.text(
+            u'[%i] ' % self.metadata['engine_id'] +
+            out + u'Out[%i]: ' % self.execution_count +
+            normal + text_out
+        )
+    
     def _repr_html_(self):
         pyout = self.metadata['pyout'] or {'data':{}}
         return pyout['data'].get("text/html")
     
     def _repr_latex_(self):
         pyout = self.metadata['pyout'] or {'data':{}}
         return pyout['data'].get("text/latex")
     
     def _repr_json_(self):
         pyout = self.metadata['pyout'] or {'data':{}}
         return pyout['data'].get("application/json")
     
     def _repr_javascript_(self):
         pyout = self.metadata['pyout'] or {'data':{}}
         return pyout['data'].get("application/javascript")
     
     def _repr_png_(self):
         pyout = self.metadata['pyout'] or {'data':{}}
         return pyout['data'].get("image/png")
     
     def _repr_jpeg_(self):
         pyout = self.metadata['pyout'] or {'data':{}}
         return pyout['data'].get("image/jpeg")
     
     def _repr_svg_(self):
         pyout = self.metadata['pyout'] or {'data':{}}
         return pyout['data'].get("image/svg+xml")
 
 
 class Metadata(dict):
     """Subclass of dict for initializing metadata values.
 
     Attribute access works on keys.
 
     These objects have a strict set of keys - errors will raise if you try
     to add new keys.
     """
     def __init__(self, *args, **kwargs):
         dict.__init__(self)
         md = {'msg_id' : None,
               'submitted' : None,
               'started' : None,
               'completed' : None,
               'received' : None,
               'engine_uuid' : None,
               'engine_id' : None,
               'follow' : None,
               'after' : None,
               'status' : None,
 
               'pyin' : None,
               'pyout' : None,
               'pyerr' : None,
               'stdout' : '',
               'stderr' : '',
               'outputs' : [],
             }
         self.update(md)
         self.update(dict(*args, **kwargs))
 
     def __getattr__(self, key):
         """getattr aliased to getitem"""
         if key in self.iterkeys():
             return self[key]
         else:
             raise AttributeError(key)
 
     def __setattr__(self, key, value):
         """setattr aliased to setitem, with strict"""
         if key in self.iterkeys():
             self[key] = value
         else:
             raise AttributeError(key)
 
     def __setitem__(self, key, value):
         """strict static key enforcement"""
         if key in self.iterkeys():
             dict.__setitem__(self, key, value)
         else:
             raise KeyError(key)
 
 
 class Client(HasTraits):
     """A semi-synchronous client to the IPython ZMQ cluster
 
     Parameters
     ----------
 
     url_or_file : bytes or unicode; zmq url or path to ipcontroller-client.json
         Connection information for the Hub's registration.  If a json connector
         file is given, then likely no further configuration is necessary.
         [Default: use profile]
     profile : bytes
         The name of the Cluster profile to be used to find connector information.
         If run from an IPython application, the default profile will be the same
         as the running application, otherwise it will be 'default'.
     context : zmq.Context
         Pass an existing zmq.Context instance, otherwise the client will create its own.
     debug : bool
         flag for lots of message printing for debug purposes
     timeout : int/float
         time (in seconds) to wait for connection replies from the Hub
         [Default: 10]
 
     #-------------- session related args ----------------
 
     config : Config object
         If specified, this will be relayed to the Session for configuration
     username : str
         set username for the session object
     packer : str (import_string) or callable
         Can be either the simple keyword 'json' or 'pickle', or an import_string to a
         function to serialize messages. Must support same input as
         JSON, and output must be bytes.
         You can pass a callable directly as `pack`
     unpacker : str (import_string) or callable
         The inverse of packer.  Only necessary if packer is specified as *not* one
         of 'json' or 'pickle'.
 
     #-------------- ssh related args ----------------
     # These are args for configuring the ssh tunnel to be used
     # credentials are used to forward connections over ssh to the Controller
     # Note that the ip given in `addr` needs to be relative to sshserver
     # The most basic case is to leave addr as pointing to localhost (127.0.0.1),
     # and set sshserver as the same machine the Controller is on. However,
     # the only requirement is that sshserver is able to see the Controller
     # (i.e. is within the same trusted network).
 
     sshserver : str
         A string of the form passed to ssh, i.e. 'server.tld' or '[e-mail]:port'
         If keyfile or password is specified, and this is not, it will default to
         the ip given in addr.
     sshkey : str; path to ssh private key file
         This specifies a key to be used in ssh login, default None.
         Regular default ssh keys will be used without specifying this argument.
     password : str
         Your ssh password to sshserver. Note that if this is left None,
         you will be prompted for it if passwordless key based login is unavailable.
     paramiko : bool
         flag for whether to use paramiko instead of shell ssh for tunneling.
         [default: True on win32, False else]
 
     ------- exec authentication args -------
     If even localhost is untrusted, you can have some protection against
     unauthorized execution by signing messages with HMAC digests.
     Messages are still sent as cleartext, so if someone can snoop your
     loopback traffic this will not protect your privacy, but will prevent
     unauthorized execution.
 
     exec_key : str
         an authentication key or file containing a key
         default: None
 
 
     Attributes
     ----------
 
     ids : list of int engine IDs
         requesting the ids attribute always synchronizes
         the registration state. To request ids without synchronization,
         use semi-private _ids attributes.
 
     history : list of msg_ids
         a list of msg_ids, keeping track of all the execution
         messages you have submitted in order.
 
     outstanding : set of msg_ids
         a set of msg_ids that have been submitted, but whose
         results have not yet been received.
 
     results : dict
         a dict of all our results, keyed by msg_id
 
     block : bool
         determines default behavior when block not specified
         in execution methods
 
     Methods
     -------
 
     spin
         flushes incoming results and registration state changes
         control methods spin, and requesting `ids` also ensures up to date
 
     wait
         wait on one or more msg_ids
 
     execution methods
         apply
         legacy: execute, run
 
     data movement
         push, pull, scatter, gather
 
     query methods
         queue_status, get_result, purge, result_status
 
     control methods
         abort, shutdown
 
     """
 
 
     block = Bool(False)
     outstanding = Set()
     results = Instance('collections.defaultdict', (dict,))
     metadata = Instance('collections.defaultdict', (Metadata,))
     history = List()
     debug = Bool(False)
     _spin_thread = Any()
     _stop_spinning = Any()
 
     profile=Unicode()
     def _profile_default(self):
         if BaseIPythonApplication.initialized():
             # an IPython app *might* be running, try to get its profile
             try:
                 return BaseIPythonApplication.instance().profile
             except (AttributeError, MultipleInstanceError):
                 # could be a *different* subclass of config.Application,
                 # which would raise one of these two errors.
                 return u'default'
         else:
             return u'default'
 
 
     _outstanding_dict = Instance('collections.defaultdict', (set,))
     _ids = List()
     _connected=Bool(False)
     _ssh=Bool(False)
     _context = Instance('zmq.Context')
     _config = Dict()
     _engines=Instance(util.ReverseDict, (), {})
     # _hub_socket=Instance('zmq.Socket')
     _query_socket=Instance('zmq.Socket')
     _control_socket=Instance('zmq.Socket')
     _iopub_socket=Instance('zmq.Socket')
     _notification_socket=Instance('zmq.Socket')
     _mux_socket=Instance('zmq.Socket')
     _task_socket=Instance('zmq.Socket')
     _task_scheme=Unicode()
     _closed = False
     _ignored_control_replies=Integer(0)
     _ignored_hub_replies=Integer(0)
 
     def __new__(self, *args, **kw):
         # don't raise on positional args
         return HasTraits.__new__(self, **kw)
 
     def __init__(self, url_or_file=None, profile=None, profile_dir=None, ipython_dir=None,
             context=None, debug=False, exec_key=None,
             sshserver=None, sshkey=None, password=None, paramiko=None,
             timeout=10, **extra_args
             ):
         if profile:
             super(Client, self).__init__(debug=debug, profile=profile)
         else:
             super(Client, self).__init__(debug=debug)
         if context is None:
             context = zmq.Context.instance()
         self._context = context
         self._stop_spinning = Event()
 
         self._setup_profile_dir(self.profile, profile_dir, ipython_dir)
         if self._cd is not None:
             if url_or_file is None:
                 url_or_file = pjoin(self._cd.security_dir, 'ipcontroller-client.json')
         if url_or_file is None:
             raise ValueError(
                 "I can't find enough information to connect to a hub!"
                 " Please specify at least one of url_or_file or profile."
             )
 
         if not util.is_url(url_or_file):
             # it's not a url, try for a file
             if not os.path.exists(url_or_file):
                 if self._cd:
                     url_or_file = os.path.join(self._cd.security_dir, url_or_file)
                 if not os.path.exists(url_or_file):
                     raise IOError("Connection file not found: %r" % url_or_file)
             with open(url_or_file) as f:
                 cfg = json.loads(f.read())
         else:
             cfg = {'url':url_or_file}
 
         # sync defaults from args, json:
         if sshserver:
             cfg['ssh'] = sshserver
         if exec_key:
             cfg['exec_key'] = exec_key
         exec_key = cfg['exec_key']
         location = cfg.setdefault('location', None)
         cfg['url'] = util.disambiguate_url(cfg['url'], location)
         url = cfg['url']
         proto,addr,port = util.split_url(url)
         if location is not None and addr == '127.0.0.1':
             # location specified, and connection is expected to be local
             if location not in LOCAL_IPS and not sshserver:
                 # load ssh from JSON *only* if the controller is not on
                 # this machine
                 sshserver=cfg['ssh']
             if location not in LOCAL_IPS and not sshserver:
                 # warn if no ssh specified, but SSH is probably needed
                 # This is only a warning, because the most likely cause
                 # is a local Controller on a laptop whose IP is dynamic
                 warnings.warn("""
             Controller appears to be listening on localhost, but not on this machine.
             If this is true, you should specify Client(...,sshserver='you@%s')
             or instruct your controller to listen on an external IP."""%location,
                 RuntimeWarning)
         elif not sshserver:
             # otherwise sync with cfg
             sshserver = cfg['ssh']
 
         self._config = cfg
 
         self._ssh = bool(sshserver or sshkey or password)
         if self._ssh and sshserver is None:
             # default to ssh via localhost
             sshserver = url.split('://')[1].split(':')[0]
         if self._ssh and password is None:
             if tunnel.try_passwordless_ssh(sshserver, sshkey, paramiko):
                 password=False
             else:
                 password = getpass("SSH Password for %s: "%sshserver)
         ssh_kwargs = dict(keyfile=sshkey, password=password, paramiko=paramiko)
 
         # configure and construct the session
         if exec_key is not None:
             if os.path.isfile(exec_key):
                 extra_args['keyfile'] = exec_key
             else:
                 exec_key = cast_bytes(exec_key)
                 extra_args['key'] = exec_key
         self.session = Session(**extra_args)
 
         self._query_socket = self._context.socket(zmq.DEALER)
         self._query_socket.setsockopt(zmq.IDENTITY, self.session.bsession)
         if self._ssh:
             tunnel.tunnel_connection(self._query_socket, url, sshserver, **ssh_kwargs)
         else:
             self._query_socket.connect(url)
 
         self.session.debug = self.debug
 
         self._notification_handlers = {'registration_notification' : self._register_engine,
                                     'unregistration_notification' : self._unregister_engine,
                                     'shutdown_notification' : lambda msg: self.close(),
                                     }
         self._queue_handlers = {'execute_reply' : self._handle_execute_reply,
                                 'apply_reply' : self._handle_apply_reply}
         self._connect(sshserver, ssh_kwargs, timeout)
 
     def __del__(self):
         """cleanup sockets, but _not_ context."""
         self.close()
 
     def _setup_profile_dir(self, profile, profile_dir, ipython_dir):
         if ipython_dir is None:
             ipython_dir = get_ipython_dir()
         if profile_dir is not None:
             try:
                 self._cd = ProfileDir.find_profile_dir(profile_dir)
                 return
             except ProfileDirError:
                 pass
         elif profile is not None:
             try:
                 self._cd = ProfileDir.find_profile_dir_by_name(
                     ipython_dir, profile)
                 return
             except ProfileDirError:
                 pass
         self._cd = None
 
     def _update_engines(self, engines):
         """Update our engines dict and _ids from a dict of the form: {id:uuid}."""
         for k,v in engines.iteritems():
             eid = int(k)
             self._engines[eid] = v
             self._ids.append(eid)
         self._ids = sorted(self._ids)
         if sorted(self._engines.keys()) != range(len(self._engines)) and \
                         self._task_scheme == 'pure' and self._task_socket:
             self._stop_scheduling_tasks()
 
     def _stop_scheduling_tasks(self):
         """Stop scheduling tasks because an engine has been unregistered
         from a pure ZMQ scheduler.
         """
         self._task_socket.close()
         self._task_socket = None
         msg = "An engine has been unregistered, and we are using pure " +\
               "ZMQ task scheduling.  Task farming will be disabled."
         if self.outstanding:
             msg += " If you were running tasks when this happened, " +\
                    "some `outstanding` msg_ids may never resolve."
         warnings.warn(msg, RuntimeWarning)
 
     def _build_targets(self, targets):
         """Turn valid target IDs or 'all' into two lists:
         (int_ids, uuids).
         """
         if not self._ids:
             # flush notification socket if no engines yet, just in case
             if not self.ids:
                 raise error.NoEnginesRegistered("Can't build targets without any engines")
 
         if targets is None:
             targets = self._ids
         elif isinstance(targets, basestring):
             if targets.lower() == 'all':
                 targets = self._ids
             else:
                 raise TypeError("%r not valid str target, must be 'all'"%(targets))
         elif isinstance(targets, int):
             if targets < 0:
                 targets = self.ids[targets]
             if targets not in self._ids:
                 raise IndexError("No such engine: %i"%targets)
             targets = [targets]
 
         if isinstance(targets, slice):
             indices = range(len(self._ids))[targets]
             ids = self.ids
             targets = [ ids[i] for i in indices ]
 
         if not isinstance(targets, (tuple, list, xrange)):
             raise TypeError("targets by int/slice/collection of ints only, not %s"%(type(targets)))
 
         return [cast_bytes(self._engines[t]) for t in targets], list(targets)
 
     def _connect(self, sshserver, ssh_kwargs, timeout):
         """setup all our socket connections to the cluster. This is called from
         __init__."""
 
         # Maybe allow reconnecting?
         if self._connected:
             return
         self._connected=True
 
         def connect_socket(s, url):
             url = util.disambiguate_url(url, self._config['location'])
             if self._ssh:
                 return tunnel.tunnel_connection(s, url, sshserver, **ssh_kwargs)
             else:
                 return s.connect(url)
 
         self.session.send(self._query_socket, 'connection_request')
         # use Poller because zmq.select has wrong units in pyzmq 2.1.7
         poller = zmq.Poller()
         poller.register(self._query_socket, zmq.POLLIN)
         # poll expects milliseconds, timeout is seconds
         evts = poller.poll(timeout*1000)
         if not evts:
             raise error.TimeoutError("Hub connection request timed out")
         idents,msg = self.session.recv(self._query_socket,mode=0)
         if self.debug:
             pprint(msg)
         msg = Message(msg)
         content = msg.content
         self._config['registration'] = dict(content)
         if content.status == 'ok':
             ident = self.session.bsession
             if content.mux:
                 self._mux_socket = self._context.socket(zmq.DEALER)
                 self._mux_socket.setsockopt(zmq.IDENTITY, ident)
                 connect_socket(self._mux_socket, content.mux)
             if content.task:
                 self._task_scheme, task_addr = content.task
                 self._task_socket = self._context.socket(zmq.DEALER)
                 self._task_socket.setsockopt(zmq.IDENTITY, ident)
                 connect_socket(self._task_socket, task_addr)
             if content.notification:
                 self._notification_socket = self._context.socket(zmq.SUB)
                 connect_socket(self._notification_socket, content.notification)
                 self._notification_socket.setsockopt(zmq.SUBSCRIBE, b'')
             # if content.query:
             #     self._query_socket = self._context.socket(zmq.DEALER)
             #     self._query_socket.setsockopt(zmq.IDENTITY, self.session.bsession)
             #     connect_socket(self._query_socket, content.query)
             if content.control:
                 self._control_socket = self._context.socket(zmq.DEALER)
                 self._control_socket.setsockopt(zmq.IDENTITY, ident)
                 connect_socket(self._control_socket, content.control)
             if content.iopub:
                 self._iopub_socket = self._context.socket(zmq.SUB)
                 self._iopub_socket.setsockopt(zmq.SUBSCRIBE, b'')
                 self._iopub_socket.setsockopt(zmq.IDENTITY, ident)
                 connect_socket(self._iopub_socket, content.iopub)
             self._update_engines(dict(content.engines))
         else:
             self._connected = False
             raise Exception("Failed to connect!")
 
     #--------------------------------------------------------------------------
     # handlers and callbacks for incoming messages
     #--------------------------------------------------------------------------
 
     def _unwrap_exception(self, content):
         """unwrap exception, and remap engine_id to int."""
         e = error.unwrap_exception(content)
         # print e.traceback
         if e.engine_info:
             e_uuid = e.engine_info['engine_uuid']
             eid = self._engines[e_uuid]
             e.engine_info['engine_id'] = eid
         return e
 
     def _extract_metadata(self, header, parent, content):
         md = {'msg_id' : parent['msg_id'],
               'received' : datetime.now(),
               'engine_uuid' : header.get('engine', None),
               'follow' : parent.get('follow', []),
               'after' : parent.get('after', []),
               'status' : content['status'],
             }
 
         if md['engine_uuid'] is not None:
             md['engine_id'] = self._engines.get(md['engine_uuid'], None)
 
         if 'date' in parent:
             md['submitted'] = parent['date']
         if 'started' in header:
             md['started'] = header['started']
         if 'date' in header:
             md['completed'] = header['date']
         return md
 
     def _register_engine(self, msg):
         """Register a new engine, and update our connection info."""
         content = msg['content']
         eid = content['id']
         d = {eid : content['queue']}
         self._update_engines(d)
 
     def _unregister_engine(self, msg):
         """Unregister an engine that has died."""
         content = msg['content']
         eid = int(content['id'])
         if eid in self._ids:
             self._ids.remove(eid)
             uuid = self._engines.pop(eid)
 
             self._handle_stranded_msgs(eid, uuid)
 
         if self._task_socket and self._task_scheme == 'pure':
             self._stop_scheduling_tasks()
 
     def _handle_stranded_msgs(self, eid, uuid):
         """Handle messages known to be on an engine when the engine unregisters.
 
         It is possible that this will fire prematurely - that is, an engine will
         go down after completing a result, and the client will be notified
         of the unregistration and later receive the successful result.
         """
 
         outstanding = self._outstanding_dict[uuid]
 
         for msg_id in list(outstanding):
             if msg_id in self.results:
                 # we already
                 continue
             try:
                 raise error.EngineError("Engine %r died while running task %r"%(eid, msg_id))
             except:
                 content = error.wrap_exception()
             # build a fake message:
             parent = {}
             header = {}
             parent['msg_id'] = msg_id
             header['engine'] = uuid
             header['date'] = datetime.now()
             msg = dict(parent_header=parent, header=header, content=content)
             self._handle_apply_reply(msg)
 
     def _handle_execute_reply(self, msg):
         """Save the reply to an execute_request into our results.
 
         execute messages are never actually used. apply is used instead.
         """
 
         parent = msg['parent_header']
         msg_id = parent['msg_id']
         if msg_id not in self.outstanding:
             if msg_id in self.history:
                 print ("got stale result: %s"%msg_id)
             else:
                 print ("got unknown result: %s"%msg_id)
         else:
             self.outstanding.remove(msg_id)
 
         content = msg['content']
         header = msg['header']
 
         # construct metadata:
         md = self.metadata[msg_id]
         md.update(self._extract_metadata(header, parent, content))
         # is this redundant?
         self.metadata[msg_id] = md
         
         e_outstanding = self._outstanding_dict[md['engine_uuid']]
         if msg_id in e_outstanding:
             e_outstanding.remove(msg_id)
 
         # construct result:
         if content['status'] == 'ok':
             self.results[msg_id] = ExecuteReply(msg_id, content, md)
         elif content['status'] == 'aborted':
             self.results[msg_id] = error.TaskAborted(msg_id)
         elif content['status'] == 'resubmitted':
             # TODO: handle resubmission
             pass
         else:
             self.results[msg_id] = self._unwrap_exception(content)
 
     def _handle_apply_reply(self, msg):
         """Save the reply to an apply_request into our results."""
         parent = msg['parent_header']
         msg_id = parent['msg_id']
         if msg_id not in self.outstanding:
             if msg_id in self.history:
                 print ("got stale result: %s"%msg_id)
                 print self.results[msg_id]
                 print msg
             else:
                 print ("got unknown result: %s"%msg_id)
         else:
             self.outstanding.remove(msg_id)
         content = msg['content']
         header = msg['header']
 
         # construct metadata:
         md = self.metadata[msg_id]
         md.update(self._extract_metadata(header, parent, content))
         # is this redundant?
         self.metadata[msg_id] = md
 
         e_outstanding = self._outstanding_dict[md['engine_uuid']]
         if msg_id in e_outstanding:
             e_outstanding.remove(msg_id)
 
         # construct result:
         if content['status'] == 'ok':
             self.results[msg_id] = util.unserialize_object(msg['buffers'])[0]
         elif content['status'] == 'aborted':
             self.results[msg_id] = error.TaskAborted(msg_id)
         elif content['status'] == 'resubmitted':
             # TODO: handle resubmission
             pass
         else:
             self.results[msg_id] = self._unwrap_exception(content)
 
     def _flush_notifications(self):
         """Flush notifications of engine registrations waiting
         in ZMQ queue."""
         idents,msg = self.session.recv(self._notification_socket, mode=zmq.NOBLOCK)
         while msg is not None:
             if self.debug:
                 pprint(msg)
             msg_type = msg['header']['msg_type']
             handler = self._notification_handlers.get(msg_type, None)
             if handler is None:
                 raise Exception("Unhandled message type: %s"%msg.msg_type)
             else:
                 handler(msg)
             idents,msg = self.session.recv(self._notification_socket, mode=zmq.NOBLOCK)
 
     def _flush_results(self, sock):
         """Flush task or queue results waiting in ZMQ queue."""
         idents,msg = self.session.recv(sock, mode=zmq.NOBLOCK)
         while msg is not None:
             if self.debug:
                 pprint(msg)
             msg_type = msg['header']['msg_type']
             handler = self._queue_handlers.get(msg_type, None)
             if handler is None:
                 raise Exception("Unhandled message type: %s"%msg.msg_type)
             else:
                 handler(msg)
             idents,msg = self.session.recv(sock, mode=zmq.NOBLOCK)
 
     def _flush_control(self, sock):
         """Flush replies from the control channel waiting
         in the ZMQ queue.
 
         Currently: ignore them."""
         if self._ignored_control_replies <= 0:
             return
         idents,msg = self.session.recv(sock, mode=zmq.NOBLOCK)
         while msg is not None:
             self._ignored_control_replies -= 1
             if self.debug:
                 pprint(msg)
             idents,msg = self.session.recv(sock, mode=zmq.NOBLOCK)
 
     def _flush_ignored_control(self):
         """flush ignored control replies"""
         while self._ignored_control_replies > 0:
             self.session.recv(self._control_socket)
             self._ignored_control_replies -= 1
 
     def _flush_ignored_hub_replies(self):
         ident,msg = self.session.recv(self._query_socket, mode=zmq.NOBLOCK)
         while msg is not None:
             ident,msg = self.session.recv(self._query_socket, mode=zmq.NOBLOCK)
 
     def _flush_iopub(self, sock):
         """Flush replies from the iopub channel waiting
         in the ZMQ queue.
         """
         idents,msg = self.session.recv(sock, mode=zmq.NOBLOCK)
         while msg is not None:
             if self.debug:
                 pprint(msg)
             parent = msg['parent_header']
             # ignore IOPub messages with no parent.
             # Caused by print statements or warnings from before the first execution.
             if not parent:
                 continue
             msg_id = parent['msg_id']
             content = msg['content']
             header = msg['header']
             msg_type = msg['header']['msg_type']
 
             # init metadata:
             md = self.metadata[msg_id]
 
             if msg_type == 'stream':
                 name = content['name']
                 s = md[name] or ''
                 md[name] = s + content['data']
             elif msg_type == 'pyerr':
                 md.update({'pyerr' : self._unwrap_exception(content)})
             elif msg_type == 'pyin':
                 md.update({'pyin' : content['code']})
             elif msg_type == 'display_data':
                 md['outputs'].append(content)
             elif msg_type == 'pyout':
                 md['pyout'] = content
             else:
                 # unhandled msg_type (status, etc.)
                 pass
 
             # reduntant?
             self.metadata[msg_id] = md
 
             idents,msg = self.session.recv(sock, mode=zmq.NOBLOCK)
 
     #--------------------------------------------------------------------------
     # len, getitem
     #--------------------------------------------------------------------------
 
     def __len__(self):
         """len(client) returns # of engines."""
         return len(self.ids)
 
     def __getitem__(self, key):
         """index access returns DirectView multiplexer objects
 
         Must be int, slice, or list/tuple/xrange of ints"""
         if not isinstance(key, (int, slice, tuple, list, xrange)):
             raise TypeError("key by int/slice/iterable of ints only, not %s"%(type(key)))
         else:
             return self.direct_view(key)
 
     #--------------------------------------------------------------------------
     # Begin public methods
     #--------------------------------------------------------------------------
 
     @property
     def ids(self):
         """Always up-to-date ids property."""
         self._flush_notifications()
         # always copy:
         return list(self._ids)
 
     def close(self):
         if self._closed:
             return
         self.stop_spin_thread()
         snames = filter(lambda n: n.endswith('socket'), dir(self))
         for socket in map(lambda name: getattr(self, name), snames):
             if isinstance(socket, zmq.Socket) and not socket.closed:
                 socket.close()
         self._closed = True
 
     def _spin_every(self, interval=1):
         """target func for use in spin_thread"""
         while True:
             if self._stop_spinning.is_set():
                 return
             time.sleep(interval)
             self.spin()
 
     def spin_thread(self, interval=1):
         """call Client.spin() in a background thread on some regular interval
         
         This helps ensure that messages don't pile up too much in the zmq queue
         while you are working on other things, or just leaving an idle terminal.
         
         It also helps limit potential padding of the `received` timestamp
         on AsyncResult objects, used for timings.
         
         Parameters
         ----------
         
         interval : float, optional
             The interval on which to spin the client in the background thread
             (simply passed to time.sleep).
         
         Notes
         -----
         
         For precision timing, you may want to use this method to put a bound
         on the jitter (in seconds) in `received` timestamps used
         in AsyncResult.wall_time.
         
         """
         if self._spin_thread is not None:
             self.stop_spin_thread()
         self._stop_spinning.clear()
         self._spin_thread = Thread(target=self._spin_every, args=(interval,))
         self._spin_thread.daemon = True
         self._spin_thread.start()
     
     def stop_spin_thread(self):
         """stop background spin_thread, if any"""
         if self._spin_thread is not None:
             self._stop_spinning.set()
             self._spin_thread.join()
             self._spin_thread = None
 
     def spin(self):
         """Flush any registration notifications and execution results
         waiting in the ZMQ queue.
         """
         if self._notification_socket:
             self._flush_notifications()
         if self._iopub_socket:
             self._flush_iopub(self._iopub_socket)
         if self._mux_socket:
             self._flush_results(self._mux_socket)
         if self._task_socket:
             self._flush_results(self._task_socket)
         if self._control_socket:
             self._flush_control(self._control_socket)
         if self._query_socket:
             self._flush_ignored_hub_replies()
 
     def wait(self, jobs=None, timeout=-1):
         """waits on one or more `jobs`, for up to `timeout` seconds.
 
         Parameters
         ----------
 
         jobs : int, str, or list of ints and/or strs, or one or more AsyncResult objects
                 ints are indices to self.history
                 strs are msg_ids
                 default: wait on all outstanding messages
         timeout : float
                 a time in seconds, after which to give up.
                 default is -1, which means no timeout
 
         Returns
         -------
 
         True : when all msg_ids are done
         False : timeout reached, some msg_ids still outstanding
         """
         tic = time.time()
         if jobs is None:
             theids = self.outstanding
         else:
             if isinstance(jobs, (int, basestring, AsyncResult)):
                 jobs = [jobs]
             theids = set()
             for job in jobs:
                 if isinstance(job, int):
                     # index access
                     job = self.history[job]
                 elif isinstance(job, AsyncResult):
                     map(theids.add, job.msg_ids)
                     continue
                 theids.add(job)
         if not theids.intersection(self.outstanding):
             return True
         self.spin()
         while theids.intersection(self.outstanding):
             if timeout >= 0 and ( time.time()-tic ) > timeout:
                 break
             time.sleep(1e-3)
             self.spin()
         return len(theids.intersection(self.outstanding)) == 0
 
     #--------------------------------------------------------------------------
     # Control methods
     #--------------------------------------------------------------------------
 
     @spin_first
     def clear(self, targets=None, block=None):
         """Clear the namespace in target(s)."""
         block = self.block if block is None else block
         targets = self._build_targets(targets)[0]
         for t in targets:
             self.session.send(self._control_socket, 'clear_request', content={}, ident=t)
         error = False
         if block:
             self._flush_ignored_control()
             for i in range(len(targets)):
                 idents,msg = self.session.recv(self._control_socket,0)
                 if self.debug:
                     pprint(msg)
                 if msg['content']['status'] != 'ok':
                     error = self._unwrap_exception(msg['content'])
         else:
             self._ignored_control_replies += len(targets)
         if error:
             raise error
 
 
     @spin_first
     def abort(self, jobs=None, targets=None, block=None):
         """Abort specific jobs from the execution queues of target(s).
 
         This is a mechanism to prevent jobs that have already been submitted
         from executing.
 
         Parameters
         ----------
 
         jobs : msg_id, list of msg_ids, or AsyncResult
             The jobs to be aborted
             
             If unspecified/None: abort all outstanding jobs.
 
         """
         block = self.block if block is None else block
         jobs = jobs if jobs is not None else list(self.outstanding)
         targets = self._build_targets(targets)[0]
         
         msg_ids = []
         if isinstance(jobs, (basestring,AsyncResult)):
             jobs = [jobs]
         bad_ids = filter(lambda obj: not isinstance(obj, (basestring, AsyncResult)), jobs)
         if bad_ids:
             raise TypeError("Invalid msg_id type %r, expected str or AsyncResult"%bad_ids[0])
         for j in jobs:
             if isinstance(j, AsyncResult):
                 msg_ids.extend(j.msg_ids)
             else:
                 msg_ids.append(j)
         content = dict(msg_ids=msg_ids)
         for t in targets:
             self.session.send(self._control_socket, 'abort_request',
                     content=content, ident=t)
         error = False
         if block:
             self._flush_ignored_control()
             for i in range(len(targets)):
                 idents,msg = self.session.recv(self._control_socket,0)
                 if self.debug:
                     pprint(msg)
                 if msg['content']['status'] != 'ok':
                     error = self._unwrap_exception(msg['content'])
         else:
             self._ignored_control_replies += len(targets)
         if error:
             raise error
 
     @spin_first
     def shutdown(self, targets=None, restart=False, hub=False, block=None):
         """Terminates one or more engine processes, optionally including the hub."""
         block = self.block if block is None else block
         if hub:
             targets = 'all'
         targets = self._build_targets(targets)[0]
         for t in targets:
             self.session.send(self._control_socket, 'shutdown_request',
                         content={'restart':restart},ident=t)
         error = False
         if block or hub:
             self._flush_ignored_control()
             for i in range(len(targets)):
                 idents,msg = self.session.recv(self._control_socket, 0)
                 if self.debug:
                     pprint(msg)
                 if msg['content']['status'] != 'ok':
                     error = self._unwrap_exception(msg['content'])
         else:
             self._ignored_control_replies += len(targets)
 
         if hub:
             time.sleep(0.25)
             self.session.send(self._query_socket, 'shutdown_request')
             idents,msg = self.session.recv(self._query_socket, 0)
             if self.debug:
                 pprint(msg)
             if msg['content']['status'] != 'ok':
                 error = self._unwrap_exception(msg['content'])
 
         if error:
             raise error
 
     #--------------------------------------------------------------------------
     # Execution related methods
     #--------------------------------------------------------------------------
 
     def _maybe_raise(self, result):
         """wrapper for maybe raising an exception if apply failed."""
         if isinstance(result, error.RemoteError):
             raise result
 
         return result
 
     def send_apply_request(self, socket, f, args=None, kwargs=None, subheader=None, track=False,
                             ident=None):
         """construct and send an apply message via a socket.
 
         This is the principal method with which all engine execution is performed by views.
         """
 
         if self._closed:
             raise RuntimeError("Client cannot be used after its sockets have been closed")
         
         # defaults:
         args = args if args is not None else []
         kwargs = kwargs if kwargs is not None else {}
         subheader = subheader if subheader is not None else {}
 
         # validate arguments
         if not callable(f) and not isinstance(f, Reference):
             raise TypeError("f must be callable, not %s"%type(f))
         if not isinstance(args, (tuple, list)):
             raise TypeError("args must be tuple or list, not %s"%type(args))
         if not isinstance(kwargs, dict):
             raise TypeError("kwargs must be dict, not %s"%type(kwargs))
         if not isinstance(subheader, dict):
             raise TypeError("subheader must be dict, not %s"%type(subheader))
 
         bufs = util.pack_apply_message(f,args,kwargs)
 
         msg = self.session.send(socket, "apply_request", buffers=bufs, ident=ident,
                             subheader=subheader, track=track)
 
         msg_id = msg['header']['msg_id']
         self.outstanding.add(msg_id)
         if ident:
             # possibly routed to a specific engine
             if isinstance(ident, list):
                 ident = ident[-1]
             if ident in self._engines.values():
                 # save for later, in case of engine death
                 self._outstanding_dict[ident].add(msg_id)
         self.history.append(msg_id)
         self.metadata[msg_id]['submitted'] = datetime.now()
 
         return msg
 
     def send_execute_request(self, socket, code, silent=True, subheader=None, ident=None):
         """construct and send an execute request via a socket.
 
         """
 
         if self._closed:
             raise RuntimeError("Client cannot be used after its sockets have been closed")
         
         # defaults:
         subheader = subheader if subheader is not None else {}
 
         # validate arguments
         if not isinstance(code, basestring):
             raise TypeError("code must be text, not %s" % type(code))
         if not isinstance(subheader, dict):
             raise TypeError("subheader must be dict, not %s" % type(subheader))
         
         content = dict(code=code, silent=bool(silent), user_variables=[], user_expressions={})
 
 
         msg = self.session.send(socket, "execute_request", content=content, ident=ident,
                             subheader=subheader)
 
         msg_id = msg['header']['msg_id']
         self.outstanding.add(msg_id)
         if ident:
             # possibly routed to a specific engine
             if isinstance(ident, list):
                 ident = ident[-1]
             if ident in self._engines.values():
                 # save for later, in case of engine death
                 self._outstanding_dict[ident].add(msg_id)
         self.history.append(msg_id)
         self.metadata[msg_id]['submitted'] = datetime.now()
 
         return msg
 
     #--------------------------------------------------------------------------
     # construct a View object
     #--------------------------------------------------------------------------
 
     def load_balanced_view(self, targets=None):
         """construct a DirectView object.
 
         If no arguments are specified, create a LoadBalancedView
         using all engines.
 
         Parameters
         ----------
 
         targets: list,slice,int,etc. [default: use all engines]
             The subset of engines across which to load-balance
         """
         if targets == 'all':
             targets = None
         if targets is not None:
             targets = self._build_targets(targets)[1]
         return LoadBalancedView(client=self, socket=self._task_socket, targets=targets)
 
     def direct_view(self, targets='all'):
         """construct a DirectView object.
 
         If no targets are specified, create a DirectView using all engines.
         
         rc.direct_view('all') is distinguished from rc[:] in that 'all' will
         evaluate the target engines at each execution, whereas rc[:] will connect to
         all *current* engines, and that list will not change.
         
         That is, 'all' will always use all engines, whereas rc[:] will not use
         engines added after the DirectView is constructed.
 
         Parameters
         ----------
 
         targets: list,slice,int,etc. [default: use all engines]
             The engines to use for the View
         """
         single = isinstance(targets, int)
         # allow 'all' to be lazily evaluated at each execution
         if targets != 'all':
             targets = self._build_targets(targets)[1]
         if single:
             targets = targets[0]
         return DirectView(client=self, socket=self._mux_socket, targets=targets)
 
     #--------------------------------------------------------------------------
     # Query methods
     #--------------------------------------------------------------------------
 
     @spin_first
     def get_result(self, indices_or_msg_ids=None, block=None):
         """Retrieve a result by msg_id or history index, wrapped in an AsyncResult object.
 
         If the client already has the results, no request to the Hub will be made.
 
         This is a convenient way to construct AsyncResult objects, which are wrappers
         that include metadata about execution, and allow for awaiting results that
         were not submitted by this Client.
 
         It can also be a convenient way to retrieve the metadata associated with
         blocking execution, since it always retrieves
 
         Examples
         --------
         ::
 
             In [10]: r = client.apply()
 
         Parameters
         ----------
 
         indices_or_msg_ids : integer history index, str msg_id, or list of either
             The indices or msg_ids of indices to be retrieved
 
         block : bool
             Whether to wait for the result to be done
 
         Returns
         -------
 
         AsyncResult
             A single AsyncResult object will always be returned.
 
         AsyncHubResult
             A subclass of AsyncResult that retrieves results from the Hub
 
         """
         block = self.block if block is None else block
         if indices_or_msg_ids is None:
             indices_or_msg_ids = -1
 
         if not isinstance(indices_or_msg_ids, (list,tuple)):
             indices_or_msg_ids = [indices_or_msg_ids]
 
         theids = []
         for id in indices_or_msg_ids:
             if isinstance(id, int):
                 id = self.history[id]
             if not isinstance(id, basestring):
                 raise TypeError("indices must be str or int, not %r"%id)
             theids.append(id)
 
         local_ids = filter(lambda msg_id: msg_id in self.history or msg_id in self.results, theids)
         remote_ids = filter(lambda msg_id: msg_id not in local_ids, theids)
 
         if remote_ids:
             ar = AsyncHubResult(self, msg_ids=theids)
         else:
             ar = AsyncResult(self, msg_ids=theids)
 
         if block:
             ar.wait()
 
         return ar
 
     @spin_first
     def resubmit(self, indices_or_msg_ids=None, subheader=None, block=None):
         """Resubmit one or more tasks.
 
         in-flight tasks may not be resubmitted.
 
         Parameters
         ----------
 
         indices_or_msg_ids : integer history index, str msg_id, or list of either
             The indices or msg_ids of indices to be retrieved
 
         block : bool
             Whether to wait for the result to be done
 
         Returns
         -------
 
         AsyncHubResult
             A subclass of AsyncResult that retrieves results from the Hub
 
         """
         block = self.block if block is None else block
         if indices_or_msg_ids is None:
             indices_or_msg_ids = -1
 
         if not isinstance(indices_or_msg_ids, (list,tuple)):
             indices_or_msg_ids = [indices_or_msg_ids]
 
         theids = []
         for id in indices_or_msg_ids:
             if isinstance(id, int):
                 id = self.history[id]
             if not isinstance(id, basestring):
                 raise TypeError("indices must be str or int, not %r"%id)
             theids.append(id)
 
         content = dict(msg_ids = theids)
 
         self.session.send(self._query_socket, 'resubmit_request', content)
 
         zmq.select([self._query_socket], [], [])
         idents,msg = self.session.recv(self._query_socket, zmq.NOBLOCK)
         if self.debug:
             pprint(msg)
         content = msg['content']
         if content['status'] != 'ok':
             raise self._unwrap_exception(content)
         mapping = content['resubmitted']
         new_ids = [ mapping[msg_id] for msg_id in theids ]
 
         ar = AsyncHubResult(self, msg_ids=new_ids)
 
         if block:
             ar.wait()
 
         return ar
 
     @spin_first
     def result_status(self, msg_ids, status_only=True):
         """Check on the status of the result(s) of the apply request with `msg_ids`.
 
         If status_only is False, then the actual results will be retrieved, else
         only the status of the results will be checked.
 
         Parameters
         ----------
 
         msg_ids : list of msg_ids
             if int:
                 Passed as index to self.history for convenience.
         status_only : bool (default: True)
             if False:
                 Retrieve the actual results of completed tasks.
 
         Returns
         -------
 
         results : dict
             There will always be the keys 'pending' and 'completed', which will
             be lists of msg_ids that are incomplete or complete. If `status_only`
             is False, then completed results will be keyed by their `msg_id`.
         """
         if not isinstance(msg_ids, (list,tuple)):
             msg_ids = [msg_ids]
 
         theids = []
         for msg_id in msg_ids:
             if isinstance(msg_id, int):
                 msg_id = self.history[msg_id]
             if not isinstance(msg_id, basestring):
                 raise TypeError("msg_ids must be str, not %r"%msg_id)
             theids.append(msg_id)
 
         completed = []
         local_results = {}
 
         # comment this block out to temporarily disable local shortcut:
         for msg_id in theids:
             if msg_id in self.results:
                 completed.append(msg_id)
                 local_results[msg_id] = self.results[msg_id]
                 theids.remove(msg_id)
 
         if theids: # some not locally cached
             content = dict(msg_ids=theids, status_only=status_only)
             msg = self.session.send(self._query_socket, "result_request", content=content)
             zmq.select([self._query_socket], [], [])
             idents,msg = self.session.recv(self._query_socket, zmq.NOBLOCK)
             if self.debug:
                 pprint(msg)
             content = msg['content']
             if content['status'] != 'ok':
                 raise self._unwrap_exception(content)
             buffers = msg['buffers']
         else:
             content = dict(completed=[],pending=[])
 
         content['completed'].extend(completed)
 
         if status_only:
             return content
 
         failures = []
         # load cached results into result:
         content.update(local_results)
 
         # update cache with results:
         for msg_id in sorted(theids):
             if msg_id in content['completed']:
                 rec = content[msg_id]
                 parent = rec['header']
                 header = rec['result_header']
                 rcontent = rec['result_content']
                 iodict = rec['io']
                 if isinstance(rcontent, str):
                     rcontent = self.session.unpack(rcontent)
 
                 md = self.metadata[msg_id]
                 md.update(self._extract_metadata(header, parent, rcontent))
                 if rec.get('received'):
                     md['received'] = rec['received']
                 md.update(iodict)
 
                 if rcontent['status'] == 'ok':
                     res,buffers = util.unserialize_object(buffers)
                 else:
                     print rcontent
                     res = self._unwrap_exception(rcontent)
                     failures.append(res)
 
                 self.results[msg_id] = res
                 content[msg_id] = res
 
         if len(theids) == 1 and failures:
                 raise failures[0]
 
         error.collect_exceptions(failures, "result_status")
         return content
 
     @spin_first
     def queue_status(self, targets='all', verbose=False):
         """Fetch the status of engine queues.
 
         Parameters
         ----------
 
         targets : int/str/list of ints/strs
                 the engines whose states are to be queried.
                 default : all
         verbose : bool
                 Whether to return lengths only, or lists of ids for each element
         """
         if targets == 'all':
             # allow 'all' to be evaluated on the engine
             engine_ids = None
         else:
             engine_ids = self._build_targets(targets)[1]
         content = dict(targets=engine_ids, verbose=verbose)
         self.session.send(self._query_socket, "queue_request", content=content)
         idents,msg = self.session.recv(self._query_socket, 0)
         if self.debug:
             pprint(msg)
         content = msg['content']
         status = content.pop('status')
         if status != 'ok':
             raise self._unwrap_exception(content)
         content = rekey(content)
         if isinstance(targets, int):
             return content[targets]
         else:
             return content
 
     @spin_first
     def purge_results(self, jobs=[], targets=[]):
         """Tell the Hub to forget results.
 
         Individual results can be purged by msg_id, or the entire
         history of specific targets can be purged.
 
         Use `purge_results('all')` to scrub everything from the Hub's db.
 
         Parameters
         ----------
 
         jobs : str or list of str or AsyncResult objects
                 the msg_ids whose results should be forgotten.
         targets : int/str/list of ints/strs
                 The targets, by int_id, whose entire history is to be purged.
 
                 default : None
         """
         if not targets and not jobs:
             raise ValueError("Must specify at least one of `targets` and `jobs`")
         if targets:
             targets = self._build_targets(targets)[1]
 
         # construct msg_ids from jobs
         if jobs == 'all':
             msg_ids = jobs
         else:
             msg_ids = []
             if isinstance(jobs, (basestring,AsyncResult)):
                 jobs = [jobs]
             bad_ids = filter(lambda obj: not isinstance(obj, (basestring, AsyncResult)), jobs)
             if bad_ids:
                 raise TypeError("Invalid msg_id type %r, expected str or AsyncResult"%bad_ids[0])
             for j in jobs:
                 if isinstance(j, AsyncResult):
                     msg_ids.extend(j.msg_ids)
                 else:
                     msg_ids.append(j)
 
         content = dict(engine_ids=targets, msg_ids=msg_ids)
         self.session.send(self._query_socket, "purge_request", content=content)
         idents, msg = self.session.recv(self._query_socket, 0)
         if self.debug:
             pprint(msg)
         content = msg['content']
         if content['status'] != 'ok':
             raise self._unwrap_exception(content)
 
     @spin_first
     def hub_history(self):
         """Get the Hub's history
 
         Just like the Client, the Hub has a history, which is a list of msg_ids.
         This will contain the history of all clients, and, depending on configuration,
         may contain history across multiple cluster sessions.
 
         Any msg_id returned here is a valid argument to `get_result`.
 
         Returns
         -------
 
         msg_ids : list of strs
                 list of all msg_ids, ordered by task submission time.
         """
 
         self.session.send(self._query_socket, "history_request", content={})
         idents, msg = self.session.recv(self._query_socket, 0)
 
         if self.debug:
             pprint(msg)
         content = msg['content']
         if content['status'] != 'ok':
             raise self._unwrap_exception(content)
         else:
             return content['history']
 
     @spin_first
     def db_query(self, query, keys=None):
         """Query the Hub's TaskRecord database
 
         This will return a list of task record dicts that match `query`
 
         Parameters
         ----------
 
         query : mongodb query dict
             The search dict. See mongodb query docs for details.
         keys : list of strs [optional]
             The subset of keys to be returned.  The default is to fetch everything but buffers.
             'msg_id' will *always* be included.
         """
         if isinstance(keys, basestring):
             keys = [keys]
         content = dict(query=query, keys=keys)
         self.session.send(self._query_socket, "db_request", content=content)
         idents, msg = self.session.recv(self._query_socket, 0)
         if self.debug:
             pprint(msg)
         content = msg['content']
         if content['status'] != 'ok':
             raise self._unwrap_exception(content)
 
         records = content['records']
 
         buffer_lens = content['buffer_lens']
         result_buffer_lens = content['result_buffer_lens']
         buffers = msg['buffers']
         has_bufs = buffer_lens is not None
         has_rbufs = result_buffer_lens is not None
         for i,rec in enumerate(records):
             # relink buffers
             if has_bufs:
                 blen = buffer_lens[i]
                 rec['buffers'], buffers = buffers[:blen],buffers[blen:]
             if has_rbufs:
                 blen = result_buffer_lens[i]
                 rec['result_buffers'], buffers = buffers[:blen],buffers[blen:]
 
         return records
 
 __all__ = [ 'Client' ]

IPython/parallel/tests/clienttest.py

@@ -1,144 +1,213 b''
 """base class for parallel client tests
 
 Authors:
 
 * Min RK
 """
 
 #-------------------------------------------------------------------------------
 #  Copyright (C) 2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-------------------------------------------------------------------------------
 
 import sys
 import tempfile
 import time
+from StringIO import StringIO
 
 from nose import SkipTest
 
 import zmq
 from zmq.tests import BaseZMQTestCase
 
 from IPython.external.decorator import decorator
 
 from IPython.parallel import error
 from IPython.parallel import Client
 
 from IPython.parallel.tests import launchers, add_engines
 
 # simple tasks for use in apply tests
 
 def segfault():
     """this will segfault"""
     import ctypes
     ctypes.memset(-1,0,1)
 
 def crash():
     """from stdlib crashers in the test suite"""
     import types
     if sys.platform.startswith('win'):
         import ctypes
         ctypes.windll.kernel32.SetErrorMode(0x0002);
     args = [ 0, 0, 0, 0, b'\x04\x71\x00\x00', (), (), (), '', '', 1, b'']
     if sys.version_info[0] >= 3:
         # Python3 adds 'kwonlyargcount' as the second argument to Code
         args.insert(1, 0)
         
     co = types.CodeType(*args)
     exec(co)
 
 def wait(n):
     """sleep for a time"""
     import time
     time.sleep(n)
     return n
 
 def raiser(eclass):
     """raise an exception"""
     raise eclass()
 
+def generate_output():
+    """function for testing output
+    
+    publishes two outputs of each type, and returns
+    a rich displayable object.
+    """
+    
+    import sys
+    from IPython.core.display import display, HTML, Math
+    
+    print "stdout"
+    print >> sys.stderr, "stderr"
+    
+    display(HTML("<b>HTML</b>"))
+    
+    print "stdout2"
+    print >> sys.stderr, "stderr2"
+    
+    display(Math(r"\alpha=\beta"))
+    
+    return Math("42")
+
 # test decorator for skipping tests when libraries are unavailable
 def skip_without(*names):
     """skip a test if some names are not importable"""
     @decorator
     def skip_without_names(f, *args, **kwargs):
         """decorator to skip tests in the absence of numpy."""
         for name in names:
             try:
                 __import__(name)
             except ImportError:
                 raise SkipTest
         return f(*args, **kwargs)
     return skip_without_names
 
+#-------------------------------------------------------------------------------
+# Classes
+#-------------------------------------------------------------------------------
+
+class CapturedIO(object):
+    """Simple object for containing captured stdout/err StringIO objects"""
+    
+    def __init__(self, stdout, stderr):
+        self.stdout_io = stdout
+        self.stderr_io = stderr
+    
+    @property
+    def stdout(self):
+        return self.stdout_io.getvalue()
+    
+    @property
+    def stderr(self):
+        return self.stderr_io.getvalue()
+
+
+class capture_output(object):
+    """context manager for capturing stdout/err"""
+    
+    def __enter__(self):
+        self.sys_stdout = sys.stdout
+        self.sys_stderr = sys.stderr
+        stdout = sys.stdout = StringIO()
+        stderr = sys.stderr = StringIO()
+        return CapturedIO(stdout, stderr)
+    
+    def __exit__(self, exc_type, exc_value, traceback):
+        sys.stdout = self.sys_stdout
+        sys.stderr = self.sys_stderr
+
+
 class ClusterTestCase(BaseZMQTestCase):
     
     def add_engines(self, n=1, block=True):
         """add multiple engines to our cluster"""
         self.engines.extend(add_engines(n))
         if block:
             self.wait_on_engines()
 
     def minimum_engines(self, n=1, block=True):
         """add engines until there are at least n connected"""
         self.engines.extend(add_engines(n, total=True))
         if block:
             self.wait_on_engines()
             
     
     def wait_on_engines(self, timeout=5):
         """wait for our engines to connect."""
         n = len(self.engines)+self.base_engine_count
         tic = time.time()
         while time.time()-tic < timeout and len(self.client.ids) < n:
             time.sleep(0.1)
         
         assert not len(self.client.ids) < n, "waiting for engines timed out"
     
     def connect_client(self):
         """connect a client with my Context, and track its sockets for cleanup"""
         c = Client(profile='iptest', context=self.context)
         for name in filter(lambda n:n.endswith('socket'), dir(c)):
             s = getattr(c, name)
             s.setsockopt(zmq.LINGER, 0)
             self.sockets.append(s)
         return c
     
     def assertRaisesRemote(self, etype, f, *args, **kwargs):
         try:
             try:
                 f(*args, **kwargs)
             except error.CompositeError as e:
                 e.raise_exception()
         except error.RemoteError as e:
             self.assertEquals(etype.__name__, e.ename, "Should have raised %r, but raised %r"%(etype.__name__, e.ename))
         else:
             self.fail("should have raised a RemoteError")
             
+    def _wait_for(self, f, timeout=10):
+        """wait for a condition"""
+        tic = time.time()
+        while time.time() <= tic + timeout:
+            if f():
+                return
+            time.sleep(0.1)
+            self.client.spin()
+        if not f():
+            print "Warning: Awaited condition never arrived"
+    
     def setUp(self):
         BaseZMQTestCase.setUp(self)
         self.client = self.connect_client()
         # start every test with clean engine namespaces:
         self.client.clear(block=True)
         self.base_engine_count=len(self.client.ids)
         self.engines=[]
     
     def tearDown(self):
         # self.client.clear(block=True)
         # close fds:
         for e in filter(lambda e: e.poll() is not None, launchers):
             launchers.remove(e)
         
         # allow flushing of incoming messages to prevent crash on socket close
         self.client.wait(timeout=2)
         # time.sleep(2)
         self.client.spin()
         self.client.close()
         BaseZMQTestCase.tearDown(self)
         # this will be redundant when pyzmq merges PR #88
         # self.context.term()
         # print tempfile.TemporaryFile().fileno(),
         # sys.stdout.flush()
         
  No newline at end of file

IPython/parallel/tests/test_view.py

@@ -1,694 +1,572 b''
 # -*- coding: utf-8 -*-
 """test View objects
 
 Authors:
 
 * Min RK
 """
 #-------------------------------------------------------------------------------
 #  Copyright (C) 2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-------------------------------------------------------------------------------
 
 #-------------------------------------------------------------------------------
 # Imports
 #-------------------------------------------------------------------------------
 
 import sys
 import time
 from tempfile import mktemp
 from StringIO import StringIO
 
 import zmq
 from nose import SkipTest
 
 from IPython.testing import decorators as dec
 from IPython.testing.ipunittest import ParametricTestCase
 
 from IPython import parallel  as pmod
 from IPython.parallel import error
 from IPython.parallel import AsyncResult, AsyncHubResult, AsyncMapResult
 from IPython.parallel import DirectView
 from IPython.parallel.util import interactive
 
 from IPython.parallel.tests import add_engines
 
 from .clienttest import ClusterTestCase, crash, wait, skip_without
 
 def setup():
     add_engines(3, total=True)
 
 class TestView(ClusterTestCase, ParametricTestCase):
     
     def test_z_crash_mux(self):
         """test graceful handling of engine death (direct)"""
         raise SkipTest("crash tests disabled, due to undesirable crash reports")
         # self.add_engines(1)
         eid = self.client.ids[-1]
         ar = self.client[eid].apply_async(crash)
         self.assertRaisesRemote(error.EngineError, ar.get, 10)
         eid = ar.engine_id
         tic = time.time()
         while eid in self.client.ids and time.time()-tic < 5:
             time.sleep(.01)
             self.client.spin()
         self.assertFalse(eid in self.client.ids, "Engine should have died")
     
     def test_push_pull(self):
         """test pushing and pulling"""
         data = dict(a=10, b=1.05, c=range(10), d={'e':(1,2),'f':'hi'})
         t = self.client.ids[-1]
         v = self.client[t]
         push = v.push
         pull = v.pull
         v.block=True
         nengines = len(self.client)
         push({'data':data})
         d = pull('data')
         self.assertEquals(d, data)
         self.client[:].push({'data':data})
         d = self.client[:].pull('data', block=True)
         self.assertEquals(d, nengines*[data])
         ar = push({'data':data}, block=False)
         self.assertTrue(isinstance(ar, AsyncResult))
         r = ar.get()
         ar = self.client[:].pull('data', block=False)
         self.assertTrue(isinstance(ar, AsyncResult))
         r = ar.get()
         self.assertEquals(r, nengines*[data])
         self.client[:].push(dict(a=10,b=20))
         r = self.client[:].pull(('a','b'), block=True)
         self.assertEquals(r, nengines*[[10,20]])
     
     def test_push_pull_function(self):
         "test pushing and pulling functions"
         def testf(x):
             return 2.0*x
         
         t = self.client.ids[-1]
         v = self.client[t]
         v.block=True
         push = v.push
         pull = v.pull
         execute = v.execute
         push({'testf':testf})
         r = pull('testf')
         self.assertEqual(r(1.0), testf(1.0))
         execute('r = testf(10)')
         r = pull('r')
         self.assertEquals(r, testf(10))
         ar = self.client[:].push({'testf':testf}, block=False)
         ar.get()
         ar = self.client[:].pull('testf', block=False)
         rlist = ar.get()
         for r in rlist:
             self.assertEqual(r(1.0), testf(1.0))
         execute("def g(x): return x*x")
         r = pull(('testf','g'))
         self.assertEquals((r[0](10),r[1](10)), (testf(10), 100))
     
     def test_push_function_globals(self):
         """test that pushed functions have access to globals"""
         @interactive
         def geta():
             return a
         # self.add_engines(1)
         v = self.client[-1]
         v.block=True
         v['f'] = geta
         self.assertRaisesRemote(NameError, v.execute, 'b=f()')
         v.execute('a=5')
         v.execute('b=f()')
         self.assertEquals(v['b'], 5)
     
     def test_push_function_defaults(self):
         """test that pushed functions preserve default args"""
         def echo(a=10):
             return a
         v = self.client[-1]
         v.block=True
         v['f'] = echo
         v.execute('b=f()')
         self.assertEquals(v['b'], 10)
 
     def test_get_result(self):
         """test getting results from the Hub."""
         c = pmod.Client(profile='iptest')
         # self.add_engines(1)
         t = c.ids[-1]
         v = c[t]
         v2 = self.client[t]
         ar = v.apply_async(wait, 1)
         # give the monitor time to notice the message
         time.sleep(.25)
         ahr = v2.get_result(ar.msg_ids)
         self.assertTrue(isinstance(ahr, AsyncHubResult))
         self.assertEquals(ahr.get(), ar.get())
         ar2 = v2.get_result(ar.msg_ids)
         self.assertFalse(isinstance(ar2, AsyncHubResult))
         c.spin()
         c.close()
     
     def test_run_newline(self):
         """test that run appends newline to files"""
         tmpfile = mktemp()
         with open(tmpfile, 'w') as f:
             f.write("""def g():
                 return 5
                 """)
         v = self.client[-1]
         v.run(tmpfile, block=True)
         self.assertEquals(v.apply_sync(lambda f: f(), pmod.Reference('g')), 5)
 
     def test_apply_tracked(self):
         """test tracking for apply"""
         # self.add_engines(1)
         t = self.client.ids[-1]
         v = self.client[t]
         v.block=False
         def echo(n=1024*1024, **kwargs):
             with v.temp_flags(**kwargs):
                 return v.apply(lambda x: x, 'x'*n)
         ar = echo(1, track=False)
         self.assertTrue(isinstance(ar._tracker, zmq.MessageTracker))
         self.assertTrue(ar.sent)
         ar = echo(track=True)
         self.assertTrue(isinstance(ar._tracker, zmq.MessageTracker))
         self.assertEquals(ar.sent, ar._tracker.done)
         ar._tracker.wait()
         self.assertTrue(ar.sent)
         
     def test_push_tracked(self):
         t = self.client.ids[-1]
         ns = dict(x='x'*1024*1024)
         v = self.client[t]
         ar = v.push(ns, block=False, track=False)
         self.assertTrue(isinstance(ar._tracker, zmq.MessageTracker))
         self.assertTrue(ar.sent)
         
         ar = v.push(ns, block=False, track=True)
         self.assertTrue(isinstance(ar._tracker, zmq.MessageTracker))
         ar._tracker.wait()
         self.assertEquals(ar.sent, ar._tracker.done)
         self.assertTrue(ar.sent)
         ar.get()
         
     def test_scatter_tracked(self):
         t = self.client.ids
         x='x'*1024*1024
         ar = self.client[t].scatter('x', x, block=False, track=False)
         self.assertTrue(isinstance(ar._tracker, zmq.MessageTracker))
         self.assertTrue(ar.sent)
         
         ar = self.client[t].scatter('x', x, block=False, track=True)
         self.assertTrue(isinstance(ar._tracker, zmq.MessageTracker))
         self.assertEquals(ar.sent, ar._tracker.done)
         ar._tracker.wait()
         self.assertTrue(ar.sent)
         ar.get()
     
     def test_remote_reference(self):
         v = self.client[-1]
         v['a'] = 123
         ra = pmod.Reference('a')
         b = v.apply_sync(lambda x: x, ra)
         self.assertEquals(b, 123)
 
 
     def test_scatter_gather(self):
         view = self.client[:]
         seq1 = range(16)
         view.scatter('a', seq1)
         seq2 = view.gather('a', block=True)
         self.assertEquals(seq2, seq1)
         self.assertRaisesRemote(NameError, view.gather, 'asdf', block=True)
     
     @skip_without('numpy')
     def test_scatter_gather_numpy(self):
         import numpy
         from numpy.testing.utils import assert_array_equal, assert_array_almost_equal
         view = self.client[:]
         a = numpy.arange(64)
         view.scatter('a', a)
         b = view.gather('a', block=True)
         assert_array_equal(b, a)
     
     def test_scatter_gather_lazy(self):
         """scatter/gather with targets='all'"""
         view = self.client.direct_view(targets='all')
         x = range(64)
         view.scatter('x', x)
         gathered = view.gather('x', block=True)
         self.assertEquals(gathered, x)
         
 
     @dec.known_failure_py3
     @skip_without('numpy')
     def test_push_numpy_nocopy(self):
         import numpy
         view = self.client[:]
         a = numpy.arange(64)
         view['A'] = a
         @interactive
         def check_writeable(x):
             return x.flags.writeable
         
         for flag in view.apply_sync(check_writeable, pmod.Reference('A')):
             self.assertFalse(flag, "array is writeable, push shouldn't have pickled it")
         
         view.push(dict(B=a))
         for flag in view.apply_sync(check_writeable, pmod.Reference('B')):
             self.assertFalse(flag, "array is writeable, push shouldn't have pickled it")
     
     @skip_without('numpy')
     def test_apply_numpy(self):
         """view.apply(f, ndarray)"""
         import numpy
         from numpy.testing.utils import assert_array_equal, assert_array_almost_equal
         
         A = numpy.random.random((100,100))
         view = self.client[-1]
         for dt in [ 'int32', 'uint8', 'float32', 'float64' ]:
             B = A.astype(dt)
             C = view.apply_sync(lambda x:x, B)
             assert_array_equal(B,C)
     
     def test_map(self):
         view = self.client[:]
         def f(x):
             return x**2
         data = range(16)
         r = view.map_sync(f, data)
         self.assertEquals(r, map(f, data))
     
     def test_map_iterable(self):
         """test map on iterables (direct)"""
         view = self.client[:]
         # 101 is prime, so it won't be evenly distributed
         arr = range(101)
         # ensure it will be an iterator, even in Python 3
         it = iter(arr)
         r = view.map_sync(lambda x:x, arr)
         self.assertEquals(r, list(arr))
     
     def test_scatterGatherNonblocking(self):
         data = range(16)
         view = self.client[:]
         view.scatter('a', data, block=False)
         ar = view.gather('a', block=False)
         self.assertEquals(ar.get(), data)
     
     @skip_without('numpy')
     def test_scatter_gather_numpy_nonblocking(self):
         import numpy
         from numpy.testing.utils import assert_array_equal, assert_array_almost_equal
         a = numpy.arange(64)
         view = self.client[:]
         ar = view.scatter('a', a, block=False)
         self.assertTrue(isinstance(ar, AsyncResult))
         amr = view.gather('a', block=False)
         self.assertTrue(isinstance(amr, AsyncMapResult))
         assert_array_equal(amr.get(), a)
 
     def test_execute(self):
         view = self.client[:]
         # self.client.debug=True
         execute = view.execute
         ar = execute('c=30', block=False)
         self.assertTrue(isinstance(ar, AsyncResult))
         ar = execute('d=[0,1,2]', block=False)
         self.client.wait(ar, 1)
         self.assertEquals(len(ar.get()), len(self.client))
         for c in view['c']:
             self.assertEquals(c, 30)
     
     def test_abort(self):
         view = self.client[-1]
         ar = view.execute('import time; time.sleep(1)', block=False)
         ar2 = view.apply_async(lambda : 2)
         ar3 = view.apply_async(lambda : 3)
         view.abort(ar2)
         view.abort(ar3.msg_ids)
         self.assertRaises(error.TaskAborted, ar2.get)
         self.assertRaises(error.TaskAborted, ar3.get)
     
     def test_abort_all(self):
         """view.abort() aborts all outstanding tasks"""
         view = self.client[-1]
         ars = [ view.apply_async(time.sleep, 0.25) for i in range(10) ]
         view.abort()
         view.wait(timeout=5)
         for ar in ars[5:]:
             self.assertRaises(error.TaskAborted, ar.get)
     
     def test_temp_flags(self):
         view = self.client[-1]
         view.block=True
         with view.temp_flags(block=False):
             self.assertFalse(view.block)
         self.assertTrue(view.block)
     
     @dec.known_failure_py3
     def test_importer(self):
         view = self.client[-1]
         view.clear(block=True)
         with view.importer:
             import re
         
         @interactive
         def findall(pat, s):
             # this globals() step isn't necessary in real code
             # only to prevent a closure in the test
             re = globals()['re']
             return re.findall(pat, s)
         
         self.assertEquals(view.apply_sync(findall, '\w+', 'hello world'), 'hello world'.split())
     
-    # parallel magic tests
-    
-    def test_magic_px_blocking(self):
-        ip = get_ipython()
-        v = self.client[-1]
-        v.activate()
-        v.block=True
-
-        ip.magic('px a=5')
-        self.assertEquals(v['a'], 5)
-        ip.magic('px a=10')
-        self.assertEquals(v['a'], 10)
-        sio = StringIO()
-        savestdout = sys.stdout
-        sys.stdout = sio
-        # just 'print a' worst ~99% of the time, but this ensures that
-        # the stdout message has arrived when the result is finished:
-        ip.magic('px import sys,time;print (a); sys.stdout.flush();time.sleep(0.2)')
-        sys.stdout = savestdout
-        buf = sio.getvalue()
-        self.assertTrue('[stdout:' in buf, buf)
-        self.assertTrue(buf.rstrip().endswith('10'))
-        self.assertRaisesRemote(ZeroDivisionError, ip.magic, 'px 1/0')
-
-    def test_magic_px_nonblocking(self):
-        ip = get_ipython()
-        v = self.client[-1]
-        v.activate()
-        v.block=False
-
-        ip.magic('px a=5')
-        self.assertEquals(v['a'], 5)
-        ip.magic('px a=10')
-        self.assertEquals(v['a'], 10)
-        sio = StringIO()
-        savestdout = sys.stdout
-        sys.stdout = sio
-        ip.magic('px print a')
-        sys.stdout = savestdout
-        buf = sio.getvalue()
-        self.assertFalse('[stdout:%i]'%v.targets in buf)
-        ip.magic('px 1/0')
-        ar = v.get_result(-1)
-        self.assertRaisesRemote(ZeroDivisionError, ar.get)
-    
-    def test_magic_autopx_blocking(self):
-        ip = get_ipython()
-        v = self.client[-1]
-        v.activate()
-        v.block=True
-
-        sio = StringIO()
-        savestdout = sys.stdout
-        sys.stdout = sio
-        ip.magic('autopx')
-        ip.run_cell('\n'.join(('a=5','b=10','c=0')))
-        ip.run_cell('b*=2')
-        ip.run_cell('print (b)')
-        ip.run_cell("b/c")
-        ip.magic('autopx')
-        sys.stdout = savestdout
-        output = sio.getvalue().strip()
-        self.assertTrue(output.startswith('%autopx enabled'))
-        self.assertTrue(output.endswith('%autopx disabled'))
-        self.assertTrue('RemoteError: ZeroDivisionError' in output)
-        ar = v.get_result(-1)
-        self.assertEquals(v['a'], 5)
-        self.assertEquals(v['b'], 20)
-        self.assertRaisesRemote(ZeroDivisionError, ar.get)
-
-    def test_magic_autopx_nonblocking(self):
-        ip = get_ipython()
-        v = self.client[-1]
-        v.activate()
-        v.block=False
-
-        sio = StringIO()
-        savestdout = sys.stdout
-        sys.stdout = sio
-        ip.magic('autopx')
-        ip.run_cell('\n'.join(('a=5','b=10','c=0')))
-        ip.run_cell('print (b)')
-        ip.run_cell('import time; time.sleep(0.1)')
-        ip.run_cell("b/c")
-        ip.run_cell('b*=2')
-        ip.magic('autopx')
-        sys.stdout = savestdout
-        output = sio.getvalue().strip()
-        self.assertTrue(output.startswith('%autopx enabled'))
-        self.assertTrue(output.endswith('%autopx disabled'))
-        self.assertFalse('ZeroDivisionError' in output)
-        ar = v.get_result(-2)
-        self.assertRaisesRemote(ZeroDivisionError, ar.get)
-        # prevent TaskAborted on pulls, due to ZeroDivisionError
-        time.sleep(0.5)
-        self.assertEquals(v['a'], 5)
-        # b*=2 will not fire, due to abort
-        self.assertEquals(v['b'], 10)
-    
-    def test_magic_result(self):
-        ip = get_ipython()
-        v = self.client[-1]
-        v.activate()
-        v['a'] = 111
-        ra = v['a']
-        
-        ar = ip.magic('result')
-        self.assertEquals(ar.msg_ids, [v.history[-1]])
-        self.assertEquals(ar.get(), 111)
-        ar = ip.magic('result -2')
-        self.assertEquals(ar.msg_ids, [v.history[-2]])
-    
     def test_unicode_execute(self):
         """test executing unicode strings"""
         v = self.client[-1]
         v.block=True
         if sys.version_info[0] >= 3:
             code="a='é'"
         else:
             code=u"a=u'é'"
         v.execute(code)
         self.assertEquals(v['a'], u'é')
         
     def test_unicode_apply_result(self):
         """test unicode apply results"""
         v = self.client[-1]
         r = v.apply_sync(lambda : u'é')
         self.assertEquals(r, u'é')
     
     def test_unicode_apply_arg(self):
         """test passing unicode arguments to apply"""
         v = self.client[-1]
         
         @interactive
         def check_unicode(a, check):
             assert isinstance(a, unicode), "%r is not unicode"%a
             assert isinstance(check, bytes), "%r is not bytes"%check
             assert a.encode('utf8') == check, "%s != %s"%(a,check)
         
         for s in [ u'é', u'ßø®∫',u'asdf' ]:
             try:
                 v.apply_sync(check_unicode, s, s.encode('utf8'))
             except error.RemoteError as e:
                 if e.ename == 'AssertionError':
                     self.fail(e.evalue)
                 else:
                     raise e
     
     def test_map_reference(self):
         """view.map(<Reference>, *seqs) should work"""
         v = self.client[:]
         v.scatter('n', self.client.ids, flatten=True)
         v.execute("f = lambda x,y: x*y")
         rf = pmod.Reference('f')
         nlist = list(range(10))
         mlist = nlist[::-1]
         expected = [ m*n for m,n in zip(mlist, nlist) ]
         result = v.map_sync(rf, mlist, nlist)
         self.assertEquals(result, expected)
 
     def test_apply_reference(self):
         """view.apply(<Reference>, *args) should work"""
         v = self.client[:]
         v.scatter('n', self.client.ids, flatten=True)
         v.execute("f = lambda x: n*x")
         rf = pmod.Reference('f')
         result = v.apply_sync(rf, 5)
         expected = [ 5*id for id in self.client.ids ]
         self.assertEquals(result, expected)
     
     def test_eval_reference(self):
         v = self.client[self.client.ids[0]]
         v['g'] = range(5)
         rg = pmod.Reference('g[0]')
         echo = lambda x:x
         self.assertEquals(v.apply_sync(echo, rg), 0)
     
     def test_reference_nameerror(self):
         v = self.client[self.client.ids[0]]
         r = pmod.Reference('elvis_has_left')
         echo = lambda x:x
         self.assertRaisesRemote(NameError, v.apply_sync, echo, r)
 
     def test_single_engine_map(self):
         e0 = self.client[self.client.ids[0]]
         r = range(5)
         check = [ -1*i for i in r ]
         result = e0.map_sync(lambda x: -1*x, r)
         self.assertEquals(result, check)
     
     def test_len(self):
         """len(view) makes sense"""
         e0 = self.client[self.client.ids[0]]
         yield self.assertEquals(len(e0), 1)
         v = self.client[:]
         yield self.assertEquals(len(v), len(self.client.ids))
         v = self.client.direct_view('all')
         yield self.assertEquals(len(v), len(self.client.ids))
         v = self.client[:2]
         yield self.assertEquals(len(v), 2)
         v = self.client[:1]
         yield self.assertEquals(len(v), 1)
         v = self.client.load_balanced_view()
         yield self.assertEquals(len(v), len(self.client.ids))
         # parametric tests seem to require manual closing?
         self.client.close()
 
     
     # begin execute tests
-    def _wait_for(self, f, timeout=10):
-        tic = time.time()
-        while time.time() <= tic + timeout:
-            if f():
-                return
-            time.sleep(0.1)
-            self.client.spin()
-        if not f():
-            print "Warning: Awaited condition never arrived"
-            
     
     def test_execute_reply(self):
         e0 = self.client[self.client.ids[0]]
         e0.block = True
         ar = e0.execute("5", silent=False)
         er = ar.get()
         self._wait_for(lambda : bool(er.pyout))
         self.assertEquals(str(er), "<ExecuteReply[%i]: 5>" % er.execution_count)
         self.assertEquals(er.pyout['data']['text/plain'], '5')
 
     def test_execute_reply_stdout(self):
         e0 = self.client[self.client.ids[0]]
         e0.block = True
         ar = e0.execute("print (5)", silent=False)
         er = ar.get()
         self._wait_for(lambda : bool(er.stdout))
         self.assertEquals(er.stdout.strip(), '5')
         
     def test_execute_pyout(self):
         """execute triggers pyout with silent=False"""
         view = self.client[:]
         ar = view.execute("5", silent=False, block=True)
         self._wait_for(lambda : all(ar.pyout))
         
         expected = [{'text/plain' : '5'}] * len(view)
         mimes = [ out['data'] for out in ar.pyout ]
         self.assertEquals(mimes, expected)
     
     def test_execute_silent(self):
         """execute does not trigger pyout with silent=True"""
         view = self.client[:]
         ar = view.execute("5", block=True)
         expected = [None] * len(view)
         self.assertEquals(ar.pyout, expected)
     
     def test_execute_magic(self):
         """execute accepts IPython commands"""
         view = self.client[:]
         view.execute("a = 5")
         ar = view.execute("%whos", block=True)
         # this will raise, if that failed
         ar.get(5)
         self._wait_for(lambda : all(ar.stdout))
         for stdout in ar.stdout:
             lines = stdout.splitlines()
             self.assertEquals(lines[0].split(), ['Variable', 'Type', 'Data/Info'])
             found = False
             for line in lines[2:]:
                 split = line.split()
                 if split == ['a', 'int', '5']:
                     found = True
                     break
             self.assertTrue(found, "whos output wrong: %s" % stdout)
     
     def test_execute_displaypub(self):
         """execute tracks display_pub output"""
         view = self.client[:]
         view.execute("from IPython.core.display import *")
         ar = view.execute("[ display(i) for i in range(5) ]", block=True)
         
         self._wait_for(lambda : all(len(er.outputs) >= 5 for er in ar))
         expected = [ {u'text/plain' : unicode(j)} for j in range(5) ]
         for outputs in ar.outputs:
             mimes = [ out['data'] for out in outputs ]
             self.assertEquals(mimes, expected)
     
     def test_apply_displaypub(self):
         """apply tracks display_pub output"""
         view = self.client[:]
         view.execute("from IPython.core.display import *")
         
         @interactive
         def publish():
             [ display(i) for i in range(5) ]
         
         ar = view.apply_async(publish)
         ar.get(5)
         self._wait_for(lambda : all(len(out) >= 5 for out in ar.outputs))
         expected = [ {u'text/plain' : unicode(j)} for j in range(5) ]
         for outputs in ar.outputs:
             mimes = [ out['data'] for out in outputs ]
             self.assertEquals(mimes, expected)
     
     def test_execute_raises(self):
         """exceptions in execute requests raise appropriately"""
         view = self.client[-1]
         ar = view.execute("1/0")
         self.assertRaisesRemote(ZeroDivisionError, ar.get, 2)
     
     @dec.skipif_not_matplotlib
     def test_magic_pylab(self):
         """%pylab works on engines"""
         view = self.client[-1]
         ar = view.execute("%pylab inline")
         # at least check if this raised:
         reply = ar.get(5)
         # include imports, in case user config
         ar = view.execute("plot(rand(100))", silent=False)
         reply = ar.get(5)
         self._wait_for(lambda : all(ar.outputs))
         self.assertEquals(len(reply.outputs), 1)
         output = reply.outputs[0]
         self.assertTrue("data" in output)
         data = output['data']
         self.assertTrue("image/png" in data)
         
 

IPython/zmq/displayhook.py

@@ -1,73 +1,75 b''
 import __builtin__
 import sys
 from base64 import encodestring
 
 from IPython.core.displayhook import DisplayHook
 from IPython.utils.traitlets import Instance, Dict
 from session import extract_header, Session
 
 class ZMQDisplayHook(object):
     """A simple displayhook that publishes the object's repr over a ZeroMQ
     socket."""
     topic=None
 
     def __init__(self, session, pub_socket):
         self.session = session
         self.pub_socket = pub_socket
         self.parent_header = {}
 
     def __call__(self, obj):
         if obj is None:
             return
 
         __builtin__._ = obj
         sys.stdout.flush()
         sys.stderr.flush()
         msg = self.session.send(self.pub_socket, u'pyout', {u'data':repr(obj)},
                                parent=self.parent_header, ident=self.topic)
 
     def set_parent(self, parent):
         self.parent_header = extract_header(parent)
 
 
 def _encode_binary(format_dict):
+    encoded = format_dict.copy()
     pngdata = format_dict.get('image/png')
-    if pngdata is not None:
-        format_dict['image/png'] = encodestring(pngdata).decode('ascii')
+    if isinstance(pngdata, bytes):
+        encoded['image/png'] = encodestring(pngdata).decode('ascii')
     jpegdata = format_dict.get('image/jpeg')
-    if jpegdata is not None:
-        format_dict['image/jpeg'] = encodestring(jpegdata).decode('ascii')
+    if isinstance(jpegdata, bytes):
+        encoded['image/jpeg'] = encodestring(jpegdata).decode('ascii')
+    
+    return encoded
 
 
 class ZMQShellDisplayHook(DisplayHook):
     """A displayhook subclass that publishes data using ZeroMQ. This is intended
     to work with an InteractiveShell instance. It sends a dict of different
     representations of the object."""
     topic=None
 
     session = Instance(Session)
     pub_socket = Instance('zmq.Socket')
     parent_header = Dict({})
 
     def set_parent(self, parent):
         """Set the parent for outbound messages."""
         self.parent_header = extract_header(parent)
 
     def start_displayhook(self):
         self.msg = self.session.msg(u'pyout', {}, parent=self.parent_header)
 
     def write_output_prompt(self):
         """Write the output prompt."""
         self.msg['content']['execution_count'] = self.prompt_count
 
     def write_format_data(self, format_dict):
-        _encode_binary(format_dict)
-        self.msg['content']['data'] = format_dict
+        self.msg['content']['data'] = _encode_binary(format_dict)
 
     def finish_displayhook(self):
         """Finish up all displayhook activities."""
         sys.stdout.flush()
         sys.stderr.flush()
         self.session.send(self.pub_socket, self.msg, ident=self.topic)
         self.msg = None
 

IPython/zmq/ipkernel.py

@@ -1,919 +1,921 b''
 #!/usr/bin/env python
 """A simple interactive kernel that talks to a frontend over 0MQ.
 
 Things to do:
 
 * Implement `set_parent` logic. Right before doing exec, the Kernel should
   call set_parent on all the PUB objects with the message about to be executed.
 * Implement random port and security key logic.
 * Implement control messages.
 * Implement event loop and poll version.
 """
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 from __future__ import print_function
 
 # Standard library imports
 import __builtin__
 import atexit
 import sys
 import time
 import traceback
 import logging
 import uuid
 
 from datetime import datetime
 from signal import (
         signal, getsignal, default_int_handler, SIGINT, SIG_IGN
 )
 
 # System library imports
 import zmq
 from zmq.eventloop import ioloop
 from zmq.eventloop.zmqstream import ZMQStream
 
 # Local imports
 from IPython.core import pylabtools
 from IPython.config.configurable import Configurable
 from IPython.config.application import boolean_flag, catch_config_error
 from IPython.core.application import ProfileDir
 from IPython.core.error import StdinNotImplementedError
 from IPython.core.shellapp import (
     InteractiveShellApp, shell_flags, shell_aliases
 )
 from IPython.utils import io
 from IPython.utils import py3compat
 from IPython.utils.frame import extract_module_locals
 from IPython.utils.jsonutil import json_clean
 from IPython.utils.traitlets import (
     Any, Instance, Float, Dict, CaselessStrEnum, List, Set, Integer, Unicode
 )
 
 from entry_point import base_launch_kernel
 from kernelapp import KernelApp, kernel_flags, kernel_aliases
 from serialize import serialize_object, unpack_apply_message
 from session import Session, Message
 from zmqshell import ZMQInteractiveShell
 
 
 #-----------------------------------------------------------------------------
 # Main kernel class
 #-----------------------------------------------------------------------------
 
 class Kernel(Configurable):
 
     #---------------------------------------------------------------------------
     # Kernel interface
     #---------------------------------------------------------------------------
 
     # attribute to override with a GUI
     eventloop = Any(None)
     def _eventloop_changed(self, name, old, new):
         """schedule call to eventloop from IOLoop"""
         loop = ioloop.IOLoop.instance()
         loop.add_timeout(time.time()+0.1, self.enter_eventloop)
 
     shell = Instance('IPython.core.interactiveshell.InteractiveShellABC')
     session = Instance(Session)
     profile_dir = Instance('IPython.core.profiledir.ProfileDir')
     shell_streams = List()
     control_stream = Instance(ZMQStream)
     iopub_socket = Instance(zmq.Socket)
     stdin_socket = Instance(zmq.Socket)
     log = Instance(logging.Logger)
     
     user_module = Any()
     def _user_module_changed(self, name, old, new):
         if self.shell is not None:
             self.shell.user_module = new
     
     user_ns = Dict(default_value=None)
     def _user_ns_changed(self, name, old, new):
         if self.shell is not None:
             self.shell.user_ns = new
             self.shell.init_user_ns()
 
     # identities:
     int_id = Integer(-1)
     ident = Unicode()
 
     def _ident_default(self):
         return unicode(uuid.uuid4())
 
 
     # Private interface
     
     # Time to sleep after flushing the stdout/err buffers in each execute
     # cycle.  While this introduces a hard limit on the minimal latency of the
     # execute cycle, it helps prevent output synchronization problems for
     # clients.
     # Units are in seconds.  The minimum zmq latency on local host is probably
     # ~150 microseconds, set this to 500us for now.  We may need to increase it
     # a little if it's not enough after more interactive testing.
     _execute_sleep = Float(0.0005, config=True)
 
     # Frequency of the kernel's event loop.
     # Units are in seconds, kernel subclasses for GUI toolkits may need to
     # adapt to milliseconds.
     _poll_interval = Float(0.05, config=True)
 
     # If the shutdown was requested over the network, we leave here the
     # necessary reply message so it can be sent by our registered atexit
     # handler.  This ensures that the reply is only sent to clients truly at
     # the end of our shutdown process (which happens after the underlying
     # IPython shell's own shutdown).
     _shutdown_message = None
 
     # This is a dict of port number that the kernel is listening on. It is set
     # by record_ports and used by connect_request.
     _recorded_ports = Dict()
     
     # set of aborted msg_ids
     aborted = Set()
 
 
     def __init__(self, **kwargs):
         super(Kernel, self).__init__(**kwargs)
 
         # Initialize the InteractiveShell subclass
         self.shell = ZMQInteractiveShell.instance(config=self.config,
             profile_dir = self.profile_dir,
             user_module = self.user_module,
             user_ns     = self.user_ns,
         )
         self.shell.displayhook.session = self.session
         self.shell.displayhook.pub_socket = self.iopub_socket
         self.shell.displayhook.topic = self._topic('pyout')
         self.shell.display_pub.session = self.session
         self.shell.display_pub.pub_socket = self.iopub_socket
 
         # TMP - hack while developing
         self.shell._reply_content = None
 
         # Build dict of handlers for message types
         msg_types = [ 'execute_request', 'complete_request',
                       'object_info_request', 'history_request',
                       'connect_request', 'shutdown_request',
                       'apply_request',
                     ]
         self.shell_handlers = {}
         for msg_type in msg_types:
             self.shell_handlers[msg_type] = getattr(self, msg_type)
         
         control_msg_types = msg_types + [ 'clear_request', 'abort_request' ]
         self.control_handlers = {}
         for msg_type in control_msg_types:
             self.control_handlers[msg_type] = getattr(self, msg_type)
 
     def dispatch_control(self, msg):
         """dispatch control requests"""
         idents,msg = self.session.feed_identities(msg, copy=False)
         try:
             msg = self.session.unserialize(msg, content=True, copy=False)
         except:
             self.log.error("Invalid Control Message", exc_info=True)
             return
 
         self.log.debug("Control received: %s", msg)
 
         header = msg['header']
         msg_id = header['msg_id']
         msg_type = header['msg_type']
 
         handler = self.control_handlers.get(msg_type, None)
         if handler is None:
             self.log.error("UNKNOWN CONTROL MESSAGE TYPE: %r", msg_type)
         else:
             try:
                 handler(self.control_stream, idents, msg)
             except Exception:
                 self.log.error("Exception in control handler:", exc_info=True)
     
     def dispatch_shell(self, stream, msg):
         """dispatch shell requests"""
         # flush control requests first
         if self.control_stream:
             self.control_stream.flush()
         
         idents,msg = self.session.feed_identities(msg, copy=False)
         try:
             msg = self.session.unserialize(msg, content=True, copy=False)
         except:
             self.log.error("Invalid Message", exc_info=True)
             return
 
         header = msg['header']
         msg_id = header['msg_id']
         msg_type = msg['header']['msg_type']
         
         # Print some info about this message and leave a '--->' marker, so it's
         # easier to trace visually the message chain when debugging.  Each
         # handler prints its message at the end.
         self.log.debug('\n*** MESSAGE TYPE:%s***', msg_type)
         self.log.debug('   Content: %s\n   --->\n   ', msg['content'])
 
         if msg_id in self.aborted:
             self.aborted.remove(msg_id)
             # is it safe to assume a msg_id will not be resubmitted?
             reply_type = msg_type.split('_')[0] + '_reply'
             status = {'status' : 'aborted'}
             sub = {'engine' : self.ident}
             sub.update(status)
             reply_msg = self.session.send(stream, reply_type, subheader=sub,
                         content=status, parent=msg, ident=idents)
             return
         
         handler = self.shell_handlers.get(msg_type, None)
         if handler is None:
             self.log.error("UNKNOWN MESSAGE TYPE: %r", msg_type)
         else:
             # ensure default_int_handler during handler call
             sig = signal(SIGINT, default_int_handler)
             try:
                 handler(stream, idents, msg)
             except Exception:
                 self.log.error("Exception in message handler:", exc_info=True)
             finally:
                 signal(SIGINT, sig)
     
     def enter_eventloop(self):
         """enter eventloop"""
         self.log.info("entering eventloop")
         # restore default_int_handler
         signal(SIGINT, default_int_handler)
         while self.eventloop is not None:
             try:
                 self.eventloop(self)
             except KeyboardInterrupt:
                 # Ctrl-C shouldn't crash the kernel
                 self.log.error("KeyboardInterrupt caught in kernel")
                 continue
             else:
                 # eventloop exited cleanly, this means we should stop (right?)
                 self.eventloop = None
                 break
         self.log.info("exiting eventloop")
         # if eventloop exits, IOLoop should stop
         ioloop.IOLoop.instance().stop()
 
     def start(self):
         """register dispatchers for streams"""
         self.shell.exit_now = False
         if self.control_stream:
             self.control_stream.on_recv(self.dispatch_control, copy=False)
 
         def make_dispatcher(stream):
             def dispatcher(msg):
                 return self.dispatch_shell(stream, msg)
             return dispatcher
 
         for s in self.shell_streams:
             s.on_recv(make_dispatcher(s), copy=False)
     
     def do_one_iteration(self):
         """step eventloop just once"""
         if self.control_stream:
             self.control_stream.flush()
         for stream in self.shell_streams:
             # handle at most one request per iteration
             stream.flush(zmq.POLLIN, 1)
             stream.flush(zmq.POLLOUT)
 
 
     def record_ports(self, ports):
         """Record the ports that this kernel is using.
 
         The creator of the Kernel instance must call this methods if they
         want the :meth:`connect_request` method to return the port numbers.
         """
         self._recorded_ports = ports
 
     #---------------------------------------------------------------------------
     # Kernel request handlers
     #---------------------------------------------------------------------------
     
     def _make_subheader(self):
         """init subheader dict, for execute/apply_reply"""
         return {
             'dependencies_met' : True,
             'engine' : self.ident,
             'started': datetime.now(),
         }
     
     def _publish_pyin(self, code, parent, execution_count):
         """Publish the code request on the pyin stream."""
 
         self.session.send(self.iopub_socket, u'pyin',
                             {u'code':code, u'execution_count': execution_count},
                             parent=parent, ident=self._topic('pyin')
         )
 
     def execute_request(self, stream, ident, parent):
 
         self.session.send(self.iopub_socket,
                           u'status',
                           {u'execution_state':u'busy'},
                           parent=parent,
                           ident=self._topic('status'),
                           )
         
         try:
             content = parent[u'content']
             code = content[u'code']
             silent = content[u'silent']
         except:
             self.log.error("Got bad msg: ")
             self.log.error("%s", parent)
             return
         
         sub = self._make_subheader()
 
         shell = self.shell # we'll need this a lot here
 
         # Replace raw_input. Note that is not sufficient to replace
         # raw_input in the user namespace.
         if content.get('allow_stdin', False):
             raw_input = lambda prompt='': self._raw_input(prompt, ident, parent)
         else:
             raw_input = lambda prompt='' : self._no_raw_input()
 
         if py3compat.PY3:
             __builtin__.input = raw_input
         else:
             __builtin__.raw_input = raw_input
 
         # Set the parent message of the display hook and out streams.
         shell.displayhook.set_parent(parent)
         shell.display_pub.set_parent(parent)
         sys.stdout.set_parent(parent)
         sys.stderr.set_parent(parent)
 
         # Re-broadcast our input for the benefit of listening clients, and
         # start computing output
         if not silent:
             self._publish_pyin(code, parent, shell.execution_count)
 
         reply_content = {}
         try:
             # FIXME: the shell calls the exception handler itself.
             shell.run_cell(code, store_history=not silent, silent=silent)
         except:
             status = u'error'
             # FIXME: this code right now isn't being used yet by default,
             # because the run_cell() call above directly fires off exception
             # reporting.  This code, therefore, is only active in the scenario
             # where runlines itself has an unhandled exception.  We need to
             # uniformize this, for all exception construction to come from a
             # single location in the codbase.
             etype, evalue, tb = sys.exc_info()
             tb_list = traceback.format_exception(etype, evalue, tb)
             reply_content.update(shell._showtraceback(etype, evalue, tb_list))
         else:
             status = u'ok'
 
         reply_content[u'status'] = status
 
         # Return the execution counter so clients can display prompts
         reply_content['execution_count'] = shell.execution_count - 1
 
         # FIXME - fish exception info out of shell, possibly left there by
         # runlines.  We'll need to clean up this logic later.
         if shell._reply_content is not None:
             reply_content.update(shell._reply_content)
+            e_info = dict(engine_uuid=self.ident, engine_id=self.int_id, method='execute')
+            reply_content['engine_info'] = e_info
             # reset after use
             shell._reply_content = None
 
         # At this point, we can tell whether the main code execution succeeded
         # or not.  If it did, we proceed to evaluate user_variables/expressions
         if reply_content['status'] == 'ok':
             reply_content[u'user_variables'] = \
                          shell.user_variables(content.get(u'user_variables', []))
             reply_content[u'user_expressions'] = \
                          shell.user_expressions(content.get(u'user_expressions', {}))
         else:
             # If there was an error, don't even try to compute variables or
             # expressions
             reply_content[u'user_variables'] = {}
             reply_content[u'user_expressions'] = {}
 
         # Payloads should be retrieved regardless of outcome, so we can both
         # recover partial output (that could have been generated early in a
         # block, before an error) and clear the payload system always.
         reply_content[u'payload'] = shell.payload_manager.read_payload()
         # Be agressive about clearing the payload because we don't want
         # it to sit in memory until the next execute_request comes in.
         shell.payload_manager.clear_payload()
 
         # Flush output before sending the reply.
         sys.stdout.flush()
         sys.stderr.flush()
         # FIXME: on rare occasions, the flush doesn't seem to make it to the
         # clients... This seems to mitigate the problem, but we definitely need
         # to better understand what's going on.
         if self._execute_sleep:
             time.sleep(self._execute_sleep)
 
         # Send the reply.
         reply_content = json_clean(reply_content)
         
         sub['status'] = reply_content['status']
         if reply_content['status'] == 'error' and \
                         reply_content['ename'] == 'UnmetDependency':
                 sub['dependencies_met'] = False
 
         reply_msg = self.session.send(stream, u'execute_reply',
                                       reply_content, parent, subheader=sub,
                                       ident=ident)
         
         self.log.debug("%s", reply_msg)
 
         if not silent and reply_msg['content']['status'] == u'error':
             self._abort_queues()
 
         self.session.send(self.iopub_socket,
                           u'status',
                           {u'execution_state':u'idle'},
                           parent=parent,
                           ident=self._topic('status'))
 
     def complete_request(self, stream, ident, parent):
         txt, matches = self._complete(parent)
         matches = {'matches' : matches,
                    'matched_text' : txt,
                    'status' : 'ok'}
         matches = json_clean(matches)
         completion_msg = self.session.send(stream, 'complete_reply',
                                            matches, parent, ident)
         self.log.debug("%s", completion_msg)
 
     def object_info_request(self, stream, ident, parent):
         content = parent['content']
         object_info = self.shell.object_inspect(content['oname'],
                         detail_level = content.get('detail_level', 0)
         )
         # Before we send this object over, we scrub it for JSON usage
         oinfo = json_clean(object_info)
         msg = self.session.send(stream, 'object_info_reply',
                                 oinfo, parent, ident)
         self.log.debug("%s", msg)
 
     def history_request(self, stream, ident, parent):
         # We need to pull these out, as passing **kwargs doesn't work with
         # unicode keys before Python 2.6.5.
         hist_access_type = parent['content']['hist_access_type']
         raw = parent['content']['raw']
         output = parent['content']['output']
         if hist_access_type == 'tail':
             n = parent['content']['n']
             hist = self.shell.history_manager.get_tail(n, raw=raw, output=output,
                                                             include_latest=True)
 
         elif hist_access_type == 'range':
             session = parent['content']['session']
             start = parent['content']['start']
             stop = parent['content']['stop']
             hist = self.shell.history_manager.get_range(session, start, stop,
                                                         raw=raw, output=output)
 
         elif hist_access_type == 'search':
             pattern = parent['content']['pattern']
             hist = self.shell.history_manager.search(pattern, raw=raw,
                                                      output=output) 
 
         else:
             hist = []
         hist = list(hist)
         content = {'history' : hist}
         content = json_clean(content)
         msg = self.session.send(stream, 'history_reply',
                                 content, parent, ident)
         self.log.debug("Sending history reply with %i entries", len(hist))
 
     def connect_request(self, stream, ident, parent):
         if self._recorded_ports is not None:
             content = self._recorded_ports.copy()
         else:
             content = {}
         msg = self.session.send(stream, 'connect_reply',
                                 content, parent, ident)
         self.log.debug("%s", msg)
 
     def shutdown_request(self, stream, ident, parent):
         self.shell.exit_now = True
         content = dict(status='ok')
         content.update(parent['content'])
         self.session.send(stream, u'shutdown_reply', content, parent, ident=ident)
         # same content, but different msg_id for broadcasting on IOPub
         self._shutdown_message = self.session.msg(u'shutdown_reply',
                                                   content, parent
         )
 
         self._at_shutdown()
         # call sys.exit after a short delay
         loop = ioloop.IOLoop.instance()
         loop.add_timeout(time.time()+0.1, loop.stop)
 
     #---------------------------------------------------------------------------
     # Engine methods
     #---------------------------------------------------------------------------
 
     def apply_request(self, stream, ident, parent):
         try:
             content = parent[u'content']
             bufs = parent[u'buffers']
             msg_id = parent['header']['msg_id']
         except:
             self.log.error("Got bad msg: %s", parent, exc_info=True)
             return
 
         # Set the parent message of the display hook and out streams.
         self.shell.displayhook.set_parent(parent)
         self.shell.display_pub.set_parent(parent)
         sys.stdout.set_parent(parent)
         sys.stderr.set_parent(parent)
 
         # pyin_msg = self.session.msg(u'pyin',{u'code':code}, parent=parent)
         # self.iopub_socket.send(pyin_msg)
         # self.session.send(self.iopub_socket, u'pyin', {u'code':code},parent=parent)
         sub = self._make_subheader()
         try:
             working = self.shell.user_ns
 
             prefix = "_"+str(msg_id).replace("-","")+"_"
 
             f,args,kwargs = unpack_apply_message(bufs, working, copy=False)
 
             fname = getattr(f, '__name__', 'f')
 
             fname = prefix+"f"
             argname = prefix+"args"
             kwargname = prefix+"kwargs"
             resultname = prefix+"result"
 
             ns = { fname : f, argname : args, kwargname : kwargs , resultname : None }
             # print ns
             working.update(ns)
             code = "%s = %s(*%s,**%s)" % (resultname, fname, argname, kwargname)
             try:
                 exec code in self.shell.user_global_ns, self.shell.user_ns
                 result = working.get(resultname)
             finally:
                 for key in ns.iterkeys():
                     working.pop(key)
 
             packed_result,buf = serialize_object(result)
             result_buf = [packed_result]+buf
         except:
             exc_content = self._wrap_exception('apply')
             # exc_msg = self.session.msg(u'pyerr', exc_content, parent)
             self.session.send(self.iopub_socket, u'pyerr', exc_content, parent=parent,
                                 ident=self._topic('pyerr'))
             reply_content = exc_content
             result_buf = []
 
             if exc_content['ename'] == 'UnmetDependency':
                 sub['dependencies_met'] = False
         else:
             reply_content = {'status' : 'ok'}
 
         # put 'ok'/'error' status in header, for scheduler introspection:
         sub['status'] = reply_content['status']
 
         # flush i/o
         sys.stdout.flush()
         sys.stderr.flush()
         
         reply_msg = self.session.send(stream, u'apply_reply', reply_content,
                     parent=parent, ident=ident,buffers=result_buf, subheader=sub)
 
     #---------------------------------------------------------------------------
     # Control messages
     #---------------------------------------------------------------------------
 
     def abort_request(self, stream, ident, parent):
         """abort a specifig msg by id"""
         msg_ids = parent['content'].get('msg_ids', None)
         if isinstance(msg_ids, basestring):
             msg_ids = [msg_ids]
         if not msg_ids:
             self.abort_queues()
         for mid in msg_ids:
             self.aborted.add(str(mid))
 
         content = dict(status='ok')
         reply_msg = self.session.send(stream, 'abort_reply', content=content,
                 parent=parent, ident=ident)
         self.log.debug("%s", reply_msg)
 
     def clear_request(self, stream, idents, parent):
         """Clear our namespace."""
         self.shell.reset(False)
         msg = self.session.send(stream, 'clear_reply', ident=idents, parent=parent,
                 content = dict(status='ok'))
 
 
     #---------------------------------------------------------------------------
     # Protected interface
     #---------------------------------------------------------------------------
 
 
     def _wrap_exception(self, method=None):
         # import here, because _wrap_exception is only used in parallel,
         # and parallel has higher min pyzmq version
         from IPython.parallel.error import wrap_exception
         e_info = dict(engine_uuid=self.ident, engine_id=self.int_id, method=method)
         content = wrap_exception(e_info)
         return content
 
     def _topic(self, topic):
         """prefixed topic for IOPub messages"""
         if self.int_id >= 0:
             base = "engine.%i" % self.int_id
         else:
             base = "kernel.%s" % self.ident
         
         return py3compat.cast_bytes("%s.%s" % (base, topic))
     
     def _abort_queues(self):
         for stream in self.shell_streams:
             if stream:
                 self._abort_queue(stream)
 
     def _abort_queue(self, stream):
         poller = zmq.Poller()
         poller.register(stream.socket, zmq.POLLIN)
         while True:
             idents,msg = self.session.recv(stream, zmq.NOBLOCK, content=True)
             if msg is None:
                 return
 
             self.log.info("Aborting:")
             self.log.info("%s", msg)
             msg_type = msg['header']['msg_type']
             reply_type = msg_type.split('_')[0] + '_reply'
 
             status = {'status' : 'aborted'}
             sub = {'engine' : self.ident}
             sub.update(status)
             reply_msg = self.session.send(stream, reply_type, subheader=sub,
                         content=status, parent=msg, ident=idents)
             self.log.debug("%s", reply_msg)
             # We need to wait a bit for requests to come in. This can probably
             # be set shorter for true asynchronous clients.
             poller.poll(50)
 
 
     def _no_raw_input(self):
         """Raise StdinNotImplentedError if active frontend doesn't support
         stdin."""
         raise StdinNotImplementedError("raw_input was called, but this "
                                        "frontend does not support stdin.") 
         
     def _raw_input(self, prompt, ident, parent):
         # Flush output before making the request.
         sys.stderr.flush()
         sys.stdout.flush()
 
         # Send the input request.
         content = json_clean(dict(prompt=prompt))
         self.session.send(self.stdin_socket, u'input_request', content, parent,
                           ident=ident)
 
         # Await a response.
         while True:
             try:
                 ident, reply = self.session.recv(self.stdin_socket, 0)
             except Exception:
                 self.log.warn("Invalid Message:", exc_info=True)
             else:
                 break
         try:
             value = reply['content']['value']
         except:
             self.log.error("Got bad raw_input reply: ")
             self.log.error("%s", parent)
             value = ''
         if value == '\x04':
             # EOF
             raise EOFError
         return value
 
     def _complete(self, msg):
         c = msg['content']
         try:
             cpos = int(c['cursor_pos'])
         except:
             # If we don't get something that we can convert to an integer, at
             # least attempt the completion guessing the cursor is at the end of
             # the text, if there's any, and otherwise of the line
             cpos = len(c['text'])
             if cpos==0:
                 cpos = len(c['line'])
         return self.shell.complete(c['text'], c['line'], cpos)
 
     def _object_info(self, context):
         symbol, leftover = self._symbol_from_context(context)
         if symbol is not None and not leftover:
             doc = getattr(symbol, '__doc__', '')
         else:
             doc = ''
         object_info = dict(docstring = doc)
         return object_info
 
     def _symbol_from_context(self, context):
         if not context:
             return None, context
 
         base_symbol_string = context[0]
         symbol = self.shell.user_ns.get(base_symbol_string, None)
         if symbol is None:
             symbol = __builtin__.__dict__.get(base_symbol_string, None)
         if symbol is None:
             return None, context
 
         context = context[1:]
         for i, name in enumerate(context):
             new_symbol = getattr(symbol, name, None)
             if new_symbol is None:
                 return symbol, context[i:]
             else:
                 symbol = new_symbol
 
         return symbol, []
 
     def _at_shutdown(self):
         """Actions taken at shutdown by the kernel, called by python's atexit.
         """
         # io.rprint("Kernel at_shutdown") # dbg
         if self._shutdown_message is not None:
             self.session.send(self.iopub_socket, self._shutdown_message, ident=self._topic('shutdown'))
             self.log.debug("%s", self._shutdown_message)
         [ s.flush(zmq.POLLOUT) for s in self.shell_streams ]
 
 #-----------------------------------------------------------------------------
 # Aliases and Flags for the IPKernelApp
 #-----------------------------------------------------------------------------
 
 flags = dict(kernel_flags)
 flags.update(shell_flags)
 
 addflag = lambda *args: flags.update(boolean_flag(*args))
 
 flags['pylab'] = (
     {'IPKernelApp' : {'pylab' : 'auto'}},
     """Pre-load matplotlib and numpy for interactive use with
     the default matplotlib backend."""
 )
 
 aliases = dict(kernel_aliases)
 aliases.update(shell_aliases)
 
 # it's possible we don't want short aliases for *all* of these:
 aliases.update(dict(
     pylab='IPKernelApp.pylab',
 ))
 
 #-----------------------------------------------------------------------------
 # The IPKernelApp class
 #-----------------------------------------------------------------------------
 
 class IPKernelApp(KernelApp, InteractiveShellApp):
     name = 'ipkernel'
 
     aliases = Dict(aliases)
     flags = Dict(flags)
     classes = [Kernel, ZMQInteractiveShell, ProfileDir, Session]
     
     # configurables
     pylab = CaselessStrEnum(['tk', 'qt', 'wx', 'gtk', 'osx', 'inline', 'auto'],
         config=True,
         help="""Pre-load matplotlib and numpy for interactive use,
         selecting a particular matplotlib backend and loop integration.
         """
     )
     
     @catch_config_error
     def initialize(self, argv=None):
         super(IPKernelApp, self).initialize(argv)
         self.init_path()
         self.init_shell()
         self.init_extensions()
         self.init_code()
 
     def init_kernel(self):
         
         shell_stream = ZMQStream(self.shell_socket)
 
         kernel = Kernel(config=self.config, session=self.session,
                                 shell_streams=[shell_stream],
                                 iopub_socket=self.iopub_socket,
                                 stdin_socket=self.stdin_socket,
                                 log=self.log,
                                 profile_dir=self.profile_dir,
         )
         self.kernel = kernel
         kernel.record_ports(self.ports)
         shell = kernel.shell
         if self.pylab:
             try:
                 gui, backend = pylabtools.find_gui_and_backend(self.pylab)
                 shell.enable_pylab(gui, import_all=self.pylab_import_all)
             except Exception:
                 self.log.error("Pylab initialization failed", exc_info=True)
                 # print exception straight to stdout, because normally 
                 # _showtraceback associates the reply with an execution, 
                 # which means frontends will never draw it, as this exception 
                 # is not associated with any execute request.
                 
                 # replace pyerr-sending traceback with stdout
                 _showtraceback = shell._showtraceback
                 def print_tb(etype, evalue, stb):
                     print ("Error initializing pylab, pylab mode will not "
                            "be active", file=io.stderr)
                     print (shell.InteractiveTB.stb2text(stb), file=io.stdout)
                 shell._showtraceback = print_tb
                 
                 # send the traceback over stdout
                 shell.showtraceback(tb_offset=0)
                 
                 # restore proper _showtraceback method
                 shell._showtraceback = _showtraceback
                 
 
     def init_shell(self):
         self.shell = self.kernel.shell
         self.shell.configurables.append(self)
 
 
 #-----------------------------------------------------------------------------
 # Kernel main and launch functions
 #-----------------------------------------------------------------------------
 
 def launch_kernel(*args, **kwargs):
     """Launches a localhost IPython kernel, binding to the specified ports.
 
     This function simply calls entry_point.base_launch_kernel with the right
     first command to start an ipkernel.  See base_launch_kernel for arguments.
 
     Returns
     -------
     A tuple of form:
         (kernel_process, shell_port, iopub_port, stdin_port, hb_port)
     where kernel_process is a Popen object and the ports are integers.
     """
     return base_launch_kernel('from IPython.zmq.ipkernel import main; main()',
                               *args, **kwargs)
 
 
 def embed_kernel(module=None, local_ns=None, **kwargs):
     """Embed and start an IPython kernel in a given scope.
     
     Parameters
     ----------
     module : ModuleType, optional
         The module to load into IPython globals (default: caller)
     local_ns : dict, optional
         The namespace to load into IPython user namespace (default: caller)
     
     kwargs : various, optional
         Further keyword args are relayed to the KernelApp constructor,
         allowing configuration of the Kernel.  Will only have an effect
         on the first embed_kernel call for a given process.
     
     """
     # get the app if it exists, or set it up if it doesn't
     if IPKernelApp.initialized():
         app = IPKernelApp.instance()
     else:
         app = IPKernelApp.instance(**kwargs)
         app.initialize([])
         # Undo unnecessary sys module mangling from init_sys_modules.
         # This would not be necessary if we could prevent it
         # in the first place by using a different InteractiveShell
         # subclass, as in the regular embed case.
         main = app.kernel.shell._orig_sys_modules_main_mod
         if main is not None:
             sys.modules[app.kernel.shell._orig_sys_modules_main_name] = main
 
     # load the calling scope if not given
     (caller_module, caller_locals) = extract_module_locals(1)
     if module is None:
         module = caller_module
     if local_ns is None:
         local_ns = caller_locals
     
     app.kernel.user_module = module
     app.kernel.user_ns = local_ns
     app.start()
 
 def main():
     """Run an IPKernel as an application"""
     app = IPKernelApp.instance()
     app.initialize()
     app.start()
 
 
 if __name__ == '__main__':
     main()

IPython/zmq/zmqshell.py

@@ -1,540 +1,539 b''
 """A ZMQ-based subclass of InteractiveShell.
 
 This code is meant to ease the refactoring of the base InteractiveShell into
 something with a cleaner architecture for 2-process use, without actually
 breaking InteractiveShell itself.  So we're doing something a bit ugly, where
 we subclass and override what we want to fix.  Once this is working well, we
 can go back to the base class and refactor the code for a cleaner inheritance
 implementation that doesn't rely on so much monkeypatching.
 
 But this lets us maintain a fully working IPython as we develop the new
 machinery.  This should thus be thought of as scaffolding.
 """
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 from __future__ import print_function
 
 # Stdlib
 import inspect
 import os
 import sys
 import time
 from subprocess import Popen, PIPE
 
 # System library imports
 from zmq.eventloop import ioloop
 
 # Our own
 from IPython.core.interactiveshell import (
     InteractiveShell, InteractiveShellABC
 )
 from IPython.core import page, pylabtools
 from IPython.core.autocall import ZMQExitAutocall
 from IPython.core.displaypub import DisplayPublisher
 from IPython.core.macro import Macro
 from IPython.core.magics import MacroToEdit
 from IPython.core.payloadpage import install_payload_page
 from IPython.lib.kernel import (
     get_connection_file, get_connection_info, connect_qtconsole
 )
 from IPython.testing.skipdoctest import skip_doctest
 from IPython.utils import io
 from IPython.utils.jsonutil import json_clean
 from IPython.utils.path import get_py_filename
 from IPython.utils.process import arg_split
 from IPython.utils.traitlets import Instance, Type, Dict, CBool, CBytes
 from IPython.utils.warn import warn, error
 from IPython.zmq.displayhook import ZMQShellDisplayHook, _encode_binary
 from IPython.zmq.session import extract_header
 from session import Session
 
 #-----------------------------------------------------------------------------
 # Functions and classes
 #-----------------------------------------------------------------------------
 
 class ZMQDisplayPublisher(DisplayPublisher):
     """A display publisher that publishes data using a ZeroMQ PUB socket."""
 
     session = Instance(Session)
     pub_socket = Instance('zmq.Socket')
     parent_header = Dict({})
     topic = CBytes(b'displaypub')
 
     def set_parent(self, parent):
         """Set the parent for outbound messages."""
         self.parent_header = extract_header(parent)
     
     def _flush_streams(self):
         """flush IO Streams prior to display"""
         sys.stdout.flush()
         sys.stderr.flush()
 
     def publish(self, source, data, metadata=None):
         self._flush_streams()
         if metadata is None:
             metadata = {}
         self._validate_data(source, data, metadata)
         content = {}
         content['source'] = source
-        _encode_binary(data)
-        content['data'] = data
+        content['data'] = _encode_binary(data)
         content['metadata'] = metadata
         self.session.send(
             self.pub_socket, u'display_data', json_clean(content),
             parent=self.parent_header, ident=self.topic,
         )
 
     def clear_output(self, stdout=True, stderr=True, other=True):
         content = dict(stdout=stdout, stderr=stderr, other=other)
         
         if stdout:
             print('\r', file=sys.stdout, end='')
         if stderr:
             print('\r', file=sys.stderr, end='')
         
         self._flush_streams()
         
         self.session.send(
             self.pub_socket, u'clear_output', content,
             parent=self.parent_header, ident=self.topic,
         )
 
 class ZMQInteractiveShell(InteractiveShell):
     """A subclass of InteractiveShell for ZMQ."""
 
     displayhook_class = Type(ZMQShellDisplayHook)
     display_pub_class = Type(ZMQDisplayPublisher)
 
     # Override the traitlet in the parent class, because there's no point using
     # readline for the kernel. Can be removed when the readline code is moved
     # to the terminal frontend.
     colors_force = CBool(True)
     readline_use = CBool(False)
     # autoindent has no meaning in a zmqshell, and attempting to enable it
     # will print a warning in the absence of readline.
     autoindent = CBool(False)
 
     exiter = Instance(ZMQExitAutocall)
     def _exiter_default(self):
         return ZMQExitAutocall(self)
     
     def _exit_now_changed(self, name, old, new):
         """stop eventloop when exit_now fires"""
         if new:
             loop = ioloop.IOLoop.instance()
             loop.add_timeout(time.time()+0.1, loop.stop)
 
     keepkernel_on_exit = None
 
     # Over ZeroMQ, GUI control isn't done with PyOS_InputHook as there is no
     # interactive input being read; we provide event loop support in ipkernel
     from .eventloops import enable_gui
     enable_gui = staticmethod(enable_gui)
 
     def init_environment(self):
         """Configure the user's environment.
 
         """
         env = os.environ
         # These two ensure 'ls' produces nice coloring on BSD-derived systems
         env['TERM'] = 'xterm-color'
         env['CLICOLOR'] = '1'
         # Since normal pagers don't work at all (over pexpect we don't have
         # single-key control of the subprocess), try to disable paging in
         # subprocesses as much as possible.
         env['PAGER'] = 'cat'
         env['GIT_PAGER'] = 'cat'
         
         # And install the payload version of page.
         install_payload_page()
 
     def auto_rewrite_input(self, cmd):
         """Called to show the auto-rewritten input for autocall and friends.
 
         FIXME: this payload is currently not correctly processed by the
         frontend.
         """
         new = self.prompt_manager.render('rewrite') + cmd
         payload = dict(
             source='IPython.zmq.zmqshell.ZMQInteractiveShell.auto_rewrite_input',
             transformed_input=new,
             )
         self.payload_manager.write_payload(payload)
 
     def ask_exit(self):
         """Engage the exit actions."""
         self.exit_now = True
         payload = dict(
             source='IPython.zmq.zmqshell.ZMQInteractiveShell.ask_exit',
             exit=True,
             keepkernel=self.keepkernel_on_exit,
             )
         self.payload_manager.write_payload(payload)
 
     def _showtraceback(self, etype, evalue, stb):
 
         exc_content = {
             u'traceback' : stb,
             u'ename' : unicode(etype.__name__),
             u'evalue' : unicode(evalue)
         }
 
         dh = self.displayhook
         # Send exception info over pub socket for other clients than the caller
         # to pick up
         topic = None
         if dh.topic:
             topic = dh.topic.replace(b'pyout', b'pyerr')
         
         exc_msg = dh.session.send(dh.pub_socket, u'pyerr', json_clean(exc_content), dh.parent_header, ident=topic)
 
         # FIXME - Hack: store exception info in shell object.  Right now, the
         # caller is reading this info after the fact, we need to fix this logic
         # to remove this hack.  Even uglier, we need to store the error status
         # here, because in the main loop, the logic that sets it is being
         # skipped because runlines swallows the exceptions.
         exc_content[u'status'] = u'error'
         self._reply_content = exc_content
         # /FIXME
 
         return exc_content
 
     #------------------------------------------------------------------------
     # Magic overrides
     #------------------------------------------------------------------------
     # Once the base class stops inheriting from magic, this code needs to be
     # moved into a separate machinery as well.  For now, at least isolate here
     # the magics which this class needs to implement differently from the base
     # class, or that are unique to it.
 
     def magic_doctest_mode(self,parameter_s=''):
         """Toggle doctest mode on and off.
 
         This mode is intended to make IPython behave as much as possible like a
         plain Python shell, from the perspective of how its prompts, exceptions
         and output look.  This makes it easy to copy and paste parts of a
         session into doctests.  It does so by:
 
         - Changing the prompts to the classic ``>>>`` ones.
         - Changing the exception reporting mode to 'Plain'.
         - Disabling pretty-printing of output.
 
         Note that IPython also supports the pasting of code snippets that have
         leading '>>>' and '...' prompts in them.  This means that you can paste
         doctests from files or docstrings (even if they have leading
         whitespace), and the code will execute correctly.  You can then use
         '%history -t' to see the translated history; this will give you the
         input after removal of all the leading prompts and whitespace, which
         can be pasted back into an editor.
 
         With these features, you can switch into this mode easily whenever you
         need to do testing and changes to doctests, without having to leave
         your existing IPython session.
         """
 
         from IPython.utils.ipstruct import Struct
 
         # Shorthands
         shell = self.shell
         disp_formatter = self.shell.display_formatter
         ptformatter = disp_formatter.formatters['text/plain']
         # dstore is a data store kept in the instance metadata bag to track any
         # changes we make, so we can undo them later.
         dstore = shell.meta.setdefault('doctest_mode', Struct())
         save_dstore = dstore.setdefault
 
         # save a few values we'll need to recover later
         mode = save_dstore('mode', False)
         save_dstore('rc_pprint', ptformatter.pprint)
         save_dstore('rc_plain_text_only',disp_formatter.plain_text_only)
         save_dstore('xmode', shell.InteractiveTB.mode)
 
         if mode == False:
             # turn on
             ptformatter.pprint = False
             disp_formatter.plain_text_only = True
             shell.magic_xmode('Plain')
         else:
             # turn off
             ptformatter.pprint = dstore.rc_pprint
             disp_formatter.plain_text_only = dstore.rc_plain_text_only
             shell.magic_xmode(dstore.xmode)
 
         # Store new mode and inform on console
         dstore.mode = bool(1-int(mode))
         mode_label = ['OFF','ON'][dstore.mode]
         print('Doctest mode is:', mode_label)
 
         # Send the payload back so that clients can modify their prompt display
         payload = dict(
             source='IPython.zmq.zmqshell.ZMQInteractiveShell.magic_doctest_mode',
             mode=dstore.mode)
         self.payload_manager.write_payload(payload)
 
     @skip_doctest
     def magic_edit(self,parameter_s='',last_call=['','']):
         """Bring up an editor and execute the resulting code.
 
         Usage:
           %edit [options] [args]
 
         %edit runs an external text editor. You will need to set the command for
         this editor via the ``TerminalInteractiveShell.editor`` option in your
         configuration file before it will work.
 
         This command allows you to conveniently edit multi-line code right in
         your IPython session.
 
         If called without arguments, %edit opens up an empty editor with a
         temporary file and will execute the contents of this file when you
         close it (don't forget to save it!).
 
 
         Options:
 
         -n <number>: open the editor at a specified line number.  By default,
         the IPython editor hook uses the unix syntax 'editor +N filename', but
         you can configure this by providing your own modified hook if your
         favorite editor supports line-number specifications with a different
         syntax.
 
         -p: this will call the editor with the same data as the previous time
         it was used, regardless of how long ago (in your current session) it
         was.
 
         -r: use 'raw' input.  This option only applies to input taken from the
         user's history.  By default, the 'processed' history is used, so that
         magics are loaded in their transformed version to valid Python.  If
         this option is given, the raw input as typed as the command line is
         used instead.  When you exit the editor, it will be executed by
         IPython's own processor.
 
         -x: do not execute the edited code immediately upon exit. This is
         mainly useful if you are editing programs which need to be called with
         command line arguments, which you can then do using %run.
 
 
         Arguments:
 
         If arguments are given, the following possibilites exist:
 
         - The arguments are numbers or pairs of colon-separated numbers (like
         1 4:8 9). These are interpreted as lines of previous input to be
         loaded into the editor. The syntax is the same of the %macro command.
 
         - If the argument doesn't start with a number, it is evaluated as a
         variable and its contents loaded into the editor. You can thus edit
         any string which contains python code (including the result of
         previous edits).
 
         - If the argument is the name of an object (other than a string),
         IPython will try to locate the file where it was defined and open the
         editor at the point where it is defined. You can use `%edit function`
         to load an editor exactly at the point where 'function' is defined,
         edit it and have the file be executed automatically.
 
         If the object is a macro (see %macro for details), this opens up your
         specified editor with a temporary file containing the macro's data.
         Upon exit, the macro is reloaded with the contents of the file.
 
         Note: opening at an exact line is only supported under Unix, and some
         editors (like kedit and gedit up to Gnome 2.8) do not understand the
         '+NUMBER' parameter necessary for this feature. Good editors like
         (X)Emacs, vi, jed, pico and joe all do.
 
         - If the argument is not found as a variable, IPython will look for a
         file with that name (adding .py if necessary) and load it into the
         editor. It will execute its contents with execfile() when you exit,
         loading any code in the file into your interactive namespace.
 
         After executing your code, %edit will return as output the code you
         typed in the editor (except when it was an existing file). This way
         you can reload the code in further invocations of %edit as a variable,
         via _<NUMBER> or Out[<NUMBER>], where <NUMBER> is the prompt number of
         the output.
 
         Note that %edit is also available through the alias %ed.
 
         This is an example of creating a simple function inside the editor and
         then modifying it. First, start up the editor:
 
         In [1]: ed
         Editing... done. Executing edited code...
         Out[1]: 'def foo():n    print "foo() was defined in an editing session"n'
 
         We can then call the function foo():
 
         In [2]: foo()
         foo() was defined in an editing session
 
         Now we edit foo.  IPython automatically loads the editor with the
         (temporary) file where foo() was previously defined:
 
         In [3]: ed foo
         Editing... done. Executing edited code...
 
         And if we call foo() again we get the modified version:
 
         In [4]: foo()
         foo() has now been changed!
 
         Here is an example of how to edit a code snippet successive
         times. First we call the editor:
 
         In [5]: ed
         Editing... done. Executing edited code...
         hello
         Out[5]: "print 'hello'n"
 
         Now we call it again with the previous output (stored in _):
 
         In [6]: ed _
         Editing... done. Executing edited code...
         hello world
         Out[6]: "print 'hello world'n"
 
         Now we call it with the output #8 (stored in _8, also as Out[8]):
 
         In [7]: ed _8
         Editing... done. Executing edited code...
         hello again
         Out[7]: "print 'hello again'n"
         """
 
         opts,args = self.parse_options(parameter_s,'prn:')
 
         try:
             filename, lineno, _ = self._find_edit_target(args, opts, last_call)
         except MacroToEdit as e:
             # TODO: Implement macro editing over 2 processes.
             print("Macro editing not yet implemented in 2-process model.")
             return
 
         # Make sure we send to the client an absolute path, in case the working
         # directory of client and kernel don't match
         filename = os.path.abspath(filename)
 
         payload = {
             'source' : 'IPython.zmq.zmqshell.ZMQInteractiveShell.edit_magic',
             'filename' : filename,
             'line_number' : lineno
         }
         self.payload_manager.write_payload(payload)
 
     # A few magics that are adapted to the specifics of using pexpect and a
     # remote terminal
 
     def magic_clear(self, arg_s):
         """Clear the terminal."""
         if os.name == 'posix':
             self.shell.system("clear")
         else:
             self.shell.system("cls")
 
     if os.name == 'nt':
         # This is the usual name in windows
         magic_cls = magic_clear
 
     # Terminal pagers won't work over pexpect, but we do have our own pager
 
     def magic_less(self, arg_s):
         """Show a file through the pager.
 
         Files ending in .py are syntax-highlighted."""
         cont = open(arg_s).read()
         if arg_s.endswith('.py'):
             cont = self.shell.pycolorize(cont)
         page.page(cont)
 
     magic_more = magic_less
 
     # Man calls a pager, so we also need to redefine it
     if os.name == 'posix':
         def magic_man(self, arg_s):
             """Find the man page for the given command and display in pager."""
             page.page(self.shell.getoutput('man %s | col -b' % arg_s,
                                            split=False))
 
     # FIXME: this is specific to the GUI, so we should let the gui app load
     # magics at startup that are only for the gui.  Once the gui app has proper
     # profile and configuration management, we can have it initialize a kernel
     # with a special config file that provides these.
     def magic_guiref(self, arg_s):
         """Show a basic reference about the GUI console."""
         from IPython.core.usage import gui_reference
         page.page(gui_reference, auto_html=True)
     
     def magic_connect_info(self, arg_s):
         """Print information for connecting other clients to this kernel
         
         It will print the contents of this session's connection file, as well as
         shortcuts for local clients.
         
         In the simplest case, when called from the most recently launched kernel,
         secondary clients can be connected, simply with:
         
         $> ipython <app> --existing
         
         """
         
         from IPython.core.application import BaseIPythonApplication as BaseIPApp
         
         if BaseIPApp.initialized():
             app = BaseIPApp.instance()
             security_dir = app.profile_dir.security_dir
             profile = app.profile
         else:
             profile = 'default'
             security_dir = ''
         
         try:
             connection_file = get_connection_file()
             info = get_connection_info(unpack=False)
         except Exception as e:
             error("Could not get connection info: %r" % e)
             return
         
         # add profile flag for non-default profile
         profile_flag = "--profile %s" % profile if profile != 'default' else ""
         
         # if it's in the security dir, truncate to basename
         if security_dir == os.path.dirname(connection_file):
             connection_file = os.path.basename(connection_file)
         
         
         print (info + '\n')
         print ("Paste the above JSON into a file, and connect with:\n"
             "    $> ipython <app> --existing <file>\n"
             "or, if you are local, you can connect with just:\n"
             "    $> ipython <app> --existing {0} {1}\n"
             "or even just:\n"
             "    $> ipython <app> --existing {1}\n"
             "if this is the most recent IPython session you have started.".format(
             connection_file, profile_flag
             )
         )
 
     def magic_qtconsole(self, arg_s):
         """Open a qtconsole connected to this kernel.
         
         Useful for connecting a qtconsole to running notebooks, for better
         debugging.
         """
         try:
             p = connect_qtconsole(argv=arg_split(arg_s, os.name=='posix'))
         except Exception as e:
             error("Could not start qtconsole: %r" % e)
             return
 
     def set_next_input(self, text):
         """Send the specified text to the frontend to be presented at the next
         input cell."""
         payload = dict(
             source='IPython.zmq.zmqshell.ZMQInteractiveShell.set_next_input',
             text=text
         )
         self.payload_manager.write_payload(payload)
 
 
 InteractiveShellABC.register(ZMQInteractiveShell)

docs/source/parallel/parallel_mpi.txt

@@ -1,151 +1,149 b''
 .. _parallelmpi:
 
 =======================
 Using MPI with IPython
 =======================
 
 Often, a parallel algorithm will require moving data between the engines. One
 way of accomplishing this is by doing a pull and then a push using the
 multiengine client. However, this will be slow as all the data has to go
 through the controller to the client and then back through the controller, to
 its final destination.
 
 A much better way of moving data between engines is to use a message passing
 library, such as the Message Passing Interface (MPI) [MPI]_. IPython's
 parallel computing architecture has been designed from the ground up to
 integrate with MPI. This document describes how to use MPI with IPython.
 
 Additional installation requirements
 ====================================
 
 If you want to use MPI with IPython, you will need to install:
 
 * A standard MPI implementation such as OpenMPI [OpenMPI]_ or MPICH.
 * The mpi4py [mpi4py]_ package.
 
 .. note::
 
     The mpi4py package is not a strict requirement. However, you need to
     have *some* way of calling MPI from Python. You also need some way of
     making sure that :func:`MPI_Init` is called when the IPython engines start
     up. There are a number of ways of doing this and a good number of
     associated subtleties. We highly recommend just using mpi4py as it
     takes care of most of these problems. If you want to do something
     different, let us know and we can help you get started.
 
 Starting the engines with MPI enabled
 =====================================
 
 To use code that calls MPI, there are typically two things that MPI requires.
 
 1. The process that wants to call MPI must be started using
    :command:`mpiexec` or a batch system (like PBS) that has MPI support.
 2. Once the process starts, it must call :func:`MPI_Init`.
 
 There are a couple of ways that you can start the IPython engines and get
 these things to happen.
 
 Automatic starting using :command:`mpiexec` and :command:`ipcluster`
 --------------------------------------------------------------------
 
 The easiest approach is to use the `MPI` Launchers in :command:`ipcluster`,
 which will first start a controller and then a set of engines using
 :command:`mpiexec`::
 
     $ ipcluster start -n 4 --engines=MPIEngineSetLauncher
 
 This approach is best as interrupting :command:`ipcluster` will automatically
 stop and clean up the controller and engines.
 
 Manual starting using :command:`mpiexec`
 ----------------------------------------
 
 If you want to start the IPython engines using the :command:`mpiexec`, just
 do::
 
     $ mpiexec -n 4 ipengine --mpi=mpi4py
 
 This requires that you already have a controller running and that the FURL
 files for the engines are in place. We also have built in support for
 PyTrilinos [PyTrilinos]_, which can be used (assuming is installed) by
 starting the engines with::
 
     $ mpiexec -n 4 ipengine --mpi=pytrilinos
 
 Automatic starting using PBS and :command:`ipcluster`
 ------------------------------------------------------
 
 The :command:`ipcluster` command also has built-in integration with PBS. For
 more information on this approach, see our documentation on :ref:`ipcluster
 <parallel_process>`.
 
 Actually using MPI
 ==================
 
 Once the engines are running with MPI enabled, you are ready to go. You can
 now call any code that uses MPI in the IPython engines. And, all of this can
 be done interactively. Here we show a simple example that uses mpi4py
 [mpi4py]_ version 1.1.0 or later.
 
 First, lets define a simply function that uses MPI to calculate the sum of a
 distributed array. Save the following text in a file called :file:`psum.py`:
 
 .. sourcecode:: python
 
     from mpi4py import MPI
     import numpy as np
 
     def psum(a):
         s = np.sum(a)
         rcvBuf = np.array(0.0,'d')
         MPI.COMM_WORLD.Allreduce([s, MPI.DOUBLE],
             [rcvBuf, MPI.DOUBLE],
             op=MPI.SUM)
         return rcvBuf
 
 Now, start an IPython cluster::
 
     $ ipcluster start --profile=mpi -n 4
     
 .. note::
 
     It is assumed here that the mpi profile has been set up, as described :ref:`here
     <parallel_process>`.
 
 Finally, connect to the cluster and use this function interactively. In this
 case, we create a random array on each engine and sum up all the random arrays
 using our :func:`psum` function:
 
 .. sourcecode:: ipython
 
     In [1]: from IPython.parallel import Client
     
-    In [2]: %load_ext parallel_magic
-    
     In [3]: c = Client(profile='mpi')
     
     In [4]: view = c[:]
     
-    In [5]: view.activate()
+    In [5]: view.activate() # enabe magics
     
     # run the contents of the file on each engine:
     In [6]: view.run('psum.py')
     
-    In [6]: px a = np.random.rand(100)
+    In [6]: %px a = np.random.rand(100)
     Parallel execution on engines: [0,1,2,3]
     
-    In [8]: px s = psum(a)
+    In [8]: %px s = psum(a)
     Parallel execution on engines: [0,1,2,3]
     
     In [9]: view['s']
     Out[9]: [187.451545803,187.451545803,187.451545803,187.451545803]
 
 Any Python code that makes calls to MPI can be used in this manner, including
 compiled C, C++ and Fortran libraries that have been exposed to Python.
 
 .. [MPI] Message Passing Interface.  http://www-unix.mcs.anl.gov/mpi/
 .. [mpi4py] MPI for Python. mpi4py: http://mpi4py.scipy.org/
 .. [OpenMPI] Open MPI. http://www.open-mpi.org/
 .. [PyTrilinos] PyTrilinos. http://trilinos.sandia.gov/packages/pytrilinos/
 
 

docs/source/parallel/parallel_multiengine.txt

@@ -1,865 +1,942 b''
 .. _parallel_multiengine:
 
 ==========================
 IPython's Direct interface
 ==========================
 
 The direct, or multiengine, interface represents one possible way of working with a set of
 IPython engines. The basic idea behind the multiengine interface is that the
 capabilities of each engine are directly and explicitly exposed to the user.
 Thus, in the multiengine interface, each engine is given an id that is used to
 identify the engine and give it work to do. This interface is very intuitive
 and is designed with interactive usage in mind, and is the best place for
 new users of IPython to begin.
 
 Starting the IPython controller and engines
 ===========================================
 
 To follow along with this tutorial, you will need to start the IPython
 controller and four IPython engines. The simplest way of doing this is to use
 the :command:`ipcluster` command::
 
     $ ipcluster start -n 4
     
 For more detailed information about starting the controller and engines, see
 our :ref:`introduction <parallel_overview>` to using IPython for parallel computing.
 
 Creating a ``DirectView`` instance
 ==================================
 
 The first step is to import the IPython :mod:`IPython.parallel`
 module and then create a :class:`.Client` instance:
 
 .. sourcecode:: ipython
 
     In [1]: from IPython.parallel import Client
     
     In [2]: rc = Client()
 
 This form assumes that the default connection information (stored in
 :file:`ipcontroller-client.json` found in :file:`IPYTHONDIR/profile_default/security`) is
 accurate. If the controller was started on a remote machine, you must copy that connection
 file to the client machine, or enter its contents as arguments to the Client constructor:
 
 .. sourcecode:: ipython
 
     # If you have copied the json connector file from the controller:
     In [2]: rc = Client('/path/to/ipcontroller-client.json')
     # or to connect with a specific profile you have set up:
     In [3]: rc = Client(profile='mpi')
     
 
 To make sure there are engines connected to the controller, users can get a list
 of engine ids:
 
 .. sourcecode:: ipython
 
     In [3]: rc.ids
     Out[3]: [0, 1, 2, 3]
 
 Here we see that there are four engines ready to do work for us.
 
 For direct execution, we will make use of a :class:`DirectView` object, which can be
 constructed via list-access to the client:
 
 .. sourcecode:: ipython
 
     In [4]: dview = rc[:] # use all engines
 
 .. seealso::
 
     For more information, see the in-depth explanation of :ref:`Views <parallel_details>`.
 
 
 Quick and easy parallelism
 ==========================
 
 In many cases, you simply want to apply a Python function to a sequence of
 objects, but *in parallel*. The client interface provides a simple way
 of accomplishing this: using the DirectView's :meth:`~DirectView.map` method.
 
 Parallel map
 ------------
 
 Python's builtin :func:`map` functions allows a function to be applied to a
 sequence element-by-element. This type of code is typically trivial to
 parallelize. In fact, since IPython's interface is all about functions anyway,
 you can just use the builtin :func:`map` with a :class:`RemoteFunction`, or a 
 DirectView's :meth:`map` method:
 
 .. sourcecode:: ipython
 
     In [62]: serial_result = map(lambda x:x**10, range(32))
     
     In [63]: parallel_result = dview.map_sync(lambda x: x**10, range(32))
 
     In [67]: serial_result==parallel_result
     Out[67]: True
 
 
 .. note::
 
     The :class:`DirectView`'s version of :meth:`map` does
     not do dynamic load balancing. For a load balanced version, use a
     :class:`LoadBalancedView`.
 
 .. seealso::
     
     :meth:`map` is implemented via :class:`ParallelFunction`.
 
 Remote function decorators
 --------------------------
 
 Remote functions are just like normal functions, but when they are called,
 they execute on one or more engines, rather than locally. IPython provides
 two decorators:
 
 .. sourcecode:: ipython
 
     In [10]: @dview.remote(block=True)
        ....: def getpid():
        ....:     import os
        ....:     return os.getpid()
        ....: 
 
     In [11]: getpid()
     Out[11]: [12345, 12346, 12347, 12348]
 
 The ``@parallel`` decorator creates parallel functions, that break up an element-wise
 operations and distribute them, reconstructing the result.
 
 .. sourcecode:: ipython
 
     In [12]: import numpy as np
     
     In [13]: A = np.random.random((64,48))
     
     In [14]: @dview.parallel(block=True)
        ....: def pmul(A,B):
        ....:     return A*B
     
     In [15]: C_local = A*A
     
     In [16]: C_remote = pmul(A,A)
     
     In [17]: (C_local == C_remote).all()
     Out[17]: True
 
 Calling a ``@parallel`` function *does not* correspond to map. It is used for splitting
 element-wise operations that operate on a sequence or array.  For ``map`` behavior,
 parallel functions do have a map method.
 
 ====================    ============================    =============================
 call                    pfunc(seq)                      pfunc.map(seq)
 ====================    ============================    =============================
 # of tasks              # of engines (1 per engine)     # of engines (1 per engine)
 # of remote calls       # of engines (1 per engine)     ``len(seq)``
 argument to remote      ``seq[i:j]`` (sub-sequence)     ``seq[i]`` (single element)
 ====================    ============================    =============================
 
 A quick example to illustrate the difference in arguments for the two modes:
 
 .. sourcecode:: ipython
 
     In [16]: @dview.parallel(block=True)
        ....: def echo(x):
        ....:     return str(x)
        ....: 
 
     In [17]: echo(range(5))
     Out[17]: ['[0, 1]', '[2]', '[3]', '[4]']
 
     In [18]: echo.map(range(5))
     Out[18]: ['0', '1', '2', '3', '4']
 
 
 .. seealso::
 
     See the :func:`~.remotefunction.parallel` and :func:`~.remotefunction.remote`
     decorators for options.
 
 Calling Python functions
 ========================
 
 The most basic type of operation that can be performed on the engines is to
 execute Python code or call Python functions. Executing Python code can be
 done in blocking or non-blocking mode (non-blocking is default) using the
 :meth:`.View.execute` method, and calling functions can be done via the
 :meth:`.View.apply` method.
 
 apply
 -----
 
 The main method for doing remote execution (in fact, all methods that
 communicate with the engines are built on top of it), is :meth:`View.apply`.
 
 We strive to provide the cleanest interface we can, so `apply` has the following
 signature:
 
 .. sourcecode:: python
 
     view.apply(f, *args, **kwargs)
 
 There are various ways to call functions with IPython, and these flags are set as
 attributes of the View.  The ``DirectView`` has just two of these flags:
 
 dv.block : bool
     whether to wait for the result, or return an :class:`AsyncResult` object
     immediately
 dv.track : bool
     whether to instruct pyzmq to track when zeromq is done sending the message.
     This is primarily useful for non-copying sends of numpy arrays that you plan to
     edit in-place.  You need to know when it becomes safe to edit the buffer
     without corrupting the message.
 dv.targets : int, list of ints
     which targets this view is associated with.
 
 
 Creating a view is simple: index-access on a client creates a :class:`.DirectView`.
 
 .. sourcecode:: ipython
 
     In [4]: view = rc[1:3]
     Out[4]: <DirectView [1, 2]>
 
     In [5]: view.apply<tab>
     view.apply  view.apply_async  view.apply_sync
 
 For convenience, you can set block temporarily for a single call with the extra sync/async methods.
 
 Blocking execution
 ------------------
 
 In blocking mode, the :class:`.DirectView` object (called ``dview`` in
 these examples) submits the command to the controller, which places the
 command in the engines' queues for execution. The :meth:`apply` call then
 blocks until the engines are done executing the command:
 
 .. sourcecode:: ipython
 
     In [2]: dview = rc[:] # A DirectView of all engines
     In [3]: dview.block=True
     In [4]: dview['a'] = 5
 
     In [5]: dview['b'] = 10
 
     In [6]: dview.apply(lambda x: a+b+x, 27)
     Out[6]: [42, 42, 42, 42]
 
 You can also select blocking execution on a call-by-call basis with the :meth:`apply_sync`
 method:
 
     In [7]: dview.block=False
 
     In [8]: dview.apply_sync(lambda x: a+b+x, 27)
     Out[8]: [42, 42, 42, 42]
 
 Python commands can be executed as strings on specific engines by using a View's ``execute``
 method:
 
 .. sourcecode:: ipython
 
     In [6]: rc[::2].execute('c=a+b')
 
     In [7]: rc[1::2].execute('c=a-b')
 
     In [8]: dview['c'] # shorthand for dview.pull('c', block=True)
     Out[8]: [15, -5, 15, -5]
 
 
 Non-blocking execution
 ----------------------
 
 In non-blocking mode, :meth:`apply` submits the command to be executed and
 then returns a :class:`AsyncResult` object immediately. The
 :class:`AsyncResult` object gives you a way of getting a result at a later
 time through its :meth:`get` method.
 
 .. seealso::
 
     Docs on the :ref:`AsyncResult <parallel_asyncresult>` object.
 
 This allows you to quickly submit long running commands without blocking your
 local Python/IPython session:
 
 .. sourcecode:: ipython
     
     # define our function
     In [6]: def wait(t):
       ....:     import time
       ....:     tic = time.time()
       ....:     time.sleep(t)
       ....:     return time.time()-tic
     
     # In non-blocking mode
     In [7]: ar = dview.apply_async(wait, 2)
 
     # Now block for the result
     In [8]: ar.get()
     Out[8]: [2.0006198883056641, 1.9997570514678955, 1.9996809959411621, 2.0003249645233154]
 
     # Again in non-blocking mode
     In [9]: ar = dview.apply_async(wait, 10)
 
     # Poll to see if the result is ready
     In [10]: ar.ready()
     Out[10]: False
     
     # ask for the result, but wait a maximum of 1 second:
     In [45]: ar.get(1)
     ---------------------------------------------------------------------------
     TimeoutError                              Traceback (most recent call last)
     /home/you/<ipython-input-45-7cd858bbb8e0> in <module>()
     ----> 1 ar.get(1)
 
     /path/to/site-packages/IPython/parallel/asyncresult.pyc in get(self, timeout)
          62                 raise self._exception
          63         else:
     ---> 64             raise error.TimeoutError("Result not ready.")
          65 
          66     def ready(self):
 
     TimeoutError: Result not ready.
 
 .. Note::
 
     Note the import inside the function. This is a common model, to ensure
     that the appropriate modules are imported where the task is run. You can
     also manually import modules into the engine(s) namespace(s) via 
     :meth:`view.execute('import numpy')`.
 
 Often, it is desirable to wait until a set of :class:`AsyncResult` objects
 are done. For this, there is a the method :meth:`wait`. This method takes a
 tuple of :class:`AsyncResult` objects (or `msg_ids` or indices to the client's History),
 and blocks until all of the associated results are ready:
 
 .. sourcecode:: ipython
 
     In [72]: dview.block=False
 
     # A trivial list of AsyncResults objects
     In [73]: pr_list = [dview.apply_async(wait, 3) for i in range(10)]
 
     # Wait until all of them are done
     In [74]: dview.wait(pr_list)
 
     # Then, their results are ready using get() or the `.r` attribute
     In [75]: pr_list[0].get()
     Out[75]: [2.9982571601867676, 2.9982588291168213, 2.9987530708312988, 2.9990990161895752]
 
 
 
 The ``block`` and ``targets`` keyword arguments and attributes
 --------------------------------------------------------------
 
 Most DirectView methods (excluding :meth:`apply`) accept ``block`` and
 ``targets`` as keyword arguments. As we have seen above, these keyword arguments control the
 blocking mode and which engines the command is applied to. The :class:`View` class also has
 :attr:`block` and :attr:`targets` attributes that control the default behavior when the keyword
 arguments are not provided. Thus the following logic is used for :attr:`block` and :attr:`targets`:
 
 * If no keyword argument is provided, the instance attributes are used.
 * Keyword argument, if provided override the instance attributes for
   the duration of a single call.
   
 The following examples demonstrate how to use the instance attributes:
 
 .. sourcecode:: ipython
 
     In [16]: dview.targets = [0,2]
     
     In [17]: dview.block = False
 
     In [18]: ar = dview.apply(lambda : 10)
 
     In [19]: ar.get()
     Out[19]: [10, 10]
 
     In [16]: dview.targets = v.client.ids # all engines (4)
     
     In [21]: dview.block = True
 
     In [22]: dview.apply(lambda : 42)
     Out[22]: [42, 42, 42, 42]
 
 The :attr:`block` and :attr:`targets` instance attributes of the
 :class:`.DirectView` also determine the behavior of the parallel magic commands.
 
 Parallel magic commands
 -----------------------
 
 We provide a few IPython magic commands (``%px``, ``%autopx`` and ``%result``)
-that make it more pleasant to execute Python commands on the engines
-interactively. These are simply shortcuts to :meth:`execute` and
-:meth:`get_result` of the :class:`DirectView`. The ``%px`` magic executes a single 
-Python command on the engines specified by the :attr:`targets` attribute of the
-:class:`DirectView` instance:
+that make it a bit more pleasant to execute Python commands on the engines interactively.
+These are simply shortcuts to :meth:`.DirectView.execute`
+and :meth:`.AsyncResult.display_outputs` methods repsectively.
+The ``%px`` magic executes a single Python command on the engines
+specified by the :attr:`targets` attribute of the :class:`DirectView` instance:
 
 .. sourcecode:: ipython
 
     # Create a DirectView for all targets
     In [22]: dv = rc[:]
     
     # Make this DirectView active for parallel magic commands
     In [23]: dv.activate()
 
     In [24]: dv.block=True
 
     # import numpy here and everywhere
     In [25]: with dv.sync_imports():
        ....:    import numpy
     importing numpy on engine(s)
 
     In [27]: %px a = numpy.random.rand(2,2)
     Parallel execution on engines: [0, 1, 2, 3]
 
-    In [28]: %px ev = numpy.linalg.eigvals(a)
+    In [28]: %px numpy.linalg.eigvals(a)
     Parallel execution on engines: [0, 1, 2, 3]
+    [0] Out[68]: array([ 0.77120707, -0.19448286])
+    [1] Out[68]: array([ 1.10815921,  0.05110369])
+    [2] Out[68]: array([ 0.74625527, -0.37475081])
+    [3] Out[68]: array([ 0.72931905,  0.07159743])
 
-    In [28]: dv['ev']
-    Out[28]: [ array([ 1.09522024, -0.09645227]), 
-       ....:   array([ 1.21435496, -0.35546712]), 
-       ....:   array([ 0.72180653,  0.07133042]), 
-       ....:   array([  1.46384341,   1.04353244e-04])
-       ....: ]
+    In [29]: %px print 'hi'
+    Parallel execution on engine(s): [0, 1, 2, 3]
+    [stdout:0] hi
+    [stdout:1] hi
+    [stdout:2] hi
+    [stdout:3] hi
 
-The ``%result`` magic gets the most recent result, or takes an argument
-specifying the index of the result to be requested. It is simply a shortcut to the
-:meth:`get_result` method:
+
+Since engines are IPython as well, you can even run magics remotely:
+
+.. sourcecode:: ipython
+
+    In [28]: %px %pylab inline
+    Parallel execution on engine(s): [0, 1, 2, 3]
+    [stdout:0] 
+    Welcome to pylab, a matplotlib-based Python environment...
+    For more information, type 'help(pylab)'.
+    [stdout:1] 
+    Welcome to pylab, a matplotlib-based Python environment...
+    For more information, type 'help(pylab)'.
+    [stdout:2] 
+    Welcome to pylab, a matplotlib-based Python environment...
+    For more information, type 'help(pylab)'.
+    [stdout:3] 
+    Welcome to pylab, a matplotlib-based Python environment...
+    For more information, type 'help(pylab)'.
+
+And once in pylab mode with the inline backend,
+you can make plots and they will be displayed in your frontend
+if it suports the inline figures (e.g. notebook or qtconsole):
 
 .. sourcecode:: ipython
     
-    In [29]: dv.apply_async(lambda : ev)
+    In [40]: %px plot(rand(100))
+    Parallel execution on engine(s): [0, 1, 2, 3]
+    <plot0>
+    <plot1>
+    <plot2>
+    <plot3>
+    [0] Out[79]: [<matplotlib.lines.Line2D at 0x10a6286d0>]
+    [1] Out[79]: [<matplotlib.lines.Line2D at 0x10b9476d0>]
+    [2] Out[79]: [<matplotlib.lines.Line2D at 0x110652750>]
+    [3] Out[79]: [<matplotlib.lines.Line2D at 0x10c6566d0>]
+
+
+``%%px`` Cell Magic
+*******************
+
+`%%px` can also be used as a Cell Magic, which accepts ``--[no]block`` flags,
+and a ``--group-outputs`` argument, which adjust how the outputs of multiple
+engines are presented.
     
-    In [30]: %result
-    Out[30]: [ [ 1.28167017  0.14197338],
-       ....:    [-0.14093616  1.27877273],
-       ....:    [-0.37023573  1.06779409],
-       ....:    [ 0.83664764 -0.25602658] ]
+.. seealso::
+
+    :meth:`.AsyncResult.display_outputs` for the grouping options.
+
+.. sourcecode:: ipython
+
+    In [50]: %%px --block --group-outputs=engine
+       ....: import numpy as np
+       ....: A = np.random.random((2,2))
+       ....: ev = numpy.linalg.eigvals(A)
+       ....: print ev
+       ....: ev.max()
+       ....:
+    Parallel execution on engine(s): [0, 1, 2, 3]
+    [stdout:0] [ 0.60640442  0.95919621]
+    [0] Out[73]: 0.9591962130899806
+    [stdout:1] [ 0.38501813  1.29430871]
+    [1] Out[73]: 1.2943087091452372
+    [stdout:2] [-0.85925141  0.9387692 ]
+    [2] Out[73]: 0.93876920456230284
+    [stdout:3] [ 0.37998269  1.24218246]
+    [3] Out[73]: 1.2421824618493817
+
+``%result`` Magic
+*****************
+
+If you are using ``%px`` in non-blocking mode, you won't get output.
+You can use ``%result`` to display the outputs of the latest command,
+just as is done when ``%px`` is blocking:
+
+.. sourcecode:: ipython
+
+    In [39]: dv.block = False
+    
+    In [40]: %px print 'hi'
+    Async parallel execution on engine(s): [0, 1, 2, 3]
+    
+    In [41]: %result
+    [stdout:0] hi
+    [stdout:1] hi
+    [stdout:2] hi
+    [stdout:3] hi
+
+``%result`` simply calls :meth:`.AsyncResult.display_outputs` on the most recent request.
+You can pass integers as indices if you want a result other than the latest,
+e.g. ``%result -2``, or ``%result 0`` for the first.
+
+
+``%autopx``
+***********
 
 The ``%autopx`` magic switches to a mode where everything you type is executed
-on the engines given by the :attr:`targets` attribute:
+on the engines until you do ``%autopx`` again.
 
 .. sourcecode:: ipython
 
-    In [30]: dv.block=False
+    In [30]: dv.block=True
 
     In [31]: %autopx
-    Auto Parallel Enabled
-    Type %autopx to disable
+    %autopx enabled
 
     In [32]: max_evals = []
-    <IPython.parallel.AsyncResult object at 0x17b8a70>
 
     In [33]: for i in range(100):
        ....:     a = numpy.random.rand(10,10)
        ....:     a = a+a.transpose()
        ....:     evals = numpy.linalg.eigvals(a)
        ....:     max_evals.append(evals[0].real)
        ....:
-       ....:
-    <IPython.parallel.AsyncResult object at 0x17af8f0>
-
-    In [34]: %autopx
-    Auto Parallel Disabled
 
-    In [35]: dv.block=True
+    In [34]: print "Average max eigenvalue is: %f" % (sum(max_evals)/len(max_evals))
+    [stdout:0] Average max eigenvalue is: 10.193101
+    [stdout:1] Average max eigenvalue is: 10.064508
+    [stdout:2] Average max eigenvalue is: 10.055724
+    [stdout:3] Average max eigenvalue is: 10.086876
 
-    In [36]: px ans= "Average max eigenvalue is: %f"%(sum(max_evals)/len(max_evals))
-    Parallel execution on engines: [0, 1, 2, 3]
-    
-    In [37]: dv['ans']
-    Out[37]: [ 'Average max eigenvalue is:  10.1387247332',
-       ....:   'Average max eigenvalue is:  10.2076902286',
-       ....:   'Average max eigenvalue is:  10.1891484655',
-       ....:   'Average max eigenvalue is:  10.1158837784',]
+    In [35]: %autopx
+    Auto Parallel Disabled
 
 
 Moving Python objects around
 ============================
 
 In addition to calling functions and executing code on engines, you can
 transfer Python objects to and from your IPython session and the engines. In
 IPython, these operations are called :meth:`push` (sending an object to the
 engines) and :meth:`pull` (getting an object from the engines).
 
 Basic push and pull
 -------------------
 
 Here are some examples of how you use :meth:`push` and :meth:`pull`:
 
 .. sourcecode:: ipython
 
     In [38]: dview.push(dict(a=1.03234,b=3453))
     Out[38]: [None,None,None,None]
 
     In [39]: dview.pull('a')
     Out[39]: [ 1.03234, 1.03234, 1.03234, 1.03234]
 
     In [40]: dview.pull('b', targets=0)
     Out[40]: 3453
 
     In [41]: dview.pull(('a','b'))
     Out[41]: [ [1.03234, 3453], [1.03234, 3453], [1.03234, 3453], [1.03234, 3453] ]
     
     In [43]: dview.push(dict(c='speed'))
     Out[43]: [None,None,None,None]
 
 In non-blocking mode :meth:`push` and :meth:`pull` also return
 :class:`AsyncResult` objects:
 
 .. sourcecode:: ipython
 
     In [48]: ar = dview.pull('a', block=False)
 
     In [49]: ar.get()
     Out[49]: [1.03234, 1.03234, 1.03234, 1.03234]
 
 
 Dictionary interface
 --------------------
 
 Since a Python namespace is just a :class:`dict`, :class:`DirectView` objects provide
 dictionary-style access by key and methods such as :meth:`get` and
 :meth:`update` for convenience. This make the remote namespaces of the engines
 appear as a local dictionary. Underneath, these methods call :meth:`apply`:
 
 .. sourcecode:: ipython
 
     In [51]: dview['a']=['foo','bar']
 
     In [52]: dview['a']
     Out[52]: [ ['foo', 'bar'], ['foo', 'bar'], ['foo', 'bar'], ['foo', 'bar'] ]
 
 Scatter and gather
 ------------------
 
 Sometimes it is useful to partition a sequence and push the partitions to
 different engines. In MPI language, this is know as scatter/gather and we
 follow that terminology. However, it is important to remember that in
 IPython's :class:`Client` class, :meth:`scatter` is from the
 interactive IPython session to the engines and :meth:`gather` is from the
 engines back to the interactive IPython session. For scatter/gather operations
 between engines, MPI, pyzmq, or some other direct interconnect should be used.
 
 .. sourcecode:: ipython
 
     In [58]: dview.scatter('a',range(16))
     Out[58]: [None,None,None,None]
 
     In [59]: dview['a']
     Out[59]: [ [0, 1, 2, 3], [4, 5, 6, 7], [8, 9, 10, 11], [12, 13, 14, 15] ]
 
     In [60]: dview.gather('a')
     Out[60]: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]
 
 Other things to look at
 =======================
 
 How to do parallel list comprehensions
 --------------------------------------
 
 In many cases list comprehensions are nicer than using the map function. While
 we don't have fully parallel list comprehensions, it is simple to get the
 basic effect using :meth:`scatter` and :meth:`gather`:
 
 .. sourcecode:: ipython
 
     In [66]: dview.scatter('x',range(64))
 
     In [67]: %px y = [i**10 for i in x]
     Parallel execution on engines: [0, 1, 2, 3]
     Out[67]:
 
     In [68]: y = dview.gather('y')
 
     In [69]: print y
     [0, 1, 1024, 59049, 1048576, 9765625, 60466176, 282475249, 1073741824,...]
 
 Remote imports
 --------------
 
 Sometimes you will want to import packages both in your interactive session
 and on your remote engines.  This can be done with the :class:`ContextManager`
 created by a DirectView's :meth:`sync_imports` method:
 
 .. sourcecode:: ipython
 
     In [69]: with dview.sync_imports():
        ....:     import numpy
     importing numpy on engine(s)
 
 Any imports made inside the block will also be performed on the view's engines.
 sync_imports also takes a `local` boolean flag that defaults to True, which specifies
 whether the local imports should also be performed.  However, support for `local=False`
 has not been implemented, so only packages that can be imported locally will work
 this way.
 
 You can also specify imports via the ``@require`` decorator.  This is a decorator
 designed for use in Dependencies, but can be used to handle remote imports as well.
 Modules or module names passed to ``@require`` will be imported before the decorated
 function is called.  If they cannot be imported, the decorated function will never
 execution, and will fail with an UnmetDependencyError.
 
 .. sourcecode:: ipython
 
     In [69]: from IPython.parallel import require
 
     In [70]: @require('re'):
        ....: def findall(pat, x):
        ....:     # re is guaranteed to be available
        ....:     return re.findall(pat, x)
           
     # you can also pass modules themselves, that you already have locally:
     In [71]: @require(time):
        ....: def wait(t):
        ....:     time.sleep(t)
        ....:     return t
 
 .. _parallel_exceptions:
 
 Parallel exceptions
 -------------------
 
 In the multiengine interface, parallel commands can raise Python exceptions,
 just like serial commands. But, it is a little subtle, because a single
 parallel command can actually raise multiple exceptions (one for each engine
 the command was run on). To express this idea, we have a
 :exc:`CompositeError` exception class that will be raised in most cases. The
 :exc:`CompositeError` class is a special type of exception that wraps one or
 more other types of exceptions. Here is how it works:
 
 .. sourcecode:: ipython
 
     In [76]: dview.block=True
 
     In [77]: dview.execute('1/0')
     ---------------------------------------------------------------------------
     CompositeError                            Traceback (most recent call last)
     /home/user/<ipython-input-10-5d56b303a66c> in <module>()
     ----> 1 dview.execute('1/0')
 
     /path/to/site-packages/IPython/parallel/client/view.pyc in execute(self, code, targets, block)
         591                 default: self.block
         592         """
     --> 593         return self._really_apply(util._execute, args=(code,), block=block, targets=targets)
         594 
         595     def run(self, filename, targets=None, block=None):
 
     /home/user/<string> in _really_apply(self, f, args, kwargs, targets, block, track)
 
     /path/to/site-packages/IPython/parallel/client/view.pyc in sync_results(f, self, *args, **kwargs)
          55 def sync_results(f, self, *args, **kwargs):
          56     """sync relevant results from self.client to our results attribute."""
     ---> 57     ret = f(self, *args, **kwargs)
          58     delta = self.outstanding.difference(self.client.outstanding)
          59     completed = self.outstanding.intersection(delta)
 
     /home/user/<string> in _really_apply(self, f, args, kwargs, targets, block, track)
 
     /path/to/site-packages/IPython/parallel/client/view.pyc in save_ids(f, self, *args, **kwargs)
          44     n_previous = len(self.client.history)
          45     try:
     ---> 46         ret = f(self, *args, **kwargs)
          47     finally:
          48         nmsgs = len(self.client.history) - n_previous
 
     /path/to/site-packages/IPython/parallel/client/view.pyc in _really_apply(self, f, args, kwargs, targets, block, track)
         529         if block:
         530             try:
     --> 531                 return ar.get()
         532             except KeyboardInterrupt:
         533                 pass
 
     /path/to/site-packages/IPython/parallel/client/asyncresult.pyc in get(self, timeout)
         101                 return self._result
         102             else:
     --> 103                 raise self._exception
         104         else:
         105             raise error.TimeoutError("Result not ready.")
 
     CompositeError: one or more exceptions from call to method: _execute
     [0:apply]: ZeroDivisionError: integer division or modulo by zero
     [1:apply]: ZeroDivisionError: integer division or modulo by zero
     [2:apply]: ZeroDivisionError: integer division or modulo by zero
     [3:apply]: ZeroDivisionError: integer division or modulo by zero
 
 Notice how the error message printed when :exc:`CompositeError` is raised has
 information about the individual exceptions that were raised on each engine.
 If you want, you can even raise one of these original exceptions:
 
 .. sourcecode:: ipython
 
     In [80]: try:
        ....:     dview.execute('1/0')
        ....: except parallel.error.CompositeError, e:
        ....:     e.raise_exception()
        ....:     
        ....:     
     ---------------------------------------------------------------------------
     RemoteError                               Traceback (most recent call last)
     /home/user/<ipython-input-17-8597e7e39858> in <module>()
           2     dview.execute('1/0')
           3 except CompositeError as e:
     ----> 4     e.raise_exception()
 
     /path/to/site-packages/IPython/parallel/error.pyc in raise_exception(self, excid)
         266             raise IndexError("an exception with index %i does not exist"%excid)
         267         else:
     --> 268             raise RemoteError(en, ev, etb, ei)
         269 
         270 
 
     RemoteError: ZeroDivisionError(integer division or modulo by zero)
     Traceback (most recent call last):
       File "/path/to/site-packages/IPython/parallel/engine/streamkernel.py", line 330, in apply_request
         exec code in working,working
       File "<string>", line 1, in <module>
       File "/path/to/site-packages/IPython/parallel/util.py", line 354, in _execute
         exec code in globals()
       File "<string>", line 1, in <module>
     ZeroDivisionError: integer division or modulo by zero
 
 If you are working in IPython, you can simple type ``%debug`` after one of
 these :exc:`CompositeError` exceptions is raised, and inspect the exception
 instance:
 
 .. sourcecode:: ipython
 
     In [81]: dview.execute('1/0')
     ---------------------------------------------------------------------------
     CompositeError                            Traceback (most recent call last)
     /home/user/<ipython-input-10-5d56b303a66c> in <module>()
     ----> 1 dview.execute('1/0')
 
     /path/to/site-packages/IPython/parallel/client/view.pyc in execute(self, code, targets, block)
         591                 default: self.block
         592         """
     --> 593         return self._really_apply(util._execute, args=(code,), block=block, targets=targets)
         594 
         595     def run(self, filename, targets=None, block=None):
 
     /home/user/<string> in _really_apply(self, f, args, kwargs, targets, block, track)
 
     /path/to/site-packages/IPython/parallel/client/view.pyc in sync_results(f, self, *args, **kwargs)
          55 def sync_results(f, self, *args, **kwargs):
          56     """sync relevant results from self.client to our results attribute."""
     ---> 57     ret = f(self, *args, **kwargs)
          58     delta = self.outstanding.difference(self.client.outstanding)
          59     completed = self.outstanding.intersection(delta)
 
     /home/user/<string> in _really_apply(self, f, args, kwargs, targets, block, track)
 
     /path/to/site-packages/IPython/parallel/client/view.pyc in save_ids(f, self, *args, **kwargs)
          44     n_previous = len(self.client.history)
          45     try:
     ---> 46         ret = f(self, *args, **kwargs)
          47     finally:
          48         nmsgs = len(self.client.history) - n_previous
 
     /path/to/site-packages/IPython/parallel/client/view.pyc in _really_apply(self, f, args, kwargs, targets, block, track)
         529         if block:
         530             try:
     --> 531                 return ar.get()
         532             except KeyboardInterrupt:
         533                 pass
 
     /path/to/site-packages/IPython/parallel/client/asyncresult.pyc in get(self, timeout)
         101                 return self._result
         102             else:
     --> 103                 raise self._exception
         104         else:
         105             raise error.TimeoutError("Result not ready.")
 
     CompositeError: one or more exceptions from call to method: _execute
     [0:apply]: ZeroDivisionError: integer division or modulo by zero
     [1:apply]: ZeroDivisionError: integer division or modulo by zero
     [2:apply]: ZeroDivisionError: integer division or modulo by zero
     [3:apply]: ZeroDivisionError: integer division or modulo by zero
     
     In [82]: %debug
     > /path/to/site-packages/IPython/parallel/client/asyncresult.py(103)get()
         102             else:
     --> 103                 raise self._exception
         104         else:
     
     # With the debugger running, self._exception is the exceptions instance.  We can tab complete
     # on it and see the extra methods that are available.
     ipdb> self._exception.<tab>
     e.__class__         e.__getitem__       e.__new__           e.__setstate__      e.args
     e.__delattr__       e.__getslice__      e.__reduce__        e.__str__           e.elist
     e.__dict__          e.__hash__          e.__reduce_ex__     e.__weakref__       e.message
     e.__doc__           e.__init__          e.__repr__          e._get_engine_str   e.print_tracebacks
     e.__getattribute__  e.__module__        e.__setattr__       e._get_traceback    e.raise_exception
     ipdb> self._exception.print_tracebacks()
     [0:apply]: 
     Traceback (most recent call last):
       File "/path/to/site-packages/IPython/parallel/engine/streamkernel.py", line 330, in apply_request
         exec code in working,working
       File "<string>", line 1, in <module>
       File "/path/to/site-packages/IPython/parallel/util.py", line 354, in _execute
         exec code in globals()
       File "<string>", line 1, in <module>
     ZeroDivisionError: integer division or modulo by zero
 
 
     [1:apply]: 
     Traceback (most recent call last):
       File "/path/to/site-packages/IPython/parallel/engine/streamkernel.py", line 330, in apply_request
         exec code in working,working
       File "<string>", line 1, in <module>
       File "/path/to/site-packages/IPython/parallel/util.py", line 354, in _execute
         exec code in globals()
       File "<string>", line 1, in <module>
     ZeroDivisionError: integer division or modulo by zero
 
 
     [2:apply]: 
     Traceback (most recent call last):
       File "/path/to/site-packages/IPython/parallel/engine/streamkernel.py", line 330, in apply_request
         exec code in working,working
       File "<string>", line 1, in <module>
       File "/path/to/site-packages/IPython/parallel/util.py", line 354, in _execute
         exec code in globals()
       File "<string>", line 1, in <module>
     ZeroDivisionError: integer division or modulo by zero
 
 
     [3:apply]: 
     Traceback (most recent call last):
       File "/path/to/site-packages/IPython/parallel/engine/streamkernel.py", line 330, in apply_request
         exec code in working,working
       File "<string>", line 1, in <module>
       File "/path/to/site-packages/IPython/parallel/util.py", line 354, in _execute
         exec code in globals()
       File "<string>", line 1, in <module>
     ZeroDivisionError: integer division or modulo by zero
 
 
 All of this same error handling magic even works in non-blocking mode:
 
 .. sourcecode:: ipython
 
     In [83]: dview.block=False
 
     In [84]: ar = dview.execute('1/0')
 
     In [85]: ar.get()
     ---------------------------------------------------------------------------
     CompositeError                            Traceback (most recent call last)
     /home/user/<ipython-input-21-8531eb3d26fb> in <module>()
     ----> 1 ar.get()
 
     /path/to/site-packages/IPython/parallel/client/asyncresult.pyc in get(self, timeout)
         101                 return self._result
         102             else:
     --> 103                 raise self._exception
         104         else:
         105             raise error.TimeoutError("Result not ready.")
 
     CompositeError: one or more exceptions from call to method: _execute
     [0:apply]: ZeroDivisionError: integer division or modulo by zero
     [1:apply]: ZeroDivisionError: integer division or modulo by zero
     [2:apply]: ZeroDivisionError: integer division or modulo by zero
     [3:apply]: ZeroDivisionError: integer division or modulo by zero