update API after sagedays29

tests, docs updated to match

* Client no longer has high-level methods (only in Views)
* module functions can be pushed
* clients can have a connection timeout
* dependencies have separate switches for success/failure, not just success_only
* add `with view.temp_flags(**flags):` for temporary flags

Also updated some docs and examples

IPython/zmq/parallel/tests/test_asyncresult.py

@@ -0,0 +1,69 b''
+"""Tests for asyncresult.py"""
+
+#-------------------------------------------------------------------------------
+#  Copyright (C) 2011  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-------------------------------------------------------------------------------
+
+#-------------------------------------------------------------------------------
+# Imports
+#-------------------------------------------------------------------------------
+
+
+from IPython.zmq.parallel.error import TimeoutError
+
+from IPython.zmq.parallel.tests import add_engines
+from .clienttest import ClusterTestCase
+
+def setup():
+    add_engines(2)
+
+def wait(n):
+    import time
+    time.sleep(n)
+    return n
+
+class AsyncResultTest(ClusterTestCase):
+
+    def test_single_result(self):
+        eid = self.client.ids[-1]
+        ar = self.client[eid].apply_async(lambda : 42)
+        self.assertEquals(ar.get(), 42)
+        ar = self.client[[eid]].apply_async(lambda : 42)
+        self.assertEquals(ar.get(), [42])
+        ar = self.client[-1:].apply_async(lambda : 42)
+        self.assertEquals(ar.get(), [42])
+
+    def test_get_after_done(self):
+        ar = self.client[-1].apply_async(lambda : 42)
+        self.assertFalse(ar.ready())
+        ar.wait()
+        self.assertTrue(ar.ready())
+        self.assertEquals(ar.get(), 42)
+        self.assertEquals(ar.get(), 42)
+
+    def test_get_before_done(self):
+        ar = self.client[-1].apply_async(wait, 0.1)
+        self.assertRaises(TimeoutError, ar.get, 0)
+        ar.wait(0)
+        self.assertFalse(ar.ready())
+        self.assertEquals(ar.get(), 0.1)
+
+    def test_get_after_error(self):
+        ar = self.client[-1].apply_async(lambda : 1/0)
+        ar.wait()
+        self.assertRaisesRemote(ZeroDivisionError, ar.get)
+        self.assertRaisesRemote(ZeroDivisionError, ar.get)
+        self.assertRaisesRemote(ZeroDivisionError, ar.get_dict)
+    
+    def test_get_dict(self):
+        n = len(self.client)
+        ar = self.client[:].apply_async(lambda : 5)
+        self.assertEquals(ar.get(), [5]*n)
+        d = ar.get_dict()
+        self.assertEquals(sorted(d.keys()), sorted(self.client.ids))
+        for eid,r in d.iteritems():
+            self.assertEquals(r, 5)
+

IPython/zmq/parallel/tests/test_dependency.py

@@ -0,0 +1,101 b''
+"""Tests for dependency.py"""
+
+__docformat__ = "restructuredtext en"
+
+#-------------------------------------------------------------------------------
+#  Copyright (C) 2011  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-------------------------------------------------------------------------------
+
+#-------------------------------------------------------------------------------
+# Imports
+#-------------------------------------------------------------------------------
+
+# import
+import os
+
+from IPython.utils.pickleutil import can, uncan
+
+from IPython.zmq.parallel import dependency as dmod
+from IPython.zmq.parallel.util import interactive
+
+from IPython.zmq.parallel.tests import add_engines
+from .clienttest import ClusterTestCase
+
+def setup():
+    add_engines(1)
+
[e-mail]('time')
+def wait(n):
+    time.sleep(n)
+    return n
+
+mixed = map(str, range(10))
+completed = map(str, range(0,10,2))
+failed = map(str, range(1,10,2))
+
+class DependencyTest(ClusterTestCase):
+    
+    def setUp(self):
+        ClusterTestCase.setUp(self)
+        self.user_ns = {'__builtins__' : __builtins__}
+        self.view = self.client.load_balanced_view()
+        self.dview = self.client[-1]
+        self.succeeded = set(map(str, range(0,25,2)))
+        self.failed = set(map(str, range(1,25,2)))
+    
+    def assertMet(self, dep):
+        self.assertTrue(dep.check(self.succeeded, self.failed), "Dependency should be met")
+        
+    def assertUnmet(self, dep):
+        self.assertFalse(dep.check(self.succeeded, self.failed), "Dependency should not be met")
+        
+    def assertUnreachable(self, dep):
+        self.assertTrue(dep.unreachable(self.succeeded, self.failed), "Dependency should be unreachable")
+    
+    def assertReachable(self, dep):
+        self.assertFalse(dep.unreachable(self.succeeded, self.failed), "Dependency should be reachable")
+    
+    def cancan(self, f):
+        """decorator to pass through canning into self.user_ns"""
+        return uncan(can(f), self.user_ns)
+    
+    def test_require_imports(self):
+        """test that @require imports names"""
+        @self.cancan
+        @dmod.require('urllib')
+        @interactive
+        def encode(dikt):
+            return urllib.urlencode(dikt)
+        # must pass through canning to properly connect namespaces
+        self.assertEquals(encode(dict(a=5)), 'a=5')
+    
+    def test_success_only(self):
+        dep = dmod.Dependency(mixed, success=True, failure=False)
+        self.assertUnmet(dep)
+        self.assertUnreachable(dep)
+        dep.all=False
+        self.assertMet(dep)
+        self.assertReachable(dep)
+        dep = dmod.Dependency(completed, success=True, failure=False)
+        self.assertMet(dep)
+        self.assertReachable(dep)
+        dep.all=False
+        self.assertMet(dep)
+        self.assertReachable(dep)
+
+    def test_failure_only(self):
+        dep = dmod.Dependency(mixed, success=False, failure=True)
+        self.assertUnmet(dep)
+        self.assertUnreachable(dep)
+        dep.all=False
+        self.assertMet(dep)
+        self.assertReachable(dep)
+        dep = dmod.Dependency(completed, success=False, failure=True)
+        self.assertUnmet(dep)
+        self.assertUnreachable(dep)
+        dep.all=False
+        self.assertUnmet(dep)
+        self.assertUnreachable(dep)

IPython/zmq/parallel/tests/test_view.py

@@ -0,0 +1,287 b''
+"""test View objects"""
+#-------------------------------------------------------------------------------
+#  Copyright (C) 2011  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-------------------------------------------------------------------------------
+
+#-------------------------------------------------------------------------------
+# Imports
+#-------------------------------------------------------------------------------
+
+import time
+from tempfile import mktemp
+
+import zmq
+
+from IPython.zmq.parallel import client as clientmod
+from IPython.zmq.parallel import error
+from IPython.zmq.parallel.asyncresult import AsyncResult, AsyncHubResult, AsyncMapResult
+from IPython.zmq.parallel.view import LoadBalancedView, DirectView
+from IPython.zmq.parallel.util import interactive
+
+from IPython.zmq.parallel.tests import add_engines
+
+from .clienttest import ClusterTestCase, segfault, wait, skip_without
+
+def setup():
+    add_engines(3)
+
+class TestView(ClusterTestCase):
+    
+    def test_segfault_task(self):
+        """test graceful handling of engine death (balanced)"""
+        # self.add_engines(1)
+        ar = self.client[-1].apply_async(segfault)
+        self.assertRaisesRemote(error.EngineError, ar.get)
+        eid = ar.engine_id
+        while eid in self.client.ids:
+            time.sleep(.01)
+            self.client.spin()
+    
+    def test_segfault_mux(self):
+        """test graceful handling of engine death (direct)"""
+        # self.add_engines(1)
+        eid = self.client.ids[-1]
+        ar = self.client[eid].apply_async(segfault)
+        self.assertRaisesRemote(error.EngineError, ar.get)
+        eid = ar.engine_id
+        while eid in self.client.ids:
+            time.sleep(.01)
+            self.client.spin()
+    
+    def test_push_pull(self):
+        """test pushing and pulling"""
+        data = dict(a=10, b=1.05, c=range(10), d={'e':(1,2),'f':'hi'})
+        t = self.client.ids[-1]
+        v = self.client[t]
+        push = v.push
+        pull = v.pull
+        v.block=True
+        nengines = len(self.client)
+        push({'data':data})
+        d = pull('data')
+        self.assertEquals(d, data)
+        self.client[:].push({'data':data})
+        d = self.client[:].pull('data', block=True)
+        self.assertEquals(d, nengines*[data])
+        ar = push({'data':data}, block=False)
+        self.assertTrue(isinstance(ar, AsyncResult))
+        r = ar.get()
+        ar = self.client[:].pull('data', block=False)
+        self.assertTrue(isinstance(ar, AsyncResult))
+        r = ar.get()
+        self.assertEquals(r, nengines*[data])
+        self.client[:].push(dict(a=10,b=20))
+        r = self.client[:].pull(('a','b'))
+        self.assertEquals(r, nengines*[[10,20]])
+    
+    def test_push_pull_function(self):
+        "test pushing and pulling functions"
+        def testf(x):
+            return 2.0*x
+        
+        t = self.client.ids[-1]
+        self.client[t].block=True
+        push = self.client[t].push
+        pull = self.client[t].pull
+        execute = self.client[t].execute
+        push({'testf':testf})
+        r = pull('testf')
+        self.assertEqual(r(1.0), testf(1.0))
+        execute('r = testf(10)')
+        r = pull('r')
+        self.assertEquals(r, testf(10))
+        ar = self.client[:].push({'testf':testf}, block=False)
+        ar.get()
+        ar = self.client[:].pull('testf', block=False)
+        rlist = ar.get()
+        for r in rlist:
+            self.assertEqual(r(1.0), testf(1.0))
+        execute("def g(x): return x*x")
+        r = pull(('testf','g'))
+        self.assertEquals((r[0](10),r[1](10)), (testf(10), 100))
+    
+    def test_push_function_globals(self):
+        """test that pushed functions have access to globals"""
+        @interactive
+        def geta():
+            return a
+        # self.add_engines(1)
+        v = self.client[-1]
+        v.block=True
+        v['f'] = geta
+        self.assertRaisesRemote(NameError, v.execute, 'b=f()')
+        v.execute('a=5')
+        v.execute('b=f()')
+        self.assertEquals(v['b'], 5)
+    
+    def test_push_function_defaults(self):
+        """test that pushed functions preserve default args"""
+        def echo(a=10):
+            return a
+        v = self.client[-1]
+        v.block=True
+        v['f'] = echo
+        v.execute('b=f()')
+        self.assertEquals(v['b'], 10)
+
+    def test_get_result(self):
+        """test getting results from the Hub."""
+        c = clientmod.Client(profile='iptest')
+        # self.add_engines(1)
+        t = c.ids[-1]
+        v = c[t]
+        v2 = self.client[t]
+        ar = v.apply_async(wait, 1)
+        # give the monitor time to notice the message
+        time.sleep(.25)
+        ahr = v2.get_result(ar.msg_ids)
+        self.assertTrue(isinstance(ahr, AsyncHubResult))
+        self.assertEquals(ahr.get(), ar.get())
+        ar2 = v2.get_result(ar.msg_ids)
+        self.assertFalse(isinstance(ar2, AsyncHubResult))
+        c.spin()
+        c.close()
+    
+    def test_run_newline(self):
+        """test that run appends newline to files"""
+        tmpfile = mktemp()
+        with open(tmpfile, 'w') as f:
+            f.write("""def g():
+                return 5
+                """)
+        v = self.client[-1]
+        v.run(tmpfile, block=True)
+        self.assertEquals(v.apply_sync(lambda f: f(), clientmod.Reference('g')), 5)
+
+    def test_apply_tracked(self):
+        """test tracking for apply"""
+        # self.add_engines(1)
+        t = self.client.ids[-1]
+        v = self.client[t]
+        v.block=False
+        def echo(n=1024*1024, **kwargs):
+            with v.temp_flags(**kwargs):
+                return v.apply(lambda x: x, 'x'*n)
+        ar = echo(1, track=False)
+        self.assertTrue(isinstance(ar._tracker, zmq.MessageTracker))
+        self.assertTrue(ar.sent)
+        ar = echo(track=True)
+        self.assertTrue(isinstance(ar._tracker, zmq.MessageTracker))
+        self.assertEquals(ar.sent, ar._tracker.done)
+        ar._tracker.wait()
+        self.assertTrue(ar.sent)
+        
+    def test_push_tracked(self):
+        t = self.client.ids[-1]
+        ns = dict(x='x'*1024*1024)
+        v = self.client[t]
+        ar = v.push(ns, block=False, track=False)
+        self.assertTrue(isinstance(ar._tracker, zmq.MessageTracker))
+        self.assertTrue(ar.sent)
+        
+        ar = v.push(ns, block=False, track=True)
+        self.assertTrue(isinstance(ar._tracker, zmq.MessageTracker))
+        self.assertEquals(ar.sent, ar._tracker.done)
+        ar._tracker.wait()
+        self.assertTrue(ar.sent)
+        ar.get()
+        
+    def test_scatter_tracked(self):
+        t = self.client.ids
+        x='x'*1024*1024
+        ar = self.client[t].scatter('x', x, block=False, track=False)
+        self.assertTrue(isinstance(ar._tracker, zmq.MessageTracker))
+        self.assertTrue(ar.sent)
+        
+        ar = self.client[t].scatter('x', x, block=False, track=True)
+        self.assertTrue(isinstance(ar._tracker, zmq.MessageTracker))
+        self.assertEquals(ar.sent, ar._tracker.done)
+        ar._tracker.wait()
+        self.assertTrue(ar.sent)
+        ar.get()
+    
+    def test_remote_reference(self):
+        v = self.client[-1]
+        v['a'] = 123
+        ra = clientmod.Reference('a')
+        b = v.apply_sync(lambda x: x, ra)
+        self.assertEquals(b, 123)
+
+
+    def test_scatter_gather(self):
+        view = self.client[:]
+        seq1 = range(16)
+        view.scatter('a', seq1)
+        seq2 = view.gather('a', block=True)
+        self.assertEquals(seq2, seq1)
+        self.assertRaisesRemote(NameError, view.gather, 'asdf', block=True)
+    
+    @skip_without('numpy')
+    def test_scatter_gather_numpy(self):
+        import numpy
+        from numpy.testing.utils import assert_array_equal, assert_array_almost_equal
+        view = self.client[:]
+        a = numpy.arange(64)
+        view.scatter('a', a)
+        b = view.gather('a', block=True)
+        assert_array_equal(b, a)
+    
+    def test_map(self):
+        view = self.client[:]
+        def f(x):
+            return x**2
+        data = range(16)
+        r = view.map_sync(f, data)
+        self.assertEquals(r, map(f, data))
+    
+    def test_scatterGatherNonblocking(self):
+        data = range(16)
+        view = self.client[:]
+        view.scatter('a', data, block=False)
+        ar = view.gather('a', block=False)
+        self.assertEquals(ar.get(), data)
+    
+    @skip_without('numpy')
+    def test_scatter_gather_numpy_nonblocking(self):
+        import numpy
+        from numpy.testing.utils import assert_array_equal, assert_array_almost_equal
+        a = numpy.arange(64)
+        view = self.client[:]
+        ar = view.scatter('a', a, block=False)
+        self.assertTrue(isinstance(ar, AsyncResult))
+        amr = view.gather('a', block=False)
+        self.assertTrue(isinstance(amr, AsyncMapResult))
+        assert_array_equal(amr.get(), a)
+
+    def test_execute(self):
+        view = self.client[:]
+        # self.client.debug=True
+        execute = view.execute
+        ar = execute('c=30', block=False)
+        self.assertTrue(isinstance(ar, AsyncResult))
+        ar = execute('d=[0,1,2]', block=False)
+        self.client.wait(ar, 1)
+        self.assertEquals(len(ar.get()), len(self.client))
+        for c in view['c']:
+            self.assertEquals(c, 30)
+    
+    def test_abort(self):
+        view = self.client[-1]
+        ar = view.execute('import time; time.sleep(0.25)', block=False)
+        ar2 = view.apply_async(lambda : 2)
+        ar3 = view.apply_async(lambda : 3)
+        view.abort(ar2)
+        view.abort(ar3.msg_ids)
+        self.assertRaises(error.TaskAborted, ar2.get)
+        self.assertRaises(error.TaskAborted, ar3.get)
+        
+    def test_temp_flags(self):
+        view = self.client[-1]
+        view.block=True
+        with view.temp_flags(block=False):
+            self.assertFalse(view.block)
+        self.assertTrue(view.block)
+    

docs/examples/newparallel/fetchparse.py

@@ -0,0 +1,97 b''
+"""
+An exceptionally lousy site spider
+Ken Kinder <[e-mail]>
+
+Updated for newparallel by Min Ragan-Kelley <[e-mail]>
+
+This module gives an example of how the task interface to the 
+IPython controller works.  Before running this script start the IPython controller
+and some engines using something like::
+
+    ipclusterz start -n 4
+"""
+import sys
+from IPython.zmq.parallel import client, error
+import time
+import BeautifulSoup # this isn't necessary, but it helps throw the dependency error earlier
+
+def fetchAndParse(url, data=None):
+    import urllib2
+    import urlparse
+    import BeautifulSoup
+    links = []
+    try:
+        page = urllib2.urlopen(url, data=data)
+    except Exception:
+        return links
+    else:
+        if page.headers.type == 'text/html':
+            doc = BeautifulSoup.BeautifulSoup(page.read())
+            for node in doc.findAll('a'):
+                href = node.get('href', None)
+                if href:
+                    links.append(urlparse.urljoin(url, href))
+        return links
+
+class DistributedSpider(object):
+    
+    # Time to wait between polling for task results.
+    pollingDelay = 0.5
+    
+    def __init__(self, site):
+        self.client = client.Client()
+        self.view = self.client.load_balanced_view()
+        self.mux = self.client[:]
+        
+        self.allLinks = []
+        self.linksWorking = {}
+        self.linksDone = {}
+        
+        self.site = site
+        
+    def visitLink(self, url):
+        if url not in self.allLinks:
+            self.allLinks.append(url)
+            if url.startswith(self.site):
+                print '    ', url
+                self.linksWorking[url] = self.view.apply(fetchAndParse, url)
+        
+    def onVisitDone(self, links, url):
+        print url, ':'
+        self.linksDone[url] = None
+        del self.linksWorking[url]
+        for link in links:
+            self.visitLink(link)
+                
+    def run(self):
+        self.visitLink(self.site)
+        while self.linksWorking:
+            print len(self.linksWorking), 'pending...'
+            self.synchronize()
+            time.sleep(self.pollingDelay)
+    
+    def synchronize(self):
+        for url, ar in self.linksWorking.items():
+            # Calling get_task_result with block=False will return None if the
+            # task is not done yet.  This provides a simple way of polling.
+            try:
+                links = ar.get(0)
+            except error.TimeoutError:
+                continue
+            except Exception as e:
+                self.linksDone[url] = None
+                del self.linksWorking[url]
+                print url, ':', e.traceback
+            else:
+                self.onVisitDone(links, url)
+
+def main():
+    if len(sys.argv) > 1:
+        site = sys.argv[1]
+    else:
+        site = raw_input('Enter site to crawl: ')
+    distributedSpider = DistributedSpider(site)
+    distributedSpider.run()
+
+if __name__ == '__main__':
+    main()

docs/examples/newparallel/helloworld.py

@@ -0,0 +1,19 b''
+"""
+A Distributed Hello world
+Ken Kinder <[e-mail]>
+"""
+from IPython.zmq.parallel import client
+
+rc = client.Client()
+
+def sleep_and_echo(t, msg):
+    import time
+    time.sleep(t)
+    return msg
+    
+view = rc.load_balanced_view()
+
+world = view.apply_async(sleep_and_echo, 3, 'World!')
+hello = view.apply_async(sleep_and_echo, 2, 'Hello')
+print "Submitted tasks:", hello.msg_ids, world.msg_ids
+print hello.get(), world.get()

IPython/utils/pickleutil.py

@@ -1,143 +1,153 b''
 # encoding: utf-8
 
 """Pickle related utilities. Perhaps this should be called 'can'."""
 
 __docformat__ = "restructuredtext en"
 
 #-------------------------------------------------------------------------------
 #  Copyright (C) 2008  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-------------------------------------------------------------------------------
 
 #-------------------------------------------------------------------------------
 # Imports
 #-------------------------------------------------------------------------------
 
-from types import FunctionType
 import copy
-
-from IPython.zmq.parallel.dependency import dependent
+import sys
+from types import FunctionType
 
 import codeutil
 
 #-------------------------------------------------------------------------------
 # Classes
 #-------------------------------------------------------------------------------
 
 
 class CannedObject(object):
     def __init__(self, obj, keys=[]):
         self.keys = keys
         self.obj = copy.copy(obj)
         for key in keys:
             setattr(self.obj, key, can(getattr(obj, key)))
             
     
     def getObject(self, g=None):
         if g is None:
             g = globals()
         for key in self.keys:
             setattr(self.obj, key, uncan(getattr(self.obj, key), g))
         return self.obj
 
 class Reference(CannedObject):
     """object for wrapping a remote reference by name."""
     def __init__(self, name):
         if not isinstance(name, basestring):
             raise TypeError("illegal name: %r"%name)
         self.name = name
     
     def __repr__(self):
         return "<Reference: %r>"%self.name
     
     def getObject(self, g=None):
         if g is None:
             g = globals()
         try:
             return g[self.name]
         except KeyError:
             raise NameError("name %r is not defined"%self.name)
     
 
 class CannedFunction(CannedObject):
     
     def __init__(self, f):
         self._checkType(f)    
         self.code = f.func_code
         self.defaults = f.func_defaults
+        self.module = f.__module__ or '__main__'
         self.__name__ = f.__name__
     
     def _checkType(self, obj):
         assert isinstance(obj, FunctionType), "Not a function type"
     
     def getObject(self, g=None):
+        # try to load function back into its module:
+        if not self.module.startswith('__'):
+            try:
+                __import__(self.module)
+            except ImportError:
+                pass
+            else:
+                g = sys.modules[self.module].__dict__
+            
         if g is None:
             g = globals()
         newFunc = FunctionType(self.code, g, self.__name__, self.defaults)
         return newFunc
 
 #-------------------------------------------------------------------------------
 # Functions
 #-------------------------------------------------------------------------------
 
-
 def can(obj):
+    # import here to prevent module-level circular imports
+    from IPython.zmq.parallel.dependency import dependent
     if isinstance(obj, dependent):
         keys = ('f','df')
         return CannedObject(obj, keys=keys)
     elif isinstance(obj, FunctionType):
         return CannedFunction(obj)
     elif isinstance(obj,dict):
         return canDict(obj)
     elif isinstance(obj, (list,tuple)):
         return canSequence(obj)
     else:
         return obj
 
 def canDict(obj):
     if isinstance(obj, dict):
         newobj = {}
         for k, v in obj.iteritems():
             newobj[k] = can(v)
         return newobj
     else:
         return obj
 
 def canSequence(obj):
     if isinstance(obj, (list, tuple)):
         t = type(obj)
         return t([can(i) for i in obj])
     else:
         return obj
 
 def uncan(obj, g=None):
     if isinstance(obj, CannedObject):
         return obj.getObject(g)
     elif isinstance(obj,dict):
         return uncanDict(obj, g)
     elif isinstance(obj, (list,tuple)):
         return uncanSequence(obj, g)
     else:
         return obj
 
 def uncanDict(obj, g=None):
     if isinstance(obj, dict):
         newobj = {}
         for k, v in obj.iteritems():
             newobj[k] = uncan(v,g)
         return newobj
     else:
         return obj
 
 def uncanSequence(obj, g=None):
     if isinstance(obj, (list, tuple)):
         t = type(obj)
         return t([uncan(i,g) for i in obj])
     else:
         return obj
 
 
 def rebindFunctionGlobals(f, glbls):
     return FunctionType(f.func_code, glbls)

IPython/zmq/parallel/asyncresult.py

@@ -1,322 +1,340 b''
 """AsyncResult objects for the client"""
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2010-2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import time
 
+from zmq import MessageTracker
+
 from IPython.external.decorator import decorator
 from . import error
 
 #-----------------------------------------------------------------------------
 # Classes
 #-----------------------------------------------------------------------------
 
+# global empty tracker that's always done:
+finished_tracker = MessageTracker()
+
 @decorator
 def check_ready(f, self, *args, **kwargs):
     """Call spin() to sync state prior to calling the method."""
     self.wait(0)
     if not self._ready:
         raise error.TimeoutError("result not ready")
     return f(self, *args, **kwargs)
 
 class AsyncResult(object):
     """Class for representing results of non-blocking calls.
     
     Provides the same interface as :py:class:`multiprocessing.pool.AsyncResult`.
     """
     
     msg_ids = None
     _targets = None
     _tracker = None
+    _single_result = False
     
     def __init__(self, client, msg_ids, fname='unknown', targets=None, tracker=None):
-        self._client = client
         if isinstance(msg_ids, basestring):
+            # always a list
             msg_ids = [msg_ids]
+        if tracker is None:
+            # default to always done
+            tracker = finished_tracker
+        self._client = client
         self.msg_ids = msg_ids
         self._fname=fname
         self._targets = targets
         self._tracker = tracker
         self._ready = False
         self._success = None
-        self._single_result = len(msg_ids) == 1
+        if len(msg_ids) == 1:
+            self._single_result = not isinstance(targets, (list, tuple))
+        else:
+            self._single_result = False
     
     def __repr__(self):
         if self._ready:
             return "<%s: finished>"%(self.__class__.__name__)
         else:
             return "<%s: %s>"%(self.__class__.__name__,self._fname)
     
     
     def _reconstruct_result(self, res):
         """Reconstruct our result from actual result list (always a list)
         
         Override me in subclasses for turning a list of results
         into the expected form.
         """
         if self._single_result:
             return res[0]
         else:
             return res
         
     def get(self, timeout=-1):
         """Return the result when it arrives. 
         
         If `timeout` is not ``None`` and the result does not arrive within
         `timeout` seconds then ``TimeoutError`` is raised. If the
         remote call raised an exception then that exception will be reraised
         by get() inside a `RemoteError`.
         """
         if not self.ready():
             self.wait(timeout)
         
         if self._ready:
             if self._success:
                 return self._result
             else:
                 raise self._exception
         else:
             raise error.TimeoutError("Result not ready.")
     
     def ready(self):
         """Return whether the call has completed."""
         if not self._ready:
             self.wait(0)
         return self._ready
     
     def wait(self, timeout=-1):
         """Wait until the result is available or until `timeout` seconds pass.
         
         This method always returns None.
         """
         if self._ready:
             return
-        self._ready = self._client.barrier(self.msg_ids, timeout)
+        self._ready = self._client.wait(self.msg_ids, timeout)
         if self._ready:
             try:
                 results = map(self._client.results.get, self.msg_ids)
                 self._result = results
                 if self._single_result:
                     r = results[0]
                     if isinstance(r, Exception):
                         raise r
                 else:
                     results = error.collect_exceptions(results, self._fname)
                 self._result = self._reconstruct_result(results)
             except Exception, e:
                 self._exception = e
                 self._success = False
             else:
                 self._success = True
             finally:
                 self._metadata = map(self._client.metadata.get, self.msg_ids)
             
     
     def successful(self):
         """Return whether the call completed without raising an exception. 
         
         Will raise ``AssertionError`` if the result is not ready.
         """
         assert self.ready()
         return self._success
     
     #----------------------------------------------------------------
     # Extra methods not in mp.pool.AsyncResult
     #----------------------------------------------------------------
     
     def get_dict(self, timeout=-1):
         """Get the results as a dict, keyed by engine_id.
         
         timeout behavior is described in `get()`.
         """
         
         results = self.get(timeout)
         engine_ids = [ md['engine_id'] for md in self._metadata ]
         bycount = sorted(engine_ids, key=lambda k: engine_ids.count(k))
         maxcount = bycount.count(bycount[-1])
         if maxcount > 1:
             raise ValueError("Cannot build dict, %i jobs ran on engine #%i"%(
                     maxcount, bycount[-1]))
         
         return dict(zip(engine_ids,results))
     
     @property
-    @check_ready
     def result(self):
         """result property wrapper for `get(timeout=0)`."""
-        return self._result
+        return self.get()
     
     # abbreviated alias:
     r = result
     
     @property
     @check_ready
     def metadata(self):
         """property for accessing execution metadata."""
         if self._single_result:
             return self._metadata[0]
         else:
             return self._metadata
     
     @property
     def result_dict(self):
         """result property as a dict."""
-        return self.get_dict(0)
+        return self.get_dict()
     
     def __dict__(self):
         return self.get_dict(0)
     
     def abort(self):
         """abort my tasks."""
         assert not self.ready(), "Can't abort, I am already done!"
         return self.client.abort(self.msg_ids, targets=self._targets, block=True)
 
     @property
     def sent(self):
-        """check whether my messages have been sent"""
-        if self._tracker is None:
-            return True
-        else:
+        """check whether my messages have been sent."""
             return self._tracker.done
 
+    def wait_for_send(self, timeout=-1):
+        """wait for pyzmq send to complete.
+        
+        This is necessary when sending arrays that you intend to edit in-place.
+        `timeout` is in seconds, and will raise TimeoutError if it is reached
+        before the send completes.
+        """
+        return self._tracker.wait(timeout)
+
     #-------------------------------------
     # dict-access
     #-------------------------------------
     
     @check_ready
     def __getitem__(self, key):
         """getitem returns result value(s) if keyed by int/slice, or metadata if key is str.
         """
         if isinstance(key, int):
             return error.collect_exceptions([self._result[key]], self._fname)[0]
         elif isinstance(key, slice):
             return error.collect_exceptions(self._result[key], self._fname)
         elif isinstance(key, basestring):
             values = [ md[key] for md in self._metadata ]
             if self._single_result:
                 return values[0]
             else:
                 return values
         else:
             raise TypeError("Invalid key type %r, must be 'int','slice', or 'str'"%type(key))
     
     @check_ready
     def __getattr__(self, key):
         """getattr maps to getitem for convenient attr access to metadata."""
         if key not in self._metadata[0].keys():
             raise AttributeError("%r object has no attribute %r"%(
                     self.__class__.__name__, key))
         return self.__getitem__(key)
     
     # asynchronous iterator:
     def __iter__(self):
         if self._single_result:
             raise TypeError("AsyncResults with a single result are not iterable.")
         try:
             rlist = self.get(0)
         except error.TimeoutError:
             # wait for each result individually
             for msg_id in self.msg_ids:
                 ar = AsyncResult(self._client, msg_id, self._fname)
                 yield ar.get()
         else:
             # already done
             for r in rlist:
                 yield r
 
 
     
 class AsyncMapResult(AsyncResult):
     """Class for representing results of non-blocking gathers.
     
     This will properly reconstruct the gather.
     """
     
     def __init__(self, client, msg_ids, mapObject, fname=''):
         AsyncResult.__init__(self, client, msg_ids, fname=fname)
         self._mapObject = mapObject
         self._single_result = False
     
     def _reconstruct_result(self, res):
         """Perform the gather on the actual results."""
         return self._mapObject.joinPartitions(res)
     
     # asynchronous iterator:
     def __iter__(self):
         try:
             rlist = self.get(0)
         except error.TimeoutError:
             # wait for each result individually
             for msg_id in self.msg_ids:
                 ar = AsyncResult(self._client, msg_id, self._fname)
                 rlist = ar.get()
                 try:
                     for r in rlist:
                         yield r
                 except TypeError:
                     # flattened, not a list
                     # this could get broken by flattened data that returns iterables
                     # but most calls to map do not expose the `flatten` argument
                     yield rlist
         else:
             # already done
             for r in rlist:
                 yield r
 
 
 class AsyncHubResult(AsyncResult):
     """Class to wrap pending results that must be requested from the Hub.
     
     Note that waiting/polling on these objects requires polling the Hubover the network,
     so use `AsyncHubResult.wait()` sparingly.
     """
     
     def wait(self, timeout=-1):
         """wait for result to complete."""
         start = time.time()
         if self._ready:
             return
         local_ids = filter(lambda msg_id: msg_id in self._client.outstanding, self.msg_ids)
-        local_ready = self._client.barrier(local_ids, timeout)
+        local_ready = self._client.wait(local_ids, timeout)
         if local_ready:
             remote_ids = filter(lambda msg_id: msg_id not in self._client.results, self.msg_ids)
             if not remote_ids:
                 self._ready = True
             else:
                 rdict = self._client.result_status(remote_ids, status_only=False)
                 pending = rdict['pending']
                 while pending and (timeout < 0 or time.time() < start+timeout):
                     rdict = self._client.result_status(remote_ids, status_only=False)
                     pending = rdict['pending']
                     if pending:
                         time.sleep(0.1)
                 if not pending:
                     self._ready = True
         if self._ready:
             try:
                 results = map(self._client.results.get, self.msg_ids)
                 self._result = results
                 if self._single_result:
                     r = results[0]
                     if isinstance(r, Exception):
                         raise r
                 else:
                     results = error.collect_exceptions(results, self._fname)
                 self._result = self._reconstruct_result(results)
             except Exception, e:
                 self._exception = e
                 self._success = False
             else:
                 self._success = True
             finally:
                 self._metadata = map(self._client.metadata.get, self.msg_ids)
         
 __all__ = ['AsyncResult', 'AsyncMapResult', 'AsyncHubResult']
  No newline at end of file

IPython/zmq/parallel/client.py

@@ -1,1591 +1,1343 b''
-"""A semi-synchronous Client for the ZMQ controller"""
+"""A semi-synchronous Client for the ZMQ cluster"""
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2010  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import os
 import json
 import time
 import warnings
 from datetime import datetime
 from getpass import getpass
 from pprint import pprint
 
 pjoin = os.path.join
 
 import zmq
 # from zmq.eventloop import ioloop, zmqstream
 
 from IPython.utils.path import get_ipython_dir
 from IPython.utils.pickleutil import Reference
 from IPython.utils.traitlets import (HasTraits, Int, Instance, CUnicode, 
                                     Dict, List, Bool, Str, Set)
 from IPython.external.decorator import decorator
 from IPython.external.ssh import tunnel
 
 from . import error
-from . import map as Map
 from . import util
 from . import streamsession as ss
 from .asyncresult import AsyncResult, AsyncMapResult, AsyncHubResult
 from .clusterdir import ClusterDir, ClusterDirError
 from .dependency import Dependency, depend, require, dependent
 from .remotefunction import remote, parallel, ParallelFunction, RemoteFunction
-from .util import ReverseDict, validate_url, disambiguate_url
 from .view import DirectView, LoadBalancedView
 
 #--------------------------------------------------------------------------
-# helpers for implementing old MEC API via client.apply
-#--------------------------------------------------------------------------
-
-def _push(user_ns, **ns):
-    """helper method for implementing `client.push` via `client.apply`"""
-    user_ns.update(ns)
-
-def _pull(user_ns, keys):
-    """helper method for implementing `client.pull` via `client.apply`"""
-    if isinstance(keys, (list,tuple, set)):
-        for key in keys:
-            if not user_ns.has_key(key):
-                raise NameError("name '%s' is not defined"%key)
-        return map(user_ns.get, keys)
-    else:
-        if not user_ns.has_key(keys):
-            raise NameError("name '%s' is not defined"%keys)
-        return user_ns.get(keys)
-
-def _clear(user_ns):
-    """helper method for implementing `client.clear` via `client.apply`"""
-    user_ns.clear()
-
-def _execute(user_ns, code):
-    """helper method for implementing `client.execute` via `client.apply`"""
-    exec code in user_ns
-    
-
-#--------------------------------------------------------------------------
 # Decorators for Client methods
 #--------------------------------------------------------------------------
 
 @decorator
-def spinfirst(f, self, *args, **kwargs):
+def spin_first(f, self, *args, **kwargs):
     """Call spin() to sync state prior to calling the method."""
     self.spin()
     return f(self, *args, **kwargs)
 
 @decorator
-def defaultblock(f, self, *args, **kwargs):
+def default_block(f, self, *args, **kwargs):
     """Default to self.block; preserve self.block."""
     block = kwargs.get('block',None)
     block = self.block if block is None else block
     saveblock = self.block
     self.block = block
     try:
         ret = f(self, *args, **kwargs)
     finally:
         self.block = saveblock
     return ret
 
 
 #--------------------------------------------------------------------------
 # Classes
 #--------------------------------------------------------------------------
 
 class Metadata(dict):
     """Subclass of dict for initializing metadata values.
     
     Attribute access works on keys.
     
     These objects have a strict set of keys - errors will raise if you try
     to add new keys.
     """
     def __init__(self, *args, **kwargs):
         dict.__init__(self)
         md = {'msg_id' : None,
               'submitted' : None,
               'started' : None,
               'completed' : None,
               'received' : None,
               'engine_uuid' : None,
               'engine_id' : None,
               'follow' : None,
               'after' : None,
               'status' : None,
 
               'pyin' : None,
               'pyout' : None,
               'pyerr' : None,
               'stdout' : '',
               'stderr' : '',
             }
         self.update(md)
         self.update(dict(*args, **kwargs))
     
     def __getattr__(self, key):
         """getattr aliased to getitem"""
         if key in self.iterkeys():
             return self[key]
         else:
             raise AttributeError(key)
 
     def __setattr__(self, key, value):
         """setattr aliased to setitem, with strict"""
         if key in self.iterkeys():
             self[key] = value
         else:
             raise AttributeError(key)
     
     def __setitem__(self, key, value):
         """strict static key enforcement"""
         if key in self.iterkeys():
             dict.__setitem__(self, key, value)
         else:
             raise KeyError(key)
         
 
 class Client(HasTraits):
-    """A semi-synchronous client to the IPython ZMQ controller
+    """A semi-synchronous client to the IPython ZMQ cluster
     
     Parameters
     ----------
     
     url_or_file : bytes; zmq url or path to ipcontroller-client.json
         Connection information for the Hub's registration.  If a json connector
         file is given, then likely no further configuration is necessary.
         [Default: use profile]
     profile : bytes
         The name of the Cluster profile to be used to find connector information.
         [Default: 'default']
     context : zmq.Context
         Pass an existing zmq.Context instance, otherwise the client will create its own.
     username : bytes
         set username to be passed to the Session object
     debug : bool
         flag for lots of message printing for debug purposes
  
     #-------------- ssh related args ----------------
     # These are args for configuring the ssh tunnel to be used
     # credentials are used to forward connections over ssh to the Controller
     # Note that the ip given in `addr` needs to be relative to sshserver
     # The most basic case is to leave addr as pointing to localhost (127.0.0.1),
     # and set sshserver as the same machine the Controller is on. However, 
     # the only requirement is that sshserver is able to see the Controller
     # (i.e. is within the same trusted network).
     
     sshserver : str
         A string of the form passed to ssh, i.e. 'server.tld' or '[e-mail]:port'
         If keyfile or password is specified, and this is not, it will default to
         the ip given in addr.
     sshkey : str; path to public ssh key file
         This specifies a key to be used in ssh login, default None.
         Regular default ssh keys will be used without specifying this argument.
     password : str 
         Your ssh password to sshserver. Note that if this is left None,
         you will be prompted for it if passwordless key based login is unavailable.
     paramiko : bool
         flag for whether to use paramiko instead of shell ssh for tunneling.
         [default: True on win32, False else]
     
-    #------- exec authentication args -------
-    # If even localhost is untrusted, you can have some protection against
-    # unauthorized execution by using a key.  Messages are still sent
-    # as cleartext, so if someone can snoop your loopback traffic this will
-    # not help against malicious attacks.
+    ------- exec authentication args -------
+    If even localhost is untrusted, you can have some protection against
+    unauthorized execution by using a key.  Messages are still sent
+    as cleartext, so if someone can snoop your loopback traffic this will
+    not help against malicious attacks.
     
     exec_key : str
         an authentication key or file containing a key
         default: None
     
     
     Attributes
     ----------
     
-    ids : set of int engine IDs
+    ids : list of int engine IDs
         requesting the ids attribute always synchronizes
         the registration state. To request ids without synchronization,
         use semi-private _ids attributes.
     
     history : list of msg_ids
         a list of msg_ids, keeping track of all the execution
         messages you have submitted in order.
     
     outstanding : set of msg_ids
         a set of msg_ids that have been submitted, but whose
         results have not yet been received.
     
     results : dict
         a dict of all our results, keyed by msg_id
     
     block : bool
         determines default behavior when block not specified
         in execution methods
     
     Methods
     -------
     
     spin
         flushes incoming results and registration state changes
         control methods spin, and requesting `ids` also ensures up to date
     
-    barrier
+    wait
         wait on one or more msg_ids
     
     execution methods
         apply
         legacy: execute, run
     
+    data movement
+        push, pull, scatter, gather
+    
     query methods
-        queue_status, get_result, purge
+        queue_status, get_result, purge, result_status
     
     control methods
         abort, shutdown
     
     """
     
     
     block = Bool(False)
     outstanding = Set()
     results = Instance('collections.defaultdict', (dict,))
     metadata = Instance('collections.defaultdict', (Metadata,))
     history = List()
     debug = Bool(False)
     profile=CUnicode('default')
     
     _outstanding_dict = Instance('collections.defaultdict', (set,))
     _ids = List()
     _connected=Bool(False)
     _ssh=Bool(False)
     _context = Instance('zmq.Context')
     _config = Dict()
-    _engines=Instance(ReverseDict, (), {})
+    _engines=Instance(util.ReverseDict, (), {})
     # _hub_socket=Instance('zmq.Socket')
     _query_socket=Instance('zmq.Socket')
     _control_socket=Instance('zmq.Socket')
     _iopub_socket=Instance('zmq.Socket')
     _notification_socket=Instance('zmq.Socket')
-    _apply_socket=Instance('zmq.Socket')
-    _mux_ident=Str()
-    _task_ident=Str()
+    _mux_socket=Instance('zmq.Socket')
+    _task_socket=Instance('zmq.Socket')
     _task_scheme=Str()
     _balanced_views=Dict()
     _direct_views=Dict()
     _closed = False
+    _ignored_control_replies=Int(0)
+    _ignored_hub_replies=Int(0)
     
     def __init__(self, url_or_file=None, profile='default', cluster_dir=None, ipython_dir=None,
             context=None, username=None, debug=False, exec_key=None,
             sshserver=None, sshkey=None, password=None, paramiko=None,
+            timeout=10
             ):
         super(Client, self).__init__(debug=debug, profile=profile)
         if context is None:
             context = zmq.Context.instance()
         self._context = context
             
         
         self._setup_cluster_dir(profile, cluster_dir, ipython_dir)
         if self._cd is not None:
             if url_or_file is None:
                 url_or_file = pjoin(self._cd.security_dir, 'ipcontroller-client.json')
-        assert url_or_file is not None, "I can't find enough information to connect to a controller!"\
+        assert url_or_file is not None, "I can't find enough information to connect to a hub!"\
             " Please specify at least one of url_or_file or profile."
         
         try:
-            validate_url(url_or_file)
+            util.validate_url(url_or_file)
         except AssertionError:
             if not os.path.exists(url_or_file):
                 if self._cd:
                     url_or_file = os.path.join(self._cd.security_dir, url_or_file)
                 assert os.path.exists(url_or_file), "Not a valid connection file or url: %r"%url_or_file
             with open(url_or_file) as f:
                 cfg = json.loads(f.read())
         else:
             cfg = {'url':url_or_file}
         
         # sync defaults from args, json:
         if sshserver:
             cfg['ssh'] = sshserver
         if exec_key:
             cfg['exec_key'] = exec_key
         exec_key = cfg['exec_key']
         sshserver=cfg['ssh']
         url = cfg['url']
         location = cfg.setdefault('location', None)
-        cfg['url'] = disambiguate_url(cfg['url'], location)
+        cfg['url'] = util.disambiguate_url(cfg['url'], location)
         url = cfg['url']
         
         self._config = cfg
         
         self._ssh = bool(sshserver or sshkey or password)
         if self._ssh and sshserver is None:
             # default to ssh via localhost
             sshserver = url.split('://')[1].split(':')[0]
         if self._ssh and password is None:
             if tunnel.try_passwordless_ssh(sshserver, sshkey, paramiko):
                 password=False
             else:
                 password = getpass("SSH Password for %s: "%sshserver)
         ssh_kwargs = dict(keyfile=sshkey, password=password, paramiko=paramiko)
         if exec_key is not None and os.path.isfile(exec_key):
             arg = 'keyfile'
         else:
             arg = 'key'
         key_arg = {arg:exec_key}
         if username is None:
             self.session = ss.StreamSession(**key_arg)
         else:
             self.session = ss.StreamSession(username, **key_arg)
         self._query_socket = self._context.socket(zmq.XREQ)
         self._query_socket.setsockopt(zmq.IDENTITY, self.session.session)
         if self._ssh:
             tunnel.tunnel_connection(self._query_socket, url, sshserver, **ssh_kwargs)
         else:
             self._query_socket.connect(url)
         
         self.session.debug = self.debug
         
         self._notification_handlers = {'registration_notification' : self._register_engine,
                                     'unregistration_notification' : self._unregister_engine,
+                                    'shutdown_notification' : lambda msg: self.close(),
                                     }
         self._queue_handlers = {'execute_reply' : self._handle_execute_reply,
                                 'apply_reply' : self._handle_apply_reply}
-        self._connect(sshserver, ssh_kwargs)
+        self._connect(sshserver, ssh_kwargs, timeout)
         
     def __del__(self):
         """cleanup sockets, but _not_ context."""
         self.close()
     
     def _setup_cluster_dir(self, profile, cluster_dir, ipython_dir):
         if ipython_dir is None:
             ipython_dir = get_ipython_dir()
         if cluster_dir is not None:
             try:
                 self._cd = ClusterDir.find_cluster_dir(cluster_dir)
                 return
             except ClusterDirError:
                 pass
         elif profile is not None:
             try:
                 self._cd = ClusterDir.find_cluster_dir_by_profile(
                     ipython_dir, profile)
                 return
             except ClusterDirError:
                 pass
         self._cd = None
     
-    @property
-    def ids(self):
-        """Always up-to-date ids property."""
-        self._flush_notifications()
-        # always copy:
-        return list(self._ids)
-        
-    def close(self):
-        if self._closed:
-            return
-        snames = filter(lambda n: n.endswith('socket'), dir(self))
-        for socket in map(lambda name: getattr(self, name), snames):
-            if isinstance(socket, zmq.Socket) and not socket.closed:
-                socket.close()
-        self._closed = True
-    
     def _update_engines(self, engines):
         """Update our engines dict and _ids from a dict of the form: {id:uuid}."""
         for k,v in engines.iteritems():
             eid = int(k)
             self._engines[eid] = bytes(v) # force not unicode
             self._ids.append(eid)
         self._ids = sorted(self._ids)
         if sorted(self._engines.keys()) != range(len(self._engines)) and \
-                        self._task_scheme == 'pure' and self._task_ident:
+                        self._task_scheme == 'pure' and self._task_socket:
             self._stop_scheduling_tasks()
     
     def _stop_scheduling_tasks(self):
         """Stop scheduling tasks because an engine has been unregistered
         from a pure ZMQ scheduler.
         """
-        self._task_ident = ''
-        # self._task_socket.close()
-        # self._task_socket = None
+        self._task_socket.close()
+        self._task_socket = None
         msg = "An engine has been unregistered, and we are using pure " +\
               "ZMQ task scheduling.  Task farming will be disabled."
         if self.outstanding:
             msg += " If you were running tasks when this happened, " +\
                    "some `outstanding` msg_ids may never resolve."
         warnings.warn(msg, RuntimeWarning)
     
     def _build_targets(self, targets):
         """Turn valid target IDs or 'all' into two lists:
         (int_ids, uuids).
         """
         if targets is None:
             targets = self._ids
         elif isinstance(targets, str):
             if targets.lower() == 'all':
                 targets = self._ids
             else:
                 raise TypeError("%r not valid str target, must be 'all'"%(targets))
         elif isinstance(targets, int):
             targets = [targets]
         return [self._engines[t] for t in targets], list(targets)
     
-    def _connect(self, sshserver, ssh_kwargs):
-        """setup all our socket connections to the controller. This is called from
+    def _connect(self, sshserver, ssh_kwargs, timeout):
+        """setup all our socket connections to the cluster. This is called from
         __init__."""
         
         # Maybe allow reconnecting?
         if self._connected:
             return
         self._connected=True
 
         def connect_socket(s, url):
-            url = disambiguate_url(url, self._config['location'])
+            url = util.disambiguate_url(url, self._config['location'])
             if self._ssh:
                 return tunnel.tunnel_connection(s, url, sshserver, **ssh_kwargs)
             else:
                 return s.connect(url)
             
         self.session.send(self._query_socket, 'connection_request')
+        r,w,x = zmq.select([self._query_socket],[],[], timeout)
+        if not r:
+            raise error.TimeoutError("Hub connection request timed out")
         idents,msg = self.session.recv(self._query_socket,mode=0)
         if self.debug:
             pprint(msg)
         msg = ss.Message(msg)
         content = msg.content
         self._config['registration'] = dict(content)
         if content.status == 'ok':
-            self._apply_socket = self._context.socket(zmq.XREP)
-            self._apply_socket.setsockopt(zmq.IDENTITY, self.session.session)
             if content.mux:
-                # self._mux_socket = self._context.socket(zmq.XREQ)
-                self._mux_ident = 'mux'
-                connect_socket(self._apply_socket, content.mux)
+                self._mux_socket = self._context.socket(zmq.XREQ)
+                self._mux_socket.setsockopt(zmq.IDENTITY, self.session.session)
+                connect_socket(self._mux_socket, content.mux)
             if content.task:
                 self._task_scheme, task_addr = content.task
-                # self._task_socket = self._context.socket(zmq.XREQ)
-                # self._task_socket.setsockopt(zmq.IDENTITY, self.session.session)
-                connect_socket(self._apply_socket, task_addr)
-                self._task_ident = 'task'
+                self._task_socket = self._context.socket(zmq.XREQ)
+                self._task_socket.setsockopt(zmq.IDENTITY, self.session.session)
+                connect_socket(self._task_socket, task_addr)
             if content.notification:
                 self._notification_socket = self._context.socket(zmq.SUB)
                 connect_socket(self._notification_socket, content.notification)
                 self._notification_socket.setsockopt(zmq.SUBSCRIBE, b'')
             # if content.query:
             #     self._query_socket = self._context.socket(zmq.XREQ)
             #     self._query_socket.setsockopt(zmq.IDENTITY, self.session.session)
             #     connect_socket(self._query_socket, content.query)
             if content.control:
                 self._control_socket = self._context.socket(zmq.XREQ)
                 self._control_socket.setsockopt(zmq.IDENTITY, self.session.session)
                 connect_socket(self._control_socket, content.control)
             if content.iopub:
                 self._iopub_socket = self._context.socket(zmq.SUB)
                 self._iopub_socket.setsockopt(zmq.SUBSCRIBE, b'')
                 self._iopub_socket.setsockopt(zmq.IDENTITY, self.session.session)
                 connect_socket(self._iopub_socket, content.iopub)
             self._update_engines(dict(content.engines))
-            # give XREP apply_socket some time to connect
-            time.sleep(0.25)
         else:
             self._connected = False
             raise Exception("Failed to connect!")
     
     #--------------------------------------------------------------------------
     # handlers and callbacks for incoming messages
     #--------------------------------------------------------------------------
     
     def _unwrap_exception(self, content):
-        """unwrap exception, and remap engineid to int."""
+        """unwrap exception, and remap engine_id to int."""
         e = error.unwrap_exception(content)
         # print e.traceback
         if e.engine_info:
             e_uuid = e.engine_info['engine_uuid']
             eid = self._engines[e_uuid]
             e.engine_info['engine_id'] = eid
         return e
     
     def _extract_metadata(self, header, parent, content):
         md = {'msg_id' : parent['msg_id'],
               'received' : datetime.now(),
               'engine_uuid' : header.get('engine', None),
               'follow' : parent.get('follow', []),
               'after' : parent.get('after', []),
               'status' : content['status'],
             }
         
         if md['engine_uuid'] is not None:
             md['engine_id'] = self._engines.get(md['engine_uuid'], None)
         
         if 'date' in parent:
             md['submitted'] = datetime.strptime(parent['date'], util.ISO8601)
         if 'started' in header:
             md['started'] = datetime.strptime(header['started'], util.ISO8601)
         if 'date' in header:
             md['completed'] = datetime.strptime(header['date'], util.ISO8601)
         return md
     
     def _register_engine(self, msg):
         """Register a new engine, and update our connection info."""
         content = msg['content']
         eid = content['id']
         d = {eid : content['queue']}
         self._update_engines(d)
 
     def _unregister_engine(self, msg):
         """Unregister an engine that has died."""
         content = msg['content']
         eid = int(content['id'])
         if eid in self._ids:
             self._ids.remove(eid)
             uuid = self._engines.pop(eid)
             
             self._handle_stranded_msgs(eid, uuid)
                 
-        if self._task_ident and self._task_scheme == 'pure':
+        if self._task_socket and self._task_scheme == 'pure':
             self._stop_scheduling_tasks()
     
     def _handle_stranded_msgs(self, eid, uuid):
         """Handle messages known to be on an engine when the engine unregisters.
         
         It is possible that this will fire prematurely - that is, an engine will
         go down after completing a result, and the client will be notified
         of the unregistration and later receive the successful result.
         """
         
         outstanding = self._outstanding_dict[uuid]
         
         for msg_id in list(outstanding):
             if msg_id in self.results:
                 # we already 
                 continue
             try:
                 raise error.EngineError("Engine %r died while running task %r"%(eid, msg_id))
             except:
                 content = error.wrap_exception()
             # build a fake message:
             parent = {}
             header = {}
             parent['msg_id'] = msg_id
             header['engine'] = uuid
             header['date'] = datetime.now().strftime(util.ISO8601)
             msg = dict(parent_header=parent, header=header, content=content)
             self._handle_apply_reply(msg)
     
     def _handle_execute_reply(self, msg):
         """Save the reply to an execute_request into our results.
         
         execute messages are never actually used. apply is used instead.
         """
         
         parent = msg['parent_header']
         msg_id = parent['msg_id']
         if msg_id not in self.outstanding:
             if msg_id in self.history:
                 print ("got stale result: %s"%msg_id)
             else:
                 print ("got unknown result: %s"%msg_id)
         else:
             self.outstanding.remove(msg_id)
         self.results[msg_id] = self._unwrap_exception(msg['content'])
     
     def _handle_apply_reply(self, msg):
         """Save the reply to an apply_request into our results."""
         parent = msg['parent_header']
         msg_id = parent['msg_id']
         if msg_id not in self.outstanding:
             if msg_id in self.history:
                 print ("got stale result: %s"%msg_id)
                 print self.results[msg_id]
                 print msg
             else:
                 print ("got unknown result: %s"%msg_id)
         else:
             self.outstanding.remove(msg_id)
         content = msg['content']
         header = msg['header']
         
         # construct metadata:
         md = self.metadata[msg_id]
         md.update(self._extract_metadata(header, parent, content))
         # is this redundant?
         self.metadata[msg_id] = md
         
         e_outstanding = self._outstanding_dict[md['engine_uuid']]
         if msg_id in e_outstanding:
             e_outstanding.remove(msg_id)
         
         # construct result:
         if content['status'] == 'ok':
             self.results[msg_id] = util.unserialize_object(msg['buffers'])[0]
         elif content['status'] == 'aborted':
-            self.results[msg_id] = error.AbortedTask(msg_id)
+            self.results[msg_id] = error.TaskAborted(msg_id)
         elif content['status'] == 'resubmitted':
             # TODO: handle resubmission
             pass
         else:
             self.results[msg_id] = self._unwrap_exception(content)
     
     def _flush_notifications(self):
         """Flush notifications of engine registrations waiting
         in ZMQ queue."""
         msg = self.session.recv(self._notification_socket, mode=zmq.NOBLOCK)
         while msg is not None:
             if self.debug:
                 pprint(msg)
             msg = msg[-1]
             msg_type = msg['msg_type']
             handler = self._notification_handlers.get(msg_type, None)
             if handler is None:
                 raise Exception("Unhandled message type: %s"%msg.msg_type)
             else:
                 handler(msg)
             msg = self.session.recv(self._notification_socket, mode=zmq.NOBLOCK)
     
     def _flush_results(self, sock):
         """Flush task or queue results waiting in ZMQ queue."""
         msg = self.session.recv(sock, mode=zmq.NOBLOCK)
         while msg is not None:
             if self.debug:
                 pprint(msg)
             msg = msg[-1]
             msg_type = msg['msg_type']
             handler = self._queue_handlers.get(msg_type, None)
             if handler is None:
                 raise Exception("Unhandled message type: %s"%msg.msg_type)
             else:
                 handler(msg)
             msg = self.session.recv(sock, mode=zmq.NOBLOCK)
     
     def _flush_control(self, sock):
         """Flush replies from the control channel waiting
         in the ZMQ queue.
         
         Currently: ignore them."""
+        if self._ignored_control_replies <= 0:
+            return
         msg = self.session.recv(sock, mode=zmq.NOBLOCK)
         while msg is not None:
+            self._ignored_control_replies -= 1
             if self.debug:
                 pprint(msg)
             msg = self.session.recv(sock, mode=zmq.NOBLOCK)
     
+    def _flush_ignored_control(self):
+        """flush ignored control replies"""
+        while self._ignored_control_replies > 0:
+            self.session.recv(self._control_socket)
+            self._ignored_control_replies -= 1
+    
+    def _flush_ignored_hub_replies(self):
+        msg = self.session.recv(self._query_socket, mode=zmq.NOBLOCK)
+        while msg is not None:
+            msg = self.session.recv(self._query_socket, mode=zmq.NOBLOCK)
+    
     def _flush_iopub(self, sock):
         """Flush replies from the iopub channel waiting
         in the ZMQ queue.
         """
         msg = self.session.recv(sock, mode=zmq.NOBLOCK)
         while msg is not None:
             if self.debug:
                 pprint(msg)
             msg = msg[-1]
             parent = msg['parent_header']
             msg_id = parent['msg_id']
             content = msg['content']
             header = msg['header']
             msg_type = msg['msg_type']
             
             # init metadata:
             md = self.metadata[msg_id]
             
             if msg_type == 'stream':
                 name = content['name']
                 s = md[name] or ''
                 md[name] = s + content['data']
             elif msg_type == 'pyerr':
                 md.update({'pyerr' : self._unwrap_exception(content)})
             else:
                 md.update({msg_type : content['data']})
             
             # reduntant?
             self.metadata[msg_id] = md
             
             msg = self.session.recv(sock, mode=zmq.NOBLOCK)
     
     #--------------------------------------------------------------------------
     # len, getitem
     #--------------------------------------------------------------------------
     
     def __len__(self):
         """len(client) returns # of engines."""
         return len(self.ids)
     
     def __getitem__(self, key):
         """index access returns DirectView multiplexer objects
         
         Must be int, slice, or list/tuple/xrange of ints"""
         if not isinstance(key, (int, slice, tuple, list, xrange)):
             raise TypeError("key by int/slice/iterable of ints only, not %s"%(type(key)))
         else:
-            return self.view(key, balanced=False)
+            return self._get_view(key, balanced=False)
     
     #--------------------------------------------------------------------------
     # Begin public methods
     #--------------------------------------------------------------------------
     
+    @property
+    def ids(self):
+        """Always up-to-date ids property."""
+        self._flush_notifications()
+        # always copy:
+        return list(self._ids)
+        
+    def close(self):
+        if self._closed:
+            return
+        snames = filter(lambda n: n.endswith('socket'), dir(self))
+        for socket in map(lambda name: getattr(self, name), snames):
+            if isinstance(socket, zmq.Socket) and not socket.closed:
+                socket.close()
+        self._closed = True
+    
     def spin(self):
         """Flush any registration notifications and execution results
         waiting in the ZMQ queue.
         """
         if self._notification_socket:
             self._flush_notifications()
-        if self._apply_socket:
-            self._flush_results(self._apply_socket)
+        if self._mux_socket:
+            self._flush_results(self._mux_socket)
+        if self._task_socket:
+            self._flush_results(self._task_socket)
         if self._control_socket:
             self._flush_control(self._control_socket)
         if self._iopub_socket:
             self._flush_iopub(self._iopub_socket)
+        if self._query_socket:
+            self._flush_ignored_hub_replies()
     
-    def barrier(self, jobs=None, timeout=-1):
+    def wait(self, jobs=None, timeout=-1):
         """waits on one or more `jobs`, for up to `timeout` seconds.
         
         Parameters
         ----------
         
         jobs : int, str, or list of ints and/or strs, or one or more AsyncResult objects
                 ints are indices to self.history
                 strs are msg_ids
                 default: wait on all outstanding messages
         timeout : float
                 a time in seconds, after which to give up.
                 default is -1, which means no timeout
         
         Returns
         -------
         
         True : when all msg_ids are done
         False : timeout reached, some msg_ids still outstanding
         """
         tic = time.time()
         if jobs is None:
             theids = self.outstanding
         else:
             if isinstance(jobs, (int, str, AsyncResult)):
                 jobs = [jobs]
             theids = set()
             for job in jobs:
                 if isinstance(job, int):
                     # index access
                     job = self.history[job]
                 elif isinstance(job, AsyncResult):
                     map(theids.add, job.msg_ids)
                     continue
                 theids.add(job)
         if not theids.intersection(self.outstanding):
             return True
         self.spin()
         while theids.intersection(self.outstanding):
             if timeout >= 0 and ( time.time()-tic ) > timeout:
                 break
             time.sleep(1e-3)
             self.spin()
         return len(theids.intersection(self.outstanding)) == 0
     
     #--------------------------------------------------------------------------
     # Control methods
     #--------------------------------------------------------------------------
     
-    @spinfirst
-    @defaultblock
+    @spin_first
+    @default_block
     def clear(self, targets=None, block=None):
         """Clear the namespace in target(s)."""
         targets = self._build_targets(targets)[0]
         for t in targets:
             self.session.send(self._control_socket, 'clear_request', content={}, ident=t)
         error = False
         if self.block:
+            self._flush_ignored_control()
             for i in range(len(targets)):
                 idents,msg = self.session.recv(self._control_socket,0)
                 if self.debug:
                     pprint(msg)
                 if msg['content']['status'] != 'ok':
                     error = self._unwrap_exception(msg['content'])
+        else:
+            self._ignored_control_replies += len(targets)
         if error:
             raise error
         
     
-    @spinfirst
-    @defaultblock
+    @spin_first
+    @default_block
     def abort(self, jobs=None, targets=None, block=None):
         """Abort specific jobs from the execution queues of target(s).
         
         This is a mechanism to prevent jobs that have already been submitted
         from executing.
         
         Parameters
         ----------
         
         jobs : msg_id, list of msg_ids, or AsyncResult
             The jobs to be aborted
         
         
         """
         targets = self._build_targets(targets)[0]
         msg_ids = []
         if isinstance(jobs, (basestring,AsyncResult)):
             jobs = [jobs]
         bad_ids = filter(lambda obj: not isinstance(obj, (basestring, AsyncResult)), jobs)
         if bad_ids:
             raise TypeError("Invalid msg_id type %r, expected str or AsyncResult"%bad_ids[0])
         for j in jobs:
             if isinstance(j, AsyncResult):
                 msg_ids.extend(j.msg_ids)
             else:
                 msg_ids.append(j)
         content = dict(msg_ids=msg_ids)
         for t in targets:
             self.session.send(self._control_socket, 'abort_request', 
                     content=content, ident=t)
         error = False
         if self.block:
+            self._flush_ignored_control()
             for i in range(len(targets)):
                 idents,msg = self.session.recv(self._control_socket,0)
                 if self.debug:
                     pprint(msg)
                 if msg['content']['status'] != 'ok':
                     error = self._unwrap_exception(msg['content'])
+        else:
+            self._ignored_control_replies += len(targets)
         if error:
             raise error
     
-    @spinfirst
-    @defaultblock
-    def shutdown(self, targets=None, restart=False, controller=False, block=None):
-        """Terminates one or more engine processes, optionally including the controller."""
-        if controller:
+    @spin_first
+    @default_block
+    def shutdown(self, targets=None, restart=False, hub=False, block=None):
+        """Terminates one or more engine processes, optionally including the hub."""
+        if hub:
             targets = 'all'
         targets = self._build_targets(targets)[0]
         for t in targets:
             self.session.send(self._control_socket, 'shutdown_request', 
                         content={'restart':restart},ident=t)
         error = False
-        if block or controller:
+        if block or hub:
+            self._flush_ignored_control()
             for i in range(len(targets)):
                 idents,msg = self.session.recv(self._control_socket,0)
                 if self.debug:
                     pprint(msg)
                 if msg['content']['status'] != 'ok':
                     error = self._unwrap_exception(msg['content'])
+        else:
+            self._ignored_control_replies += len(targets)
         
-        if controller:
+        if hub:
             time.sleep(0.25)
             self.session.send(self._query_socket, 'shutdown_request')
             idents,msg = self.session.recv(self._query_socket, 0)
             if self.debug:
                 pprint(msg)
             if msg['content']['status'] != 'ok':
                 error = self._unwrap_exception(msg['content'])
         
         if error:
             raise error
     
     #--------------------------------------------------------------------------
     # Execution methods
     #--------------------------------------------------------------------------
     
-    @defaultblock
-    def execute(self, code, targets='all', block=None):
+    @default_block
+    def _execute(self, code, targets='all', block=None):
         """Executes `code` on `targets` in blocking or nonblocking manner.
         
         ``execute`` is always `bound` (affects engine namespace)
         
         Parameters
         ----------
         
         code : str
                 the code string to be executed
         targets : int/str/list of ints/strs
                 the engines on which to execute
                 default : all
         block : bool
                 whether or not to wait until done to return
                 default: self.block
         """
-        result = self.apply(_execute, (code,), targets=targets, block=block, bound=True, balanced=False)
-        if not block:
-            return result
-    
-    def run(self, filename, targets='all', block=None):
-        """Execute contents of `filename` on engine(s). 
-        
-        This simply reads the contents of the file and calls `execute`.
-        
-        Parameters
-        ----------
-        
-        filename : str
-                The path to the file
-        targets : int/str/list of ints/strs
-                the engines on which to execute
-                default : all
-        block : bool
-                whether or not to wait until done
-                default: self.block
-        
-        """
-        with open(filename, 'r') as f:
-            # add newline in case of trailing indented whitespace
-            # which will cause SyntaxError
-            code = f.read()+'\n'
-        return self.execute(code, targets=targets, block=block)
+        return self[targets].execute(code, block=block)
     
     def _maybe_raise(self, result):
         """wrapper for maybe raising an exception if apply failed."""
         if isinstance(result, error.RemoteError):
             raise result
         
         return result
     
-    def _build_dependency(self, dep):
-        """helper for building jsonable dependencies from various input forms"""
-        if isinstance(dep, Dependency):
-            return dep.as_dict()
-        elif isinstance(dep, AsyncResult):
-            return dep.msg_ids
-        elif dep is None:
-            return []
-        else:
-            # pass to Dependency constructor
-            return list(Dependency(dep))
-        
-    @defaultblock
-    def apply(self, f, args=None, kwargs=None, bound=False, block=None,
-                        targets=None, balanced=None,
-                        after=None, follow=None, timeout=None,
-                        track=False):
-        """Call `f(*args, **kwargs)` on a remote engine(s), returning the result.
-        
-        This is the central execution command for the client.
-        
-        Parameters
-        ----------
-        
-        f : function
-            The fuction to be called remotely
-        args : tuple/list
-            The positional arguments passed to `f`
-        kwargs : dict
-            The keyword arguments passed to `f`
-        bound : bool (default: False)
-            Whether to pass the Engine(s) Namespace as the first argument to `f`.
-        block : bool (default: self.block)
-            Whether to wait for the result, or return immediately.
-            False:
-                returns AsyncResult
-            True:
-                returns actual result(s) of f(*args, **kwargs)
-                if multiple targets:
-                    list of results, matching `targets`
-        track : bool
-            whether to track non-copying sends.
-            [default False]
-
-        targets : int,list of ints, 'all', None
-            Specify the destination of the job.
-            if None:
-                Submit via Task queue for load-balancing.
-            if 'all':
-                Run on all active engines
-            if list:
-                Run on each specified engine
-            if int:
-                Run on single engine
-            Note:
-                that if `balanced=True`, and `targets` is specified,
-                then the load-balancing will be limited to balancing
-                among `targets`.
-
-        balanced : bool, default None
-            whether to load-balance.  This will default to True
-            if targets is unspecified, or False if targets is specified.
-
-            If `balanced` and `targets` are both specified, the task will
-            be assigne to *one* of the targets by the scheduler.
-
-        The following arguments are only used when balanced is True:
-
-        after : Dependency or collection of msg_ids
-            Only for load-balanced execution (targets=None)
-            Specify a list of msg_ids as a time-based dependency.
-            This job will only be run *after* the dependencies
-            have been met.
-
-        follow : Dependency or collection of msg_ids
-            Only for load-balanced execution (targets=None)
-            Specify a list of msg_ids as a location-based dependency.
-            This job will only be run on an engine where this dependency
-            is met.
-
-        timeout : float/int or None
-            Only for load-balanced execution (targets=None)
-            Specify an amount of time (in seconds) for the scheduler to
-            wait for dependencies to be met before failing with a
-            DependencyTimeout.
+    def send_apply_message(self, socket, f, args=None, kwargs=None, subheader=None, track=False,
+                            ident=None):
+        """construct and send an apply message via a socket.
         
-        Returns
-        -------
-        
-        if block is False:
-            return AsyncResult wrapping msg_ids
-            output of AsyncResult.get() is identical to that of `apply(...block=True)`
-        else:
-            if single target (or balanced):
-                return result of `f(*args, **kwargs)`
-            else:
-                return list of results, matching `targets`
+        This is the principal method with which all engine execution is performed by views.
         """
+                            
         assert not self._closed, "cannot use me anymore, I'm closed!"
         # defaults:
-        block = block if block is not None else self.block
         args = args if args is not None else []
         kwargs = kwargs if kwargs is not None else {}
+        subheader = subheader if subheader is not None else {}
         
-        if not self._ids:
-            # flush notification socket if no engines yet
-            any_ids = self.ids
-            if not any_ids:
-                raise error.NoEnginesRegistered("Can't execute without any connected engines.")
-        
-        if balanced is None:
-            if targets is None:
-                # default to balanced if targets unspecified
-                balanced = True
-            else:
-                # otherwise default to multiplexing
-                balanced = False
-        
-        if targets is None and balanced is False:
-            # default to all if *not* balanced, and targets is unspecified
-            targets = 'all'
-        
-        # enforce types of f,args,kwrags
+        # validate arguments
         if not callable(f):
             raise TypeError("f must be callable, not %s"%type(f))
         if not isinstance(args, (tuple, list)):
             raise TypeError("args must be tuple or list, not %s"%type(args))
         if not isinstance(kwargs, dict):
             raise TypeError("kwargs must be dict, not %s"%type(kwargs))
+        if not isinstance(subheader, dict):
+            raise TypeError("subheader must be dict, not %s"%type(subheader))
         
-        options  = dict(bound=bound, block=block, targets=targets, track=track)
-        
-        if balanced:
-            return self._apply_balanced(f, args, kwargs, timeout=timeout, 
-                                        after=after, follow=follow, **options)
-        elif follow or after or timeout:
-                msg = "follow, after, and timeout args are only used for"
-                msg += " load-balanced execution."
-                raise ValueError(msg)
-        else:
-            return self._apply_direct(f, args, kwargs, **options)
-    
-    def _apply_balanced(self, f, args, kwargs, bound=None, block=None, targets=None,
-                            after=None, follow=None, timeout=None, track=None):
-        """call f(*args, **kwargs) remotely in a load-balanced manner.
-        
-        This is a private method, see `apply` for details.
-        Not to be called directly!
-        """
-        
-        loc = locals()
-        for name in ('bound', 'block', 'track'):
-            assert loc[name] is not None, "kwarg %r must be specified!"%name
-        
-        if not self._task_ident:
-            msg = "Task farming is disabled"
-            if self._task_scheme == 'pure':
-                msg += " because the pure ZMQ scheduler cannot handle"
-                msg += " disappearing engines."
-            raise RuntimeError(msg)
-        
-        if self._task_scheme == 'pure':
-            # pure zmq scheme doesn't support dependencies
-            msg = "Pure ZMQ scheduler doesn't support dependencies"
-            if (follow or after):
-                # hard fail on DAG dependencies
-                raise RuntimeError(msg)
-            if isinstance(f, dependent):
-                # soft warn on functional dependencies
-                warnings.warn(msg, RuntimeWarning)
-        
-        # defaults:
-        args = args if args is not None else []
-        kwargs = kwargs if kwargs is not None else {}
-        
-        if targets:
-            idents,_ = self._build_targets(targets)
-        else:
-            idents = []
+        if not self._ids:
+            # flush notification socket if no engines yet
+            any_ids = self.ids
+            if not any_ids:
+                raise error.NoEnginesRegistered("Can't execute without any connected engines.")
+                # enforce types of f,args,kwargs
         
-        after = self._build_dependency(after)
-        follow = self._build_dependency(follow)
-        subheader = dict(after=after, follow=follow, timeout=timeout, targets=idents)
         bufs = util.pack_apply_message(f,args,kwargs)
-        content = dict(bound=bound)
         
-        msg = self.session.send(self._apply_socket, "apply_request", ident=self._task_ident,
-                content=content, buffers=bufs, subheader=subheader, track=track)
-        msg_id = msg['msg_id']
-        self.outstanding.add(msg_id)
-        self.history.append(msg_id)
-        self.metadata[msg_id]['submitted'] = datetime.now()
-        tracker = None if track is False else msg['tracker']
-        ar = AsyncResult(self, [msg_id], fname=f.__name__, targets=targets, tracker=tracker)
-        if block:
-            try:
-                return ar.get()
-            except KeyboardInterrupt:
-                return ar
-        else:
-            return ar
-    
-    def _apply_direct(self, f, args, kwargs, bound=None, block=None, targets=None,
-                                        track=None):
-        """Then underlying method for applying functions to specific engines
-        via the MUX queue.
-        
-        This is a private method, see `apply` for details.
-        Not to be called directly!
-        """
-        
-        if not self._mux_ident:
-            msg = "Multiplexing is disabled"
-            raise RuntimeError(msg)
-        
-        loc = locals()
-        for name in ('bound', 'block', 'targets', 'track'):
-            assert loc[name] is not None, "kwarg %r must be specified!"%name
+        msg = self.session.send(socket, "apply_request", buffers=bufs, ident=ident,
+                            subheader=subheader, track=track)
         
-        idents,targets = self._build_targets(targets)
-        
-        subheader = {}
-        content = dict(bound=bound)
-        bufs = util.pack_apply_message(f,args,kwargs)
-        
-        msg_ids = []
-        trackers = []
-        for ident in idents:
-            msg = self.session.send(self._apply_socket, "apply_request", 
-                    content=content, buffers=bufs, ident=[self._mux_ident, ident], subheader=subheader,
-                    track=track)
-            if track:
-                trackers.append(msg['tracker'])
             msg_id = msg['msg_id']
             self.outstanding.add(msg_id)
+        if ident:
+            # possibly routed to a specific engine
+            if isinstance(ident, list):
+                ident = ident[-1]
+            if ident in self._engines.values():
+                # save for later, in case of engine death
             self._outstanding_dict[ident].add(msg_id)
             self.history.append(msg_id)
-            msg_ids.append(msg_id)
-        
-        tracker = None if track is False else zmq.MessageTracker(*trackers)
-        ar = AsyncResult(self, msg_ids, fname=f.__name__, targets=targets, tracker=tracker)
+        self.metadata[msg_id]['submitted'] = datetime.now()
         
-        if block:
-            try:
-                return ar.get()
-            except KeyboardInterrupt:
-                return ar
-        else:
-            return ar
+        return msg
     
     #--------------------------------------------------------------------------
     # construct a View object
     #--------------------------------------------------------------------------
     
-    @defaultblock
-    def remote(self, bound=False, block=None, targets=None, balanced=None):
-        """Decorator for making a RemoteFunction"""
-        return remote(self, bound=bound, targets=targets, block=block, balanced=balanced)
-    
-    @defaultblock
-    def parallel(self, dist='b', bound=False, block=None, targets=None, balanced=None):
-        """Decorator for making a ParallelFunction"""
-        return parallel(self, bound=bound, targets=targets, block=block, balanced=balanced)
-    
     def _cache_view(self, targets, balanced):
         """save views, so subsequent requests don't create new objects."""
         if balanced:
+            # validate whether we can run
+            if not self._task_socket:
+                msg = "Task farming is disabled"
+                if self._task_scheme == 'pure':
+                    msg += " because the pure ZMQ scheduler cannot handle"
+                    msg += " disappearing engines."
+                raise RuntimeError(msg)
+            socket = self._task_socket
             view_class = LoadBalancedView
             view_cache = self._balanced_views
         else:
+            socket = self._mux_socket
             view_class = DirectView
             view_cache = self._direct_views
         
         # use str, since often targets will be a list
         key = str(targets)
         if key not in view_cache:
-            view_cache[key] = view_class(client=self, targets=targets)
+            view_cache[key] = view_class(client=self, socket=socket, targets=targets)
         
         return view_cache[key]
     
-    def view(self, targets=None, balanced=None):
+    def load_balanced_view(self, targets=None):
+        """construct a DirectView object.
+        
+        If no arguments are specified, create a LoadBalancedView
+        using all engines.
+        
+        Parameters
+        ----------
+        
+        targets: list,slice,int,etc. [default: use all engines]
+            The subset of engines across which to load-balance
+        """
+        return self._get_view(targets, balanced=True)
+    
+    def direct_view(self, targets='all'):
+        """construct a DirectView object.
+        
+        If no targets are specified, create a DirectView
+        using all engines.
+        
+        Parameters
+        ----------
+        
+        targets: list,slice,int,etc. [default: use all engines]
+            The engines to use for the View
+        """
+        return self._get_view(targets, balanced=False)
+    
+    def _get_view(self, targets, balanced):
         """Method for constructing View objects.
         
         If no arguments are specified, create a LoadBalancedView
         using all engines.  If only `targets` specified, it will
         be a DirectView.  This method is the underlying implementation
         of ``client.__getitem__``.
         
         Parameters
         ----------
         
         targets: list,slice,int,etc. [default: use all engines]
             The engines to use for the View
         balanced : bool [default: False if targets specified, True else]
             whether to build a LoadBalancedView or a DirectView
         
         """
         
-        balanced = (targets is None) if balanced is None else balanced
-        
-        if targets is None:
+        if targets in (None,'all'):
             if balanced:
                 return self._cache_view(None,True)
             else:
                 targets = slice(None)
         
         if isinstance(targets, int):
             if targets < 0:
                 targets = self.ids[targets]
             if targets not in self.ids:
                 raise IndexError("No such engine: %i"%targets)
             return self._cache_view(targets, balanced)
         
         if isinstance(targets, slice):
             indices = range(len(self.ids))[targets]
             ids = sorted(self._ids)
             targets = [ ids[i] for i in indices ]
         
         if isinstance(targets, (tuple, list, xrange)):
             _,targets = self._build_targets(list(targets))
             return self._cache_view(targets, balanced)
         else:
             raise TypeError("targets by int/slice/collection of ints only, not %s"%(type(targets)))
     
     #--------------------------------------------------------------------------
-    # Data movement
+    # Data movement (TO BE REMOVED)
     #--------------------------------------------------------------------------
     
-    @defaultblock
-    def push(self, ns, targets='all', block=None, track=False):
+    @default_block
+    def _push(self, ns, targets='all', block=None, track=False):
         """Push the contents of `ns` into the namespace on `target`"""
         if not isinstance(ns, dict):
             raise TypeError("Must be a dict, not %s"%type(ns))
-        result = self.apply(_push, kwargs=ns, targets=targets, block=block, bound=True, balanced=False, track=track)
+        result = self.apply(util._push, kwargs=ns, targets=targets, block=block, bound=True, balanced=False, track=track)
         if not block:
             return result
     
-    @defaultblock
-    def pull(self, keys, targets='all', block=None):
+    @default_block
+    def _pull(self, keys, targets='all', block=None):
         """Pull objects from `target`'s namespace by `keys`"""
         if isinstance(keys, basestring):
             pass
         elif isinstance(keys, (list,tuple,set)):
             for key in keys:
                 if not isinstance(key, basestring):
                     raise TypeError("keys must be str, not type %r"%type(key))
         else:
             raise TypeError("keys must be strs, not %r"%keys)
-        result = self.apply(_pull, (keys,), targets=targets, block=block, bound=True, balanced=False)
+        result = self.apply(util._pull, (keys,), targets=targets, block=block, bound=True, balanced=False)
         return result
     
-    @defaultblock
-    def scatter(self, key, seq, dist='b', flatten=False, targets='all', block=None, track=False):
-        """
-        Partition a Python sequence and send the partitions to a set of engines.
-        """
-        targets = self._build_targets(targets)[-1]
-        mapObject = Map.dists[dist]()
-        nparts = len(targets)
-        msg_ids = []
-        trackers = []
-        for index, engineid in enumerate(targets):
-            partition = mapObject.getPartition(seq, index, nparts)
-            if flatten and len(partition) == 1:
-                r = self.push({key: partition[0]}, targets=engineid, block=False, track=track)
-            else:
-                r = self.push({key: partition}, targets=engineid, block=False, track=track)
-            msg_ids.extend(r.msg_ids)
-            if track:
-                trackers.append(r._tracker)
-        
-        if track:
-            tracker = zmq.MessageTracker(*trackers)
-        else:
-            tracker = None
-        
-        r = AsyncResult(self, msg_ids, fname='scatter', targets=targets, tracker=tracker)
-        if block:
-            r.wait()
-        else:
-            return r
-    
-    @defaultblock
-    def gather(self, key, dist='b', targets='all', block=None):
-        """
-        Gather a partitioned sequence on a set of engines as a single local seq.
-        """
-        
-        targets = self._build_targets(targets)[-1]
-        mapObject = Map.dists[dist]()
-        msg_ids = []
-        for index, engineid in enumerate(targets):
-            msg_ids.extend(self.pull(key, targets=engineid,block=False).msg_ids)
-        
-        r = AsyncMapResult(self, msg_ids, mapObject, fname='gather')
-        if block:
-            return r.get()
-        else:
-            return r
-    
     #--------------------------------------------------------------------------
     # Query methods
     #--------------------------------------------------------------------------
     
-    @spinfirst
-    @defaultblock
+    @spin_first
+    @default_block
     def get_result(self, indices_or_msg_ids=None, block=None):
         """Retrieve a result by msg_id or history index, wrapped in an AsyncResult object.
         
         If the client already has the results, no request to the Hub will be made.
         
         This is a convenient way to construct AsyncResult objects, which are wrappers
         that include metadata about execution, and allow for awaiting results that
         were not submitted by this Client.
         
         It can also be a convenient way to retrieve the metadata associated with
         blocking execution, since it always retrieves
         
         Examples
         --------
         ::
         
             In [10]: r = client.apply()
         
         Parameters
         ----------
         
         indices_or_msg_ids : integer history index, str msg_id, or list of either
             The indices or msg_ids of indices to be retrieved
         
         block : bool
             Whether to wait for the result to be done
         
         Returns
         -------
         
         AsyncResult
             A single AsyncResult object will always be returned.
         
         AsyncHubResult
             A subclass of AsyncResult that retrieves results from the Hub
         
         """
         if indices_or_msg_ids is None:
             indices_or_msg_ids = -1
         
         if not isinstance(indices_or_msg_ids, (list,tuple)):
             indices_or_msg_ids = [indices_or_msg_ids]
         
         theids = []
         for id in indices_or_msg_ids:
             if isinstance(id, int):
                 id = self.history[id]
             if not isinstance(id, str):
                 raise TypeError("indices must be str or int, not %r"%id)
             theids.append(id)
         
         local_ids = filter(lambda msg_id: msg_id in self.history or msg_id in self.results, theids)
         remote_ids = filter(lambda msg_id: msg_id not in local_ids, theids)
         
         if remote_ids:
             ar = AsyncHubResult(self, msg_ids=theids)
         else:
             ar = AsyncResult(self, msg_ids=theids)
         
         if block:
             ar.wait()
         
         return ar
     
-    @spinfirst
+    @spin_first
     def result_status(self, msg_ids, status_only=True):
         """Check on the status of the result(s) of the apply request with `msg_ids`.
         
         If status_only is False, then the actual results will be retrieved, else
         only the status of the results will be checked.
         
         Parameters
         ----------
         
         msg_ids : list of msg_ids
             if int:
                 Passed as index to self.history for convenience.
         status_only : bool (default: True)
             if False:
                 Retrieve the actual results of completed tasks.
         
         Returns
         -------
         
         results : dict
             There will always be the keys 'pending' and 'completed', which will
             be lists of msg_ids that are incomplete or complete. If `status_only`
             is False, then completed results will be keyed by their `msg_id`.
         """
         if not isinstance(msg_ids, (list,tuple)):
             msg_ids = [msg_ids]
             
         theids = []
         for msg_id in msg_ids:
             if isinstance(msg_id, int):
                 msg_id = self.history[msg_id]
             if not isinstance(msg_id, basestring):
                 raise TypeError("msg_ids must be str, not %r"%msg_id)
             theids.append(msg_id)
         
         completed = []
         local_results = {}
         
         # comment this block out to temporarily disable local shortcut:
         for msg_id in theids:
             if msg_id in self.results:
                 completed.append(msg_id)
                 local_results[msg_id] = self.results[msg_id]
                 theids.remove(msg_id)
         
         if theids: # some not locally cached
             content = dict(msg_ids=theids, status_only=status_only)
             msg = self.session.send(self._query_socket, "result_request", content=content)
             zmq.select([self._query_socket], [], [])
             idents,msg = self.session.recv(self._query_socket, zmq.NOBLOCK)
             if self.debug:
                 pprint(msg)
             content = msg['content']
             if content['status'] != 'ok':
                 raise self._unwrap_exception(content)
             buffers = msg['buffers']
         else:
             content = dict(completed=[],pending=[])
         
         content['completed'].extend(completed)
         
         if status_only:
             return content
         
         failures = []
         # load cached results into result:
         content.update(local_results)
         # update cache with results:
         for msg_id in sorted(theids):
             if msg_id in content['completed']:
                 rec = content[msg_id]
                 parent = rec['header']
                 header = rec['result_header']
                 rcontent = rec['result_content']
                 iodict = rec['io']
                 if isinstance(rcontent, str):
                     rcontent = self.session.unpack(rcontent)
                 
                 md = self.metadata[msg_id]
                 md.update(self._extract_metadata(header, parent, rcontent))
                 md.update(iodict)
                 
                 if rcontent['status'] == 'ok':
                     res,buffers = util.unserialize_object(buffers)
                 else:
                     print rcontent
                     res = self._unwrap_exception(rcontent)
                     failures.append(res)
                 
                 self.results[msg_id] = res
                 content[msg_id] = res
         
         if len(theids) == 1 and failures:
                 raise failures[0]
         
         error.collect_exceptions(failures, "result_status")
         return content
 
-    @spinfirst
+    @spin_first
     def queue_status(self, targets='all', verbose=False):
         """Fetch the status of engine queues.
         
         Parameters
         ----------
         
         targets : int/str/list of ints/strs
                 the engines whose states are to be queried.
                 default : all
         verbose : bool
                 Whether to return lengths only, or lists of ids for each element
         """
-        targets = self._build_targets(targets)[1]
-        content = dict(targets=targets, verbose=verbose)
+        engine_ids = self._build_targets(targets)[1]
+        content = dict(targets=engine_ids, verbose=verbose)
         self.session.send(self._query_socket, "queue_request", content=content)
         idents,msg = self.session.recv(self._query_socket, 0)
         if self.debug:
             pprint(msg)
         content = msg['content']
         status = content.pop('status')
         if status != 'ok':
             raise self._unwrap_exception(content)
-        return util.rekey(content)
+        content = util.rekey(content)
+        if isinstance(targets, int):
+            return content[targets]
+        else:
+            return content
         
-    @spinfirst
+    @spin_first
     def purge_results(self, jobs=[], targets=[]):
-        """Tell the controller to forget results.
+        """Tell the Hub to forget results.
         
         Individual results can be purged by msg_id, or the entire
         history of specific targets can be purged.
         
         Parameters
         ----------
         
-        jobs : str or list of strs or AsyncResult objects
+        jobs : str or list of str or AsyncResult objects
                 the msg_ids whose results should be forgotten.
         targets : int/str/list of ints/strs
                 The targets, by uuid or int_id, whose entire history is to be purged.
-                Use `targets='all'` to scrub everything from the controller's memory.
+                Use `targets='all'` to scrub everything from the Hub's memory.
                 
                 default : None
         """
         if not targets and not jobs:
             raise ValueError("Must specify at least one of `targets` and `jobs`")
         if targets:
             targets = self._build_targets(targets)[1]
         
         # construct msg_ids from jobs
         msg_ids = []
         if isinstance(jobs, (basestring,AsyncResult)):
             jobs = [jobs]
         bad_ids = filter(lambda obj: not isinstance(obj, (basestring, AsyncResult)), jobs)
         if bad_ids:
             raise TypeError("Invalid msg_id type %r, expected str or AsyncResult"%bad_ids[0])
         for j in jobs:
             if isinstance(j, AsyncResult):
                 msg_ids.extend(j.msg_ids)
             else:
                 msg_ids.append(j)
         
         content = dict(targets=targets, msg_ids=msg_ids)
         self.session.send(self._query_socket, "purge_request", content=content)
         idents, msg = self.session.recv(self._query_socket, 0)
         if self.debug:
             pprint(msg)
         content = msg['content']
         if content['status'] != 'ok':
             raise self._unwrap_exception(content)
 
 
 __all__ = [ 'Client', 
             'depend', 
             'require', 
             'remote',
             'parallel',
             'RemoteFunction',
             'ParallelFunction',
             'DirectView',
             'LoadBalancedView',
             'AsyncResult',
             'AsyncMapResult',
             'Reference'
             ]

IPython/zmq/parallel/dependency.py

@@ -1,159 +1,184 b''
 """Dependency utilities"""
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2010-2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
-from IPython.external.decorator import decorator
-
 from .asyncresult import AsyncResult
 from .error import UnmetDependency
-
+from .util import interactive
 
 class depend(object):
     """Dependency decorator, for use with tasks.
     
     `@depend` lets you define a function for engine dependencies
     just like you use `apply` for tasks.
     
     
     Examples
     --------
     ::
     
         @depend(df, a,b, c=5)
         def f(m,n,p)
         
         view.apply(f, 1,2,3)
     
     will call df(a,b,c=5) on the engine, and if it returns False or
     raises an UnmetDependency error, then the task will not be run
     and another engine will be tried.
     """
     def __init__(self, f, *args, **kwargs):
         self.f = f
         self.args = args
         self.kwargs = kwargs
     
     def __call__(self, f):
         return dependent(f, self.f, *self.args, **self.kwargs)
 
 class dependent(object):
     """A function that depends on another function.
     This is an object to prevent the closure used
     in traditional decorators, which are not picklable.
     """
 
     def __init__(self, f, df, *dargs, **dkwargs):
         self.f = f
         self.func_name = getattr(f, '__name__', 'f')
         self.df = df
         self.dargs = dargs
         self.dkwargs = dkwargs
 
     def __call__(self, *args, **kwargs):
+        # if hasattr(self.f, 'func_globals') and hasattr(self.df, 'func_globals'):
+        #     self.df.func_globals = self.f.func_globals
         if self.df(*self.dargs, **self.dkwargs) is False:
             raise UnmetDependency()
         return self.f(*args, **kwargs)
     
     @property
     def __name__(self):
         return self.func_name
 
+@interactive
 def _require(*names):
     """Helper for @require decorator."""
+    from IPython.zmq.parallel.error import UnmetDependency
+    user_ns = globals()
     for name in names:
+        if name in user_ns:
+            continue
         try:
-            __import__(name)
+            exec 'import %s'%name in user_ns
         except ImportError:
-            return False
+            raise UnmetDependency(name)
     return True
 
 def require(*names):
     """Simple decorator for requiring names to be importable.
     
     Examples
     --------
     
     In [1]: @require('numpy')
        ...: def norm(a):
        ...:     import numpy
        ...:     return numpy.linalg.norm(a,2)
     """
     return depend(_require, *names)
 
 class Dependency(set):
     """An object for representing a set of msg_id dependencies.
     
     Subclassed from set().
     
     Parameters
     ----------
     dependencies: list/set of msg_ids or AsyncResult objects or output of Dependency.as_dict()
         The msg_ids to depend on
     all : bool [default True]
         Whether the dependency should be considered met when *all* depending tasks have completed
         or only when *any* have been completed.
-    success_only : bool [default True]
-        Whether to consider only successes for Dependencies, or consider failures as well.
-        If `all=success_only=True`, then this task will fail with an ImpossibleDependency
+    success : bool [default True]
+        Whether to consider successes as fulfilling dependencies.
+    failure : bool [default False]
+        Whether to consider failures as fulfilling dependencies.
+    
+    If `all=success=True` and `failure=False`, then the task will fail with an ImpossibleDependency
         as soon as the first depended-upon task fails.
     """
     
     all=True
-    success_only=True
+    success=True
+    failure=True
     
-    def __init__(self, dependencies=[], all=True, success_only=True):
+    def __init__(self, dependencies=[], all=True, success=True, failure=False):
         if isinstance(dependencies, dict):
             # load from dict
             all = dependencies.get('all', True)
-            success_only = dependencies.get('success_only', success_only)
+            success = dependencies.get('success', success)
+            failure = dependencies.get('failure', failure)
             dependencies = dependencies.get('dependencies', [])
         ids = []
-        if isinstance(dependencies, AsyncResult):
-            ids.extend(AsyncResult.msg_ids)
-        else:
+        
+        # extract ids from various sources:
+        if isinstance(dependencies, (basestring, AsyncResult)):
+            dependencies = [dependencies]
             for d in dependencies:
                 if isinstance(d, basestring):
                     ids.append(d)
                 elif isinstance(d, AsyncResult):
                     ids.extend(d.msg_ids)
                 else:
                     raise TypeError("invalid dependency type: %r"%type(d))
+        
         set.__init__(self, ids)
         self.all = all
-        self.success_only=success_only
+        if not (success or failure):
+            raise ValueError("Must depend on at least one of successes or failures!")
+        self.success=success
+        self.failure = failure
     
     def check(self, completed, failed=None):
-        if failed is not None and not self.success_only:
-            completed = completed.union(failed)
+        """check whether our dependencies have been met."""
         if len(self) == 0:
             return True
+        against = set()
+        if self.success:
+            against = completed
+        if failed is not None and self.failure:
+            against = against.union(failed)
         if self.all:
-            return self.issubset(completed)
+            return self.issubset(against)
         else:
-            return not self.isdisjoint(completed)
+            return not self.isdisjoint(against)
     
-    def unreachable(self, failed):
-        if len(self) == 0 or len(failed) == 0 or not self.success_only:
+    def unreachable(self, completed, failed=None):
+        """return whether this dependency has become impossible."""
+        if len(self) == 0:
             return False
-        # print self, self.success_only, self.all, failed
+        against = set()
+        if not self.success:
+            against = completed
+        if failed is not None and not self.failure:
+            against = against.union(failed)
         if self.all:
-            return not self.isdisjoint(failed)
+            return not self.isdisjoint(against)
         else:
-            return self.issubset(failed)
+            return self.issubset(against)
         
     
     def as_dict(self):
         """Represent this dependency as a dict. For json compatibility."""
         return dict(
             dependencies=list(self),
             all=self.all,
-            success_only=self.success_only,
+            success=self.success,
+            failure=self.failure
         )
     
 
 __all__ = ['depend', 'require', 'dependent', 'Dependency']
 

IPython/zmq/parallel/hub.py

@@ -1,1039 +1,1035 b''
 #!/usr/bin/env python
 """The IPython Controller Hub with 0MQ
 This is the master object that handles connections from engines and clients,
 and monitors traffic through the various queues.
 """
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2010  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 from __future__ import print_function
 
 import sys
 import time
 from datetime import datetime
 
 import zmq
 from zmq.eventloop import ioloop
 from zmq.eventloop.zmqstream import ZMQStream
 
 # internal:
 from IPython.utils.importstring import import_item
 from IPython.utils.traitlets import HasTraits, Instance, Int, CStr, Str, Dict, Set, List, Bool
 
 from .entry_point import select_random_ports
 from .factory import RegistrationFactory, LoggingFactory
 
 from . import error
 from .heartmonitor import HeartMonitor
 from .util import validate_url_container, ISO8601
 
 #-----------------------------------------------------------------------------
 # Code
 #-----------------------------------------------------------------------------
 
 def _passer(*args, **kwargs):
     return
 
 def _printer(*args, **kwargs):
     print (args)
     print (kwargs)
 
 def init_record(msg):
     """Initialize a TaskRecord based on a request."""
     header = msg['header']
     return {
         'msg_id' : header['msg_id'],
         'header' : header,
         'content': msg['content'],
         'buffers': msg['buffers'],
         'submitted': datetime.strptime(header['date'], ISO8601),
         'client_uuid' : None,
         'engine_uuid' : None,
         'started': None,
         'completed': None,
         'resubmitted': None,
         'result_header' : None,
         'result_content' : None,
         'result_buffers' : None,
         'queue' : None,
         'pyin' : None,
         'pyout': None,
         'pyerr': None,
         'stdout': '',
         'stderr': '',
     }
 
 
 class EngineConnector(HasTraits):
     """A simple object for accessing the various zmq connections of an object.
     Attributes are:
     id (int): engine ID
     uuid (str): uuid (unused?)
     queue (str): identity of queue's XREQ socket
     registration (str): identity of registration XREQ socket
     heartbeat (str): identity of heartbeat XREQ socket
     """
     id=Int(0)
     queue=Str()
     control=Str()
     registration=Str()
     heartbeat=Str()
     pending=Set()
 
 class HubFactory(RegistrationFactory):
     """The Configurable for setting up a Hub."""
     
     # name of a scheduler scheme
     scheme = Str('leastload', config=True)
     
     # port-pairs for monitoredqueues:
     hb = Instance(list, config=True)
     def _hb_default(self):
         return select_random_ports(2)
     
     mux = Instance(list, config=True)
     def _mux_default(self):
         return select_random_ports(2)
     
     task = Instance(list, config=True)
     def _task_default(self):
         return select_random_ports(2)
     
     control = Instance(list, config=True)
     def _control_default(self):
         return select_random_ports(2)
     
     iopub = Instance(list, config=True)
     def _iopub_default(self):
         return select_random_ports(2)
     
     # single ports:
     mon_port = Instance(int, config=True)
     def _mon_port_default(self):
         return select_random_ports(1)[0]
     
     notifier_port = Instance(int, config=True)
     def _notifier_port_default(self):
         return select_random_ports(1)[0]
     
     ping = Int(1000, config=True) # ping frequency
     
     engine_ip = CStr('127.0.0.1', config=True)
     engine_transport = CStr('tcp', config=True)
     
     client_ip = CStr('127.0.0.1', config=True)
     client_transport = CStr('tcp', config=True)
     
     monitor_ip = CStr('127.0.0.1', config=True)
     monitor_transport = CStr('tcp', config=True)
     
     monitor_url = CStr('')
     
     db_class = CStr('IPython.zmq.parallel.dictdb.DictDB', config=True)
     
     # not configurable
     db = Instance('IPython.zmq.parallel.dictdb.BaseDB')
     heartmonitor = Instance('IPython.zmq.parallel.heartmonitor.HeartMonitor')
     subconstructors = List()
     _constructed = Bool(False)
     
     def _ip_changed(self, name, old, new):
         self.engine_ip = new
         self.client_ip = new
         self.monitor_ip = new
         self._update_monitor_url()
     
     def _update_monitor_url(self):
         self.monitor_url = "%s://%s:%i"%(self.monitor_transport, self.monitor_ip, self.mon_port)
     
     def _transport_changed(self, name, old, new):
         self.engine_transport = new
         self.client_transport = new
         self.monitor_transport = new
         self._update_monitor_url()
         
     def __init__(self, **kwargs):
         super(HubFactory, self).__init__(**kwargs)
         self._update_monitor_url()
         # self.on_trait_change(self._sync_ips, 'ip')
         # self.on_trait_change(self._sync_transports, 'transport')
         self.subconstructors.append(self.construct_hub)
     
     
     def construct(self):
         assert not self._constructed, "already constructed!"
         
         for subc in self.subconstructors:
             subc()
         
         self._constructed = True
         
     
     def start(self):
         assert self._constructed, "must be constructed by self.construct() first!"
         self.heartmonitor.start()
         self.log.info("Heartmonitor started")
     
     def construct_hub(self):
         """construct"""
         client_iface = "%s://%s:"%(self.client_transport, self.client_ip) + "%i"
         engine_iface = "%s://%s:"%(self.engine_transport, self.engine_ip) + "%i"
         
         ctx = self.context
         loop = self.loop
         
         # Registrar socket
         q = ZMQStream(ctx.socket(zmq.XREP), loop)
         q.bind(client_iface % self.regport)
         self.log.info("Hub listening on %s for registration."%(client_iface%self.regport))
         if self.client_ip != self.engine_ip:
             q.bind(engine_iface % self.regport)
             self.log.info("Hub listening on %s for registration."%(engine_iface%self.regport))
         
         ### Engine connections ###
 
         # heartbeat
         hpub = ctx.socket(zmq.PUB)
         hpub.bind(engine_iface % self.hb[0])
         hrep = ctx.socket(zmq.XREP)
         hrep.bind(engine_iface % self.hb[1])
         self.heartmonitor = HeartMonitor(loop=loop, pingstream=ZMQStream(hpub,loop), pongstream=ZMQStream(hrep,loop), 
                                 period=self.ping, logname=self.log.name)
 
         ### Client connections ###
         # Notifier socket
         n = ZMQStream(ctx.socket(zmq.PUB), loop)
         n.bind(client_iface%self.notifier_port)
 
         ### build and launch the queues ###
 
         # monitor socket
         sub = ctx.socket(zmq.SUB)
         sub.setsockopt(zmq.SUBSCRIBE, "")
         sub.bind(self.monitor_url)
         sub.bind('inproc://monitor')
         sub = ZMQStream(sub, loop)
         
         # connect the db
         self.log.info('Hub using DB backend: %r'%(self.db_class.split()[-1]))
         # cdir = self.config.Global.cluster_dir
         self.db = import_item(self.db_class)(session=self.session.session, config=self.config)
         time.sleep(.25)
 
         # build connection dicts
         self.engine_info = {
             'control' : engine_iface%self.control[1],
             'mux': engine_iface%self.mux[1],
             'heartbeat': (engine_iface%self.hb[0], engine_iface%self.hb[1]),
             'task' : engine_iface%self.task[1],
             'iopub' : engine_iface%self.iopub[1],
             # 'monitor' : engine_iface%self.mon_port,
             }
 
         self.client_info = {
             'control' : client_iface%self.control[0],
             'mux': client_iface%self.mux[0],
             'task' : (self.scheme, client_iface%self.task[0]),
             'iopub' : client_iface%self.iopub[0],
             'notification': client_iface%self.notifier_port
             }
         self.log.debug("Hub engine addrs: %s"%self.engine_info)
         self.log.debug("Hub client addrs: %s"%self.client_info)
         self.hub = Hub(loop=loop, session=self.session, monitor=sub, heartmonitor=self.heartmonitor,
                 query=q, notifier=n, db=self.db,
                 engine_info=self.engine_info, client_info=self.client_info,
                 logname=self.log.name)
     
 
 class Hub(LoggingFactory):
     """The IPython Controller Hub with 0MQ connections
     
     Parameters
     ==========
     loop: zmq IOLoop instance
     session: StreamSession object
     <removed> context: zmq context for creating new connections (?)
     queue: ZMQStream for monitoring the command queue (SUB)
     query: ZMQStream for engine registration and client queries requests (XREP)
     heartbeat: HeartMonitor object checking the pulse of the engines
     notifier: ZMQStream for broadcasting engine registration changes (PUB)
     db: connection to db for out of memory logging of commands
                 NotImplemented
     engine_info: dict of zmq connection information for engines to connect
                 to the queues.
     client_info: dict of zmq connection information for engines to connect
                 to the queues.
     """
     # internal data structures:
     ids=Set() # engine IDs
     keytable=Dict()
     by_ident=Dict()
     engines=Dict()
     clients=Dict()
     hearts=Dict()
     pending=Set()
     queues=Dict()  # pending msg_ids keyed by engine_id
     tasks=Dict() # pending msg_ids submitted as tasks, keyed by client_id
     completed=Dict() # completed msg_ids keyed by engine_id
     all_completed=Set() # completed msg_ids keyed by engine_id
     # mia=None
     incoming_registrations=Dict()
     registration_timeout=Int()
     _idcounter=Int(0)
     
     # objects from constructor:
     loop=Instance(ioloop.IOLoop)
     query=Instance(ZMQStream)
     monitor=Instance(ZMQStream)
     heartmonitor=Instance(HeartMonitor)
     notifier=Instance(ZMQStream)
     db=Instance(object)
     client_info=Dict()
     engine_info=Dict()
     
     
     def __init__(self, **kwargs):
         """
         # universal:
         loop: IOLoop for creating future connections
         session: streamsession for sending serialized data
         # engine:
         queue: ZMQStream for monitoring queue messages
         query: ZMQStream for engine+client registration and client requests
         heartbeat: HeartMonitor object for tracking engines
         # extra:
         db: ZMQStream for db connection (NotImplemented)
         engine_info: zmq address/protocol dict for engine connections
         client_info: zmq address/protocol dict for client connections
         """
         
         super(Hub, self).__init__(**kwargs)
         self.registration_timeout = max(5000, 2*self.heartmonitor.period)
         
         # validate connection dicts:
         for k,v in self.client_info.iteritems():
             if k == 'task':
                 validate_url_container(v[1])
             else:
                 validate_url_container(v)
         # validate_url_container(self.client_info)
         validate_url_container(self.engine_info)
         
         # register our callbacks
         self.query.on_recv(self.dispatch_query)
         self.monitor.on_recv(self.dispatch_monitor_traffic)
         
         self.heartmonitor.add_heart_failure_handler(self.handle_heart_failure)
         self.heartmonitor.add_new_heart_handler(self.handle_new_heart)
         
         self.monitor_handlers = { 'in' : self.save_queue_request,
                                 'out': self.save_queue_result,
                                 'intask': self.save_task_request,
                                 'outtask': self.save_task_result,
                                 'tracktask': self.save_task_destination,
                                 'incontrol': _passer,
                                 'outcontrol': _passer,
                                 'iopub': self.save_iopub_message,
         }
         
         self.query_handlers = {'queue_request': self.queue_status,
                                 'result_request': self.get_results,
                                 'purge_request': self.purge_results,
                                 'load_request': self.check_load,
                                 'resubmit_request': self.resubmit_task,
                                 'shutdown_request': self.shutdown_request,
                                 'registration_request' : self.register_engine,
                                 'unregistration_request' : self.unregister_engine,
                                 'connection_request': self.connection_request,
         }
         
         self.log.info("hub::created hub")
     
     @property
     def _next_id(self):
         """gemerate a new ID.
         
         No longer reuse old ids, just count from 0."""
         newid = self._idcounter
         self._idcounter += 1
         return newid
         # newid = 0
         # incoming = [id[0] for id in self.incoming_registrations.itervalues()]
         # # print newid, self.ids, self.incoming_registrations
         # while newid in self.ids or newid in incoming:
         #     newid += 1
         # return newid
     
     #-----------------------------------------------------------------------------
     # message validation
     #-----------------------------------------------------------------------------
     
     def _validate_targets(self, targets):
         """turn any valid targets argument into a list of integer ids"""
         if targets is None:
             # default to all
             targets = self.ids
             
         if isinstance(targets, (int,str,unicode)):
             # only one target specified
             targets = [targets]
         _targets = []
         for t in targets:
             # map raw identities to ids
             if isinstance(t, (str,unicode)):
                 t = self.by_ident.get(t, t)
             _targets.append(t)
         targets = _targets
         bad_targets = [ t for t in targets if t not in self.ids ]
         if bad_targets:
             raise IndexError("No Such Engine: %r"%bad_targets)
         if not targets:
             raise IndexError("No Engines Registered")
         return targets
     
     #-----------------------------------------------------------------------------
     # dispatch methods (1 per stream)
     #-----------------------------------------------------------------------------
     
     # def dispatch_registration_request(self, msg):
     #     """"""
     #     self.log.debug("registration::dispatch_register_request(%s)"%msg)
     #     idents,msg = self.session.feed_identities(msg)
     #     if not idents:
     #         self.log.error("Bad Query Message: %s"%msg, exc_info=True)
     #         return
     #     try:
     #         msg = self.session.unpack_message(msg,content=True)
     #     except:
     #         self.log.error("registration::got bad registration message: %s"%msg, exc_info=True)
     #         return
     #     
     #     msg_type = msg['msg_type']
     #     content = msg['content']
     #     
     #     handler = self.query_handlers.get(msg_type, None)
     #     if handler is None:
     #         self.log.error("registration::got bad registration message: %s"%msg)
     #     else:
     #         handler(idents, msg)
     
     def dispatch_monitor_traffic(self, msg):
         """all ME and Task queue messages come through here, as well as
         IOPub traffic."""
         self.log.debug("monitor traffic: %s"%msg[:2])
         switch = msg[0]
         idents, msg = self.session.feed_identities(msg[1:])
         if not idents:
             self.log.error("Bad Monitor Message: %s"%msg)
             return
         handler = self.monitor_handlers.get(switch, None)
         if handler is not None:
             handler(idents, msg)
         else:
             self.log.error("Invalid monitor topic: %s"%switch)
         
     
     def dispatch_query(self, msg):
         """Route registration requests and queries from clients."""
         idents, msg = self.session.feed_identities(msg)
         if not idents:
             self.log.error("Bad Query Message: %s"%msg)
             return
         client_id = idents[0]
         try:
             msg = self.session.unpack_message(msg, content=True)
         except:
             content = error.wrap_exception()
             self.log.error("Bad Query Message: %s"%msg, exc_info=True)
             self.session.send(self.query, "hub_error", ident=client_id, 
                     content=content)
             return
         
         # print client_id, header, parent, content
         #switch on message type:
         msg_type = msg['msg_type']
         self.log.info("client::client %s requested %s"%(client_id, msg_type))
         handler = self.query_handlers.get(msg_type, None)
         try:
             assert handler is not None, "Bad Message Type: %s"%msg_type
         except:
             content = error.wrap_exception()
             self.log.error("Bad Message Type: %s"%msg_type, exc_info=True)
             self.session.send(self.query, "hub_error", ident=client_id, 
                     content=content)
             return
         else:
             handler(idents, msg)
             
     def dispatch_db(self, msg):
         """"""
         raise NotImplementedError
     
     #---------------------------------------------------------------------------
     # handler methods (1 per event)
     #---------------------------------------------------------------------------
     
     #----------------------- Heartbeat --------------------------------------
     
     def handle_new_heart(self, heart):
         """handler to attach to heartbeater.
         Called when a new heart starts to beat.
         Triggers completion of registration."""
         self.log.debug("heartbeat::handle_new_heart(%r)"%heart)
         if heart not in self.incoming_registrations:
             self.log.info("heartbeat::ignoring new heart: %r"%heart)
         else:
             self.finish_registration(heart)
         
     
     def handle_heart_failure(self, heart):
         """handler to attach to heartbeater.
         called when a previously registered heart fails to respond to beat request.
         triggers unregistration"""
         self.log.debug("heartbeat::handle_heart_failure(%r)"%heart)
         eid = self.hearts.get(heart, None)
         queue = self.engines[eid].queue
         if eid is None:
             self.log.info("heartbeat::ignoring heart failure %r"%heart)
         else:
             self.unregister_engine(heart, dict(content=dict(id=eid, queue=queue)))
     
     #----------------------- MUX Queue Traffic ------------------------------
     
     def save_queue_request(self, idents, msg):
         if len(idents) < 2:
             self.log.error("invalid identity prefix: %s"%idents)
             return
         queue_id, client_id = idents[:2]
         try:
             msg = self.session.unpack_message(msg, content=False)
         except:
             self.log.error("queue::client %r sent invalid message to %r: %s"%(client_id, queue_id, msg), exc_info=True)
             return
         
         eid = self.by_ident.get(queue_id, None)
         if eid is None:
             self.log.error("queue::target %r not registered"%queue_id)
             self.log.debug("queue::    valid are: %s"%(self.by_ident.keys()))
             return
             
         header = msg['header']
         msg_id = header['msg_id']
         record = init_record(msg)
         record['engine_uuid'] = queue_id
         record['client_uuid'] = client_id
         record['queue'] = 'mux'
 
         self.pending.add(msg_id)
         self.queues[eid].append(msg_id)
         self.db.add_record(msg_id, record)
     
     def save_queue_result(self, idents, msg):
         if len(idents) < 2:
             self.log.error("invalid identity prefix: %s"%idents)
             return
             
         client_id, queue_id = idents[:2]
         try:
             msg = self.session.unpack_message(msg, content=False)
         except:
             self.log.error("queue::engine %r sent invalid message to %r: %s"%(
                     queue_id,client_id, msg), exc_info=True)
             return
         
         eid = self.by_ident.get(queue_id, None)
         if eid is None:
             self.log.error("queue::unknown engine %r is sending a reply: "%queue_id)
             self.log.debug("queue::       %s"%msg[2:])
             return
         
         parent = msg['parent_header']
         if not parent:
             return
         msg_id = parent['msg_id']
         if msg_id in self.pending:
             self.pending.remove(msg_id)
             self.all_completed.add(msg_id)
             self.queues[eid].remove(msg_id)
             self.completed[eid].append(msg_id)
         elif msg_id not in self.all_completed:
             # it could be a result from a dead engine that died before delivering the
             # result
             self.log.warn("queue:: unknown msg finished %s"%msg_id)
             return
         # update record anyway, because the unregistration could have been premature
         rheader = msg['header']
         completed = datetime.strptime(rheader['date'], ISO8601)
         started = rheader.get('started', None)
         if started is not None:
             started = datetime.strptime(started, ISO8601)
         result = {
             'result_header' : rheader,
             'result_content': msg['content'],
             'started' : started,
             'completed' : completed
         }
 
         result['result_buffers'] = msg['buffers']
         self.db.update_record(msg_id, result)
         
             
     #--------------------- Task Queue Traffic ------------------------------
     
     def save_task_request(self, idents, msg):
         """Save the submission of a task."""
         client_id = idents[0]
         
         try:
             msg = self.session.unpack_message(msg, content=False)
         except:
             self.log.error("task::client %r sent invalid task message: %s"%(
                     client_id, msg), exc_info=True)
             return
         record = init_record(msg)
 
         record['client_uuid'] = client_id
         record['queue'] = 'task'
         header = msg['header']
         msg_id = header['msg_id']
         self.pending.add(msg_id)
         self.db.add_record(msg_id, record)
     
     def save_task_result(self, idents, msg):
         """save the result of a completed task."""
         client_id = idents[0]
         try:
             msg = self.session.unpack_message(msg, content=False)
         except:
             self.log.error("task::invalid task result message send to %r: %s"%(
                     client_id, msg), exc_info=True)
             raise
             return
         
         parent = msg['parent_header']
         if not parent:
             # print msg
             self.log.warn("Task %r had no parent!"%msg)
             return
         msg_id = parent['msg_id']
         
         header = msg['header']
         engine_uuid = header.get('engine', None)
         eid = self.by_ident.get(engine_uuid, None)
         
         if msg_id in self.pending:
             self.pending.remove(msg_id)
             self.all_completed.add(msg_id)
             if eid is not None:
                 self.completed[eid].append(msg_id)
                 if msg_id in self.tasks[eid]:
                     self.tasks[eid].remove(msg_id)
             completed = datetime.strptime(header['date'], ISO8601)
             started = header.get('started', None)
             if started is not None:
                 started = datetime.strptime(started, ISO8601)
             result = {
                 'result_header' : header,
                 'result_content': msg['content'],
                 'started' : started,
                 'completed' : completed,
                 'engine_uuid': engine_uuid
             }
 
             result['result_buffers'] = msg['buffers']
             self.db.update_record(msg_id, result)
             
         else:
             self.log.debug("task::unknown task %s finished"%msg_id)
     
     def save_task_destination(self, idents, msg):
         try:
             msg = self.session.unpack_message(msg, content=True)
         except:
             self.log.error("task::invalid task tracking message", exc_info=True)
             return
         content = msg['content']
         # print (content)
         msg_id = content['msg_id']
         engine_uuid = content['engine_id']
         eid = self.by_ident[engine_uuid]
         
         self.log.info("task::task %s arrived on %s"%(msg_id, eid))
         # if msg_id in self.mia:
         #     self.mia.remove(msg_id)
         # else:
         #     self.log.debug("task::task %s not listed as MIA?!"%(msg_id))
         
         self.tasks[eid].append(msg_id)
         # self.pending[msg_id][1].update(received=datetime.now(),engine=(eid,engine_uuid))
         self.db.update_record(msg_id, dict(engine_uuid=engine_uuid))
     
     def mia_task_request(self, idents, msg):
         raise NotImplementedError
         client_id = idents[0]
         # content = dict(mia=self.mia,status='ok')
         # self.session.send('mia_reply', content=content, idents=client_id)
         
         
     #--------------------- IOPub Traffic ------------------------------
     
     def save_iopub_message(self, topics, msg):
         """save an iopub message into the db"""
         # print (topics)
         try:
             msg = self.session.unpack_message(msg, content=True)
         except:
             self.log.error("iopub::invalid IOPub message", exc_info=True)
             return
         
         parent = msg['parent_header']
         if not parent:
             self.log.error("iopub::invalid IOPub message: %s"%msg)
             return
         msg_id = parent['msg_id']
         msg_type = msg['msg_type']
         content = msg['content']
         
         # ensure msg_id is in db
         try:
             rec = self.db.get_record(msg_id)
         except:
             self.log.error("iopub::IOPub message has invalid parent", exc_info=True)
             return
         # stream
         d = {}
         if msg_type == 'stream':
             name = content['name']
             s = rec[name] or ''
             d[name] = s + content['data']
             
         elif msg_type == 'pyerr':
             d['pyerr'] = content
         else:
             d[msg_type] = content['data']
         
         self.db.update_record(msg_id, d)
         
     
             
     #-------------------------------------------------------------------------
     # Registration requests
     #-------------------------------------------------------------------------
         
     def connection_request(self, client_id, msg):
         """Reply with connection addresses for clients."""
         self.log.info("client::client %s connected"%client_id)
         content = dict(status='ok')
         content.update(self.client_info)
         jsonable = {}
         for k,v in self.keytable.iteritems():
             jsonable[str(k)] = v
         content['engines'] = jsonable
         self.session.send(self.query, 'connection_reply', content, parent=msg, ident=client_id)
     
     def register_engine(self, reg, msg):
         """Register a new engine."""
         content = msg['content']
         try:
             queue = content['queue']
         except KeyError:
             self.log.error("registration::queue not specified", exc_info=True)
             return
         heart = content.get('heartbeat', None)
         """register a new engine, and create the socket(s) necessary"""
         eid = self._next_id
         # print (eid, queue, reg, heart)
         
         self.log.debug("registration::register_engine(%i, %r, %r, %r)"%(eid, queue, reg, heart))
         
         content = dict(id=eid,status='ok')
         content.update(self.engine_info)
         # check if requesting available IDs:
         if queue in self.by_ident:
             try:
                 raise KeyError("queue_id %r in use"%queue)
             except:
                 content = error.wrap_exception()
                 self.log.error("queue_id %r in use"%queue, exc_info=True)
         elif heart in self.hearts: # need to check unique hearts?
             try:
                 raise KeyError("heart_id %r in use"%heart)
             except:
                 self.log.error("heart_id %r in use"%heart, exc_info=True)
                 content = error.wrap_exception()
         else:
             for h, pack in self.incoming_registrations.iteritems():
                 if heart == h:
                     try:
                         raise KeyError("heart_id %r in use"%heart)
                     except:
                         self.log.error("heart_id %r in use"%heart, exc_info=True)
                         content = error.wrap_exception()
                     break
                 elif queue == pack[1]:
                     try:
                         raise KeyError("queue_id %r in use"%queue)
                     except:
                         self.log.error("queue_id %r in use"%queue, exc_info=True)
                         content = error.wrap_exception()
                     break
         
         msg = self.session.send(self.query, "registration_reply", 
                 content=content, 
                 ident=reg)
         
         if content['status'] == 'ok':
             if heart in self.heartmonitor.hearts:
                 # already beating
                 self.incoming_registrations[heart] = (eid,queue,reg[0],None)
                 self.finish_registration(heart)
             else:
                 purge = lambda : self._purge_stalled_registration(heart)
                 dc = ioloop.DelayedCallback(purge, self.registration_timeout, self.loop)
                 dc.start()
                 self.incoming_registrations[heart] = (eid,queue,reg[0],dc)
         else:
             self.log.error("registration::registration %i failed: %s"%(eid, content['evalue']))
         return eid
     
     def unregister_engine(self, ident, msg):
         """Unregister an engine that explicitly requested to leave."""
         try:
             eid = msg['content']['id']
         except:
             self.log.error("registration::bad engine id for unregistration: %s"%ident, exc_info=True)
             return
         self.log.info("registration::unregister_engine(%s)"%eid)
         # print (eid)
         content=dict(id=eid, queue=self.engines[eid].queue)
         self.ids.remove(eid)
         uuid = self.keytable.pop(eid)
         ec = self.engines.pop(eid)
         self.hearts.pop(ec.heartbeat)
         self.by_ident.pop(ec.queue)
         self.completed.pop(eid)
         self._handle_stranded_msgs(eid, uuid)
             ############## TODO: HANDLE IT ################
         
         if self.notifier:
             self.session.send(self.notifier, "unregistration_notification", content=content)
     
     def _handle_stranded_msgs(self, eid, uuid):
         """Handle messages known to be on an engine when the engine unregisters.
         
         It is possible that this will fire prematurely - that is, an engine will
         go down after completing a result, and the client will be notified
         that the result failed and later receive the actual result.
         """
         
         outstanding = self.queues.pop(eid)
         
         for msg_id in outstanding:
             self.pending.remove(msg_id)
             self.all_completed.add(msg_id)
             try:
                 raise error.EngineError("Engine %r died while running task %r"%(eid, msg_id))
             except:
                 content = error.wrap_exception()
             # build a fake header:
             header = {}
             header['engine'] = uuid
             header['date'] = datetime.now().strftime(ISO8601)
             rec = dict(result_content=content, result_header=header, result_buffers=[])
             rec['completed'] = header['date']
             rec['engine_uuid'] = uuid
             self.db.update_record(msg_id, rec)
     
     def finish_registration(self, heart):
         """Second half of engine registration, called after our HeartMonitor
         has received a beat from the Engine's Heart."""
         try: 
             (eid,queue,reg,purge) = self.incoming_registrations.pop(heart)
         except KeyError:
             self.log.error("registration::tried to finish nonexistant registration", exc_info=True)
             return
         self.log.info("registration::finished registering engine %i:%r"%(eid,queue))
         if purge is not None:
             purge.stop()
         control = queue
         self.ids.add(eid)
         self.keytable[eid] = queue
         self.engines[eid] = EngineConnector(id=eid, queue=queue, registration=reg, 
                                     control=control, heartbeat=heart)
         self.by_ident[queue] = eid
         self.queues[eid] = list()
         self.tasks[eid] = list()
         self.completed[eid] = list()
         self.hearts[heart] = eid
         content = dict(id=eid, queue=self.engines[eid].queue)
         if self.notifier:
             self.session.send(self.notifier, "registration_notification", content=content)
         self.log.info("engine::Engine Connected: %i"%eid)
     
     def _purge_stalled_registration(self, heart):
         if heart in self.incoming_registrations:
             eid = self.incoming_registrations.pop(heart)[0]
             self.log.info("registration::purging stalled registration: %i"%eid)
         else:
             pass
             
     #-------------------------------------------------------------------------
     # Client Requests
     #-------------------------------------------------------------------------
     
     def shutdown_request(self, client_id, msg):
         """handle shutdown request."""
-        # s = self.context.socket(zmq.XREQ)
-        # s.connect(self.client_connections['mux'])
-        # time.sleep(0.1)
-        # for eid,ec in self.engines.iteritems():
-        #     self.session.send(s, 'shutdown_request', content=dict(restart=False), ident=ec.queue)
-        # time.sleep(1)
         self.session.send(self.query, 'shutdown_reply', content={'status': 'ok'}, ident=client_id)
+        # also notify other clients of shutdown
+        self.session.send(self.notifier, 'shutdown_notice', content={'status': 'ok'})
         dc = ioloop.DelayedCallback(lambda : self._shutdown(), 1000, self.loop)
         dc.start()
     
     def _shutdown(self):
         self.log.info("hub::hub shutting down.")
         time.sleep(0.1)
         sys.exit(0)
         
     
     def check_load(self, client_id, msg):
         content = msg['content']
         try:
             targets = content['targets']
             targets = self._validate_targets(targets)
         except:
             content = error.wrap_exception()
             self.session.send(self.query, "hub_error", 
                     content=content, ident=client_id)
             return
         
         content = dict(status='ok')
         # loads = {}
         for t in targets:
             content[bytes(t)] = len(self.queues[t])+len(self.tasks[t])
         self.session.send(self.query, "load_reply", content=content, ident=client_id)
             
     
     def queue_status(self, client_id, msg):
         """Return the Queue status of one or more targets.
         if verbose: return the msg_ids
         else: return len of each type.
         keys: queue (pending MUX jobs)
             tasks (pending Task jobs)
             completed (finished jobs from both queues)"""
         content = msg['content']
         targets = content['targets']
         try:
             targets = self._validate_targets(targets)
         except:
             content = error.wrap_exception()
             self.session.send(self.query, "hub_error", 
                     content=content, ident=client_id)
             return
         verbose = content.get('verbose', False)
         content = dict(status='ok')
         for t in targets:
             queue = self.queues[t]
             completed = self.completed[t]
             tasks = self.tasks[t]
             if not verbose:
                 queue = len(queue)
                 completed = len(completed)
                 tasks = len(tasks)
             content[bytes(t)] = {'queue': queue, 'completed': completed , 'tasks': tasks}
             # pending
         self.session.send(self.query, "queue_reply", content=content, ident=client_id)
     
     def purge_results(self, client_id, msg):
         """Purge results from memory. This method is more valuable before we move
         to a DB based message storage mechanism."""
         content = msg['content']
         msg_ids = content.get('msg_ids', [])
         reply = dict(status='ok')
         if msg_ids == 'all':
             self.db.drop_matching_records(dict(completed={'$ne':None}))
         else:
             for msg_id in msg_ids:
                 if msg_id in self.all_completed:
                     self.db.drop_record(msg_id)
                 else:
                     if msg_id in self.pending:
                         try:
                             raise IndexError("msg pending: %r"%msg_id)
                         except:
                             reply = error.wrap_exception()
                     else:
                         try:
                             raise IndexError("No such msg: %r"%msg_id)
                         except:
                             reply = error.wrap_exception()
                     break
             eids = content.get('engine_ids', [])
             for eid in eids:
                 if eid not in self.engines:
                     try:
                         raise IndexError("No such engine: %i"%eid)
                     except:
                         reply = error.wrap_exception()
                     break
                 msg_ids = self.completed.pop(eid)
                 uid = self.engines[eid].queue
                 self.db.drop_matching_records(dict(engine_uuid=uid, completed={'$ne':None}))
         
         self.session.send(self.query, 'purge_reply', content=reply, ident=client_id)
     
     def resubmit_task(self, client_id, msg, buffers):
         """Resubmit a task."""
         raise NotImplementedError
     
     def get_results(self, client_id, msg):
         """Get the result of 1 or more messages."""
         content = msg['content']
         msg_ids = sorted(set(content['msg_ids']))
         statusonly = content.get('status_only', False)
         pending = []
         completed = []
         content = dict(status='ok')
         content['pending'] = pending
         content['completed'] = completed
         buffers = []
         if not statusonly:
             content['results'] = {}
             records = self.db.find_records(dict(msg_id={'$in':msg_ids}))
         for msg_id in msg_ids:
             if msg_id in self.pending:
                 pending.append(msg_id)
             elif msg_id in self.all_completed:
                 completed.append(msg_id)
                 if not statusonly:
                     rec = records[msg_id]
                     io_dict = {}
                     for key in 'pyin pyout pyerr stdout stderr'.split():
                             io_dict[key] = rec[key]
                     content[msg_id] = { 'result_content': rec['result_content'],
                                         'header': rec['header'],
                                         'result_header' : rec['result_header'],
                                         'io' : io_dict,
                                       }
                     if rec['result_buffers']:
                         buffers.extend(map(str, rec['result_buffers']))
             else:
                 try:
                     raise KeyError('No such message: '+msg_id)
                 except:
                     content = error.wrap_exception()
                 break
         self.session.send(self.query, "result_reply", content=content, 
                                             parent=msg, ident=client_id,
                                             buffers=buffers)
 

IPython/zmq/parallel/remotefunction.py

@@ -1,203 +1,200 b''
-"""Remote Functions and decorators for the client."""
+"""Remote Functions and decorators for Views."""
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2010  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import warnings
 
 from IPython.testing import decorators as testdec
 
 from . import map as Map
 from .asyncresult import AsyncMapResult
 
 #-----------------------------------------------------------------------------
 # Decorators
 #-----------------------------------------------------------------------------
 
 @testdec.skip_doctest
-def remote(client, bound=False, block=None, targets=None, balanced=None):
+def remote(view, block=None, **flags):
     """Turn a function into a remote function.
     
     This method can be used for map:
     
-    In [1]: @remote(client,block=True)
+    In [1]: @remote(view,block=True)
        ...: def func(a):
        ...:    pass
     """
     
     def remote_function(f):
-        return RemoteFunction(client, f, bound, block, targets, balanced)
+        return RemoteFunction(view, f, block=block, **flags)
     return remote_function
 
 @testdec.skip_doctest
-def parallel(client, dist='b', bound=False, block=None, targets='all', balanced=None):
+def parallel(view, dist='b', block=None, **flags):
     """Turn a function into a parallel remote function.
     
     This method can be used for map:
     
-    In [1]: @parallel(client,block=True)
+    In [1]: @parallel(view, block=True)
        ...: def func(a):
        ...:    pass
     """
     
     def parallel_function(f):
-        return ParallelFunction(client, f, dist, bound, block, targets, balanced)
+        return ParallelFunction(view, f, dist=dist, block=block, **flags)
     return parallel_function
 
 #--------------------------------------------------------------------------
 # Classes
 #--------------------------------------------------------------------------
 
 class RemoteFunction(object):
     """Turn an existing function into a remote function.
     
     Parameters
     ----------
     
-    client : Client instance
-        The client to be used to connect to engines
+    view : View instance
+        The view to be used for execution
     f : callable
         The function to be wrapped into a remote function
-    bound : bool [default: False]
-        Whether the affect the remote namespace when called
     block : bool [default: None]
         Whether to wait for results or not.  The default behavior is
-        to use the current `block` attribute of `client`
-    targets : valid target list [default: all]
-        The targets on which to execute.
-    balanced : bool
-        Whether to load-balance with the Task scheduler or not
+        to use the current `block` attribute of `view`
+    
+    **flags : remaining kwargs are passed to View.temp_flags
     """
     
-    client = None # the remote connection
+    view = None # the remote connection
     func = None # the wrapped function
     block = None # whether to block
-    bound = None # whether to affect the namespace
-    targets = None # where to execute
-    balanced = None # whether to load-balance
+    flags = None # dict of extra kwargs for temp_flags
     
-    def __init__(self, client, f, bound=False, block=None, targets=None, balanced=None):
-        self.client = client
+    def __init__(self, view, f, block=None, **flags):
+        self.view = view
         self.func = f
         self.block=block
-        self.bound=bound
-        self.targets=targets
-        if balanced is None:
-            if targets is None:
-                balanced = True
-            else:
-                balanced = False
-        self.balanced = balanced
+        self.flags=flags
     
     def __call__(self, *args, **kwargs):
-        return self.client.apply(self.func, args=args, kwargs=kwargs,
-                block=self.block, targets=self.targets, bound=self.bound, balanced=self.balanced)
+        block = self.view.block if self.block is None else self.block
+        with self.view.temp_flags(block=block, **self.flags):
+            return self.view.apply(self.func, *args, **kwargs)
     
 
 class ParallelFunction(RemoteFunction):
     """Class for mapping a function to sequences.
     
     This will distribute the sequences according the a mapper, and call
     the function on each sub-sequence.  If called via map, then the function
     will be called once on each element, rather that each sub-sequence.
     
     Parameters
     ----------
     
-    client : Client instance
-        The client to be used to connect to engines
+    view : View instance
+        The view to be used for execution
     f : callable
         The function to be wrapped into a remote function
-    bound : bool [default: False]
-        Whether the affect the remote namespace when called
+    dist : str [default: 'b']
+        The key for which mapObject to use to distribute sequences
+        options are:
+          * 'b' : use contiguous chunks in order
+          * 'r' : use round-robin striping
     block : bool [default: None]
         Whether to wait for results or not.  The default behavior is
-        to use the current `block` attribute of `client`
-    targets : valid target list [default: all]
-        The targets on which to execute.
-    balanced : bool
-        Whether to load-balance with the Task scheduler or not
-    chunk_size : int or None
+        to use the current `block` attribute of `view`
+    chunksize : int or None
         The size of chunk to use when breaking up sequences in a load-balanced manner
+    **flags : remaining kwargs are passed to View.temp_flags
     """
-    def __init__(self, client, f, dist='b', bound=False, block=None, targets='all', balanced=None, chunk_size=None):
-        super(ParallelFunction, self).__init__(client,f,bound,block,targets,balanced)
-        self.chunk_size = chunk_size
+    
+    chunksize=None
+    mapObject=None
+    
+    def __init__(self, view, f, dist='b', block=None, chunksize=None, **flags):
+        super(ParallelFunction, self).__init__(view, f, block=block, **flags)
+        self.chunksize = chunksize
         
         mapClass = Map.dists[dist]
         self.mapObject = mapClass()
     
     def __call__(self, *sequences):
+        # check that the length of sequences match
         len_0 = len(sequences[0])
         for s in sequences:
             if len(s)!=len_0:
                 msg = 'all sequences must have equal length, but %i!=%i'%(len_0,len(s))
                 raise ValueError(msg)
-        
-        if self.balanced:
-            if self.chunk_size:
-                nparts = len_0/self.chunk_size + int(len_0%self.chunk_size > 0)
+        balanced = 'Balanced' in self.view.__class__.__name__
+        if balanced:
+            if self.chunksize:
+                nparts = len_0/self.chunksize + int(len_0%self.chunksize > 0)
             else:
                 nparts = len_0
-            targets = [self.targets]*nparts
+            targets = [None]*nparts
         else:
-            if self.chunk_size:
-                warnings.warn("`chunk_size` is ignored when `balanced=False", UserWarning)
+            if self.chunksize:
+                warnings.warn("`chunksize` is ignored unless load balancing", UserWarning)
             # multiplexed:
-            targets = self.client._build_targets(self.targets)[-1]
+            targets = self.view.targets
             nparts = len(targets)
         
         msg_ids = []
         # my_f = lambda *a: map(self.func, *a)
+        client = self.view.client
         for index, t in enumerate(targets):
             args = []
             for seq in sequences:
                 part = self.mapObject.getPartition(seq, index, nparts)
                 if len(part) == 0:
                     continue
                 else:
                     args.append(part)
             if not args:
                 continue
             
             # print (args)
             if hasattr(self, '_map'):
                 f = map
                 args = [self.func]+args
             else:
                 f=self.func
-            ar = self.client.apply(f, args=args, block=False, bound=self.bound, 
-                        targets=t, balanced=self.balanced)
+            
+            view = self.view if balanced else client[t]
+            with view.temp_flags(block=False, **self.flags):
+                ar = view.apply(f, *args)
             
             msg_ids.append(ar.msg_ids[0])
         
-        r = AsyncMapResult(self.client, msg_ids, self.mapObject, fname=self.func.__name__)
+        r = AsyncMapResult(self.view.client, msg_ids, self.mapObject, fname=self.func.__name__)
+        
         if self.block:
             try:
                 return r.get()
             except KeyboardInterrupt:
                 return r
         else:
             return r
     
     def map(self, *sequences):
         """call a function on each element of a sequence remotely. 
         This should behave very much like the builtin map, but return an AsyncMapResult
         if self.block is False.
         """
         # set _map as a flag for use inside self.__call__
         self._map = True
         try:
             ret = self.__call__(*sequences)
         finally:
             del self._map
         return ret
 
 __all__ = ['remote', 'parallel', 'RemoteFunction', 'ParallelFunction']
  No newline at end of file

IPython/zmq/parallel/scheduler.py

@@ -1,590 +1,592 b''
 """The Python scheduler for rich scheduling.
 
 The Pure ZMQ scheduler does not allow routing schemes other than LRU,
 nor does it check msg_id DAG dependencies. For those, a slightly slower
 Python Scheduler exists.
 """
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2010-2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #----------------------------------------------------------------------
 # Imports
 #----------------------------------------------------------------------
 
 from __future__ import print_function
 
 import logging
 import sys
 
 from datetime import datetime, timedelta
 from random import randint, random
 from types import FunctionType
 
 try:
     import numpy
 except ImportError:
     numpy = None
 
 import zmq
 from zmq.eventloop import ioloop, zmqstream
 
 # local imports
 from IPython.external.decorator import decorator
 from IPython.utils.traitlets import Instance, Dict, List, Set
 
 from . import error
 from .dependency import Dependency
 from .entry_point import connect_logger, local_logger
 from .factory import SessionFactory
 
 
 @decorator
 def logged(f,self,*args,**kwargs):
     # print ("#--------------------")
     self.log.debug("scheduler::%s(*%s,**%s)"%(f.func_name, args, kwargs))
     # print ("#--")
     return f(self,*args, **kwargs)
 
 #----------------------------------------------------------------------
 # Chooser functions
 #----------------------------------------------------------------------
 
 def plainrandom(loads):
     """Plain random pick."""
     n = len(loads)
     return randint(0,n-1)
 
 def lru(loads):
     """Always pick the front of the line.
     
     The content of `loads` is ignored.
     
     Assumes LRU ordering of loads, with oldest first.
     """
     return 0
 
 def twobin(loads):
     """Pick two at random, use the LRU of the two.
     
     The content of loads is ignored.
     
     Assumes LRU ordering of loads, with oldest first.
     """
     n = len(loads)
     a = randint(0,n-1)
     b = randint(0,n-1)
     return min(a,b)
 
 def weighted(loads):
     """Pick two at random using inverse load as weight.
     
     Return the less loaded of the two.
     """
     # weight 0 a million times more than 1:
     weights = 1./(1e-6+numpy.array(loads))
     sums = weights.cumsum()
     t = sums[-1]
     x = random()*t
     y = random()*t
     idx = 0
     idy = 0
     while sums[idx] < x:
         idx += 1
     while sums[idy] < y:
         idy += 1
     if weights[idy] > weights[idx]:
         return idy
     else:
         return idx
 
 def leastload(loads):
     """Always choose the lowest load.
     
     If the lowest load occurs more than once, the first
     occurance will be used.  If loads has LRU ordering, this means
     the LRU of those with the lowest load is chosen.
     """
     return loads.index(min(loads))
 
 #---------------------------------------------------------------------
 # Classes
 #---------------------------------------------------------------------
 # store empty default dependency:
 MET = Dependency([])
 
 class TaskScheduler(SessionFactory):
     """Python TaskScheduler object.
     
     This is the simplest object that supports msg_id based
     DAG dependencies. *Only* task msg_ids are checked, not
     msg_ids of jobs submitted via the MUX queue.
     
     """
     
     # input arguments:
     scheme = Instance(FunctionType, default=leastload) # function for determining the destination
     client_stream = Instance(zmqstream.ZMQStream) # client-facing stream
     engine_stream = Instance(zmqstream.ZMQStream) # engine-facing stream
     notifier_stream = Instance(zmqstream.ZMQStream) # hub-facing sub stream
     mon_stream = Instance(zmqstream.ZMQStream) # hub-facing pub stream
     
     # internals:
     graph = Dict() # dict by msg_id of [ msg_ids that depend on key ]
     depending = Dict() # dict by msg_id of (msg_id, raw_msg, after, follow)
     pending = Dict() # dict by engine_uuid of submitted tasks
     completed = Dict() # dict by engine_uuid of completed tasks
     failed = Dict() # dict by engine_uuid of failed tasks
     destinations = Dict() # dict by msg_id of engine_uuids where jobs ran (reverse of completed+failed)
     clients = Dict() # dict by msg_id for who submitted the task
     targets = List() # list of target IDENTs
     loads = List() # list of engine loads
     all_completed = Set() # set of all completed tasks
     all_failed = Set() # set of all failed tasks
     all_done = Set() # set of all finished tasks=union(completed,failed)
     all_ids = Set() # set of all submitted task IDs
     blacklist = Dict() # dict by msg_id of locations where a job has encountered UnmetDependency
     auditor = Instance('zmq.eventloop.ioloop.PeriodicCallback')
     
     
     def start(self):
         self.engine_stream.on_recv(self.dispatch_result, copy=False)
         self._notification_handlers = dict(
             registration_notification = self._register_engine,
             unregistration_notification = self._unregister_engine
         )
         self.notifier_stream.on_recv(self.dispatch_notification)
         self.auditor = ioloop.PeriodicCallback(self.audit_timeouts, 2e3, self.loop) # 1 Hz
         self.auditor.start()
         self.log.info("Scheduler started...%r"%self)
     
     def resume_receiving(self):
         """Resume accepting jobs."""
         self.client_stream.on_recv(self.dispatch_submission, copy=False)
     
     def stop_receiving(self):
         """Stop accepting jobs while there are no engines.
         Leave them in the ZMQ queue."""
         self.client_stream.on_recv(None)
     
     #-----------------------------------------------------------------------
     # [Un]Registration Handling
     #-----------------------------------------------------------------------
     
     def dispatch_notification(self, msg):
         """dispatch register/unregister events."""
         idents,msg = self.session.feed_identities(msg)
         msg = self.session.unpack_message(msg)
         msg_type = msg['msg_type']
         handler = self._notification_handlers.get(msg_type, None)
         if handler is None:
             raise Exception("Unhandled message type: %s"%msg_type)
         else:
             try:
                 handler(str(msg['content']['queue']))
             except KeyError:
                 self.log.error("task::Invalid notification msg: %s"%msg)
     
     @logged
     def _register_engine(self, uid):
         """New engine with ident `uid` became available."""
         # head of the line:
         self.targets.insert(0,uid)
         self.loads.insert(0,0)
         # initialize sets
         self.completed[uid] = set()
         self.failed[uid] = set()
         self.pending[uid] = {}
         if len(self.targets) == 1:
             self.resume_receiving()
 
     def _unregister_engine(self, uid):
         """Existing engine with ident `uid` became unavailable."""
         if len(self.targets) == 1:
             # this was our only engine
             self.stop_receiving()
         
         # handle any potentially finished tasks:
         self.engine_stream.flush()
         
         self.completed.pop(uid)
         self.failed.pop(uid)
         # don't pop destinations, because it might be used later
         # map(self.destinations.pop, self.completed.pop(uid))
         # map(self.destinations.pop, self.failed.pop(uid))
         
         idx = self.targets.index(uid)
         self.targets.pop(idx)
         self.loads.pop(idx)
         
         # wait 5 seconds before cleaning up pending jobs, since the results might
         # still be incoming
         if self.pending[uid]:
             dc = ioloop.DelayedCallback(lambda : self.handle_stranded_tasks(uid), 5000, self.loop)
             dc.start()
     
     @logged
     def handle_stranded_tasks(self, engine):
         """Deal with jobs resident in an engine that died."""
         lost = self.pending.pop(engine)
         
         for msg_id, (raw_msg, targets, MET, follow, timeout) in lost.iteritems():
             self.all_failed.add(msg_id)
             self.all_done.add(msg_id)
             idents,msg = self.session.feed_identities(raw_msg, copy=False)
             msg = self.session.unpack_message(msg, copy=False, content=False)
             parent = msg['header']
             idents = [idents[0],engine]+idents[1:]
-            print (idents)
+            # print (idents)
             try:
                 raise error.EngineError("Engine %r died while running task %r"%(engine, msg_id))
             except:
                 content = error.wrap_exception()
             msg = self.session.send(self.client_stream, 'apply_reply', content, 
                                                     parent=parent, ident=idents)
             self.session.send(self.mon_stream, msg, ident=['outtask']+idents)
             self.update_graph(msg_id)
     
     
     #-----------------------------------------------------------------------
     # Job Submission
     #-----------------------------------------------------------------------
     @logged
     def dispatch_submission(self, raw_msg):
         """Dispatch job submission to appropriate handlers."""
         # ensure targets up to date:
         self.notifier_stream.flush()
         try:
             idents, msg = self.session.feed_identities(raw_msg, copy=False)
             msg = self.session.unpack_message(msg, content=False, copy=False)
         except:
             self.log.error("task::Invaid task: %s"%raw_msg, exc_info=True)
             return
         
         # send to monitor
         self.mon_stream.send_multipart(['intask']+raw_msg, copy=False)
         
         header = msg['header']
         msg_id = header['msg_id']
         self.all_ids.add(msg_id)
         
         # targets
         targets = set(header.get('targets', []))
         
         # time dependencies
         after = Dependency(header.get('after', []))
         if after.all:
+            if after.success:
             after.difference_update(self.all_completed)
-            if not after.success_only:
+            if after.failure:
                 after.difference_update(self.all_failed)
         if after.check(self.all_completed, self.all_failed):
             # recast as empty set, if `after` already met,
             # to prevent unnecessary set comparisons
             after = MET
         
         # location dependencies
         follow = Dependency(header.get('follow', []))
         
         # turn timeouts into datetime objects:
         timeout = header.get('timeout', None)
         if timeout:
             timeout = datetime.now() + timedelta(0,timeout,0)
         
         args = [raw_msg, targets, after, follow, timeout]
         
         # validate and reduce dependencies:
         for dep in after,follow:
             # check valid:
             if msg_id in dep or dep.difference(self.all_ids):
                 self.depending[msg_id] = args
                 return self.fail_unreachable(msg_id, error.InvalidDependency)
             # check if unreachable:
-            if dep.unreachable(self.all_failed):
+            if dep.unreachable(self.all_completed, self.all_failed):
                 self.depending[msg_id] = args
                 return self.fail_unreachable(msg_id)
         
         if after.check(self.all_completed, self.all_failed):
             # time deps already met, try to run
             if not self.maybe_run(msg_id, *args):
                 # can't run yet
                 self.save_unmet(msg_id, *args)
         else:
             self.save_unmet(msg_id, *args)
     
     # @logged
     def audit_timeouts(self):
         """Audit all waiting tasks for expired timeouts."""
         now = datetime.now()
         for msg_id in self.depending.keys():
             # must recheck, in case one failure cascaded to another:
             if msg_id in self.depending:
                 raw,after,targets,follow,timeout = self.depending[msg_id]
                 if timeout and timeout < now:
                     self.fail_unreachable(msg_id, timeout=True)
                 
     @logged
     def fail_unreachable(self, msg_id, why=error.ImpossibleDependency):
         """a task has become unreachable, send a reply with an ImpossibleDependency
         error."""
         if msg_id not in self.depending:
             self.log.error("msg %r already failed!"%msg_id)
             return
         raw_msg,targets,after,follow,timeout = self.depending.pop(msg_id)
         for mid in follow.union(after):
             if mid in self.graph:
                 self.graph[mid].remove(msg_id)
         
         # FIXME: unpacking a message I've already unpacked, but didn't save:
         idents,msg = self.session.feed_identities(raw_msg, copy=False)
         msg = self.session.unpack_message(msg, copy=False, content=False)
         header = msg['header']
         
         try:
             raise why()
         except:
             content = error.wrap_exception()
         
         self.all_done.add(msg_id)
         self.all_failed.add(msg_id)
         
         msg = self.session.send(self.client_stream, 'apply_reply', content, 
                                                 parent=header, ident=idents)
         self.session.send(self.mon_stream, msg, ident=['outtask']+idents)
         
         self.update_graph(msg_id, success=False)
     
     @logged
     def maybe_run(self, msg_id, raw_msg, targets, after, follow, timeout):
         """check location dependencies, and run if they are met."""
         blacklist = self.blacklist.setdefault(msg_id, set())
         if follow or targets or blacklist:
             # we need a can_run filter
             def can_run(idx):
                 target = self.targets[idx]
                 # check targets
                 if targets and target not in targets:
                     return False
                 # check blacklist
                 if target in blacklist:
                     return False
                 # check follow
                 return follow.check(self.completed[target], self.failed[target])
             
             indices = filter(can_run, range(len(self.targets)))
             if not indices:
                 # couldn't run
                 if follow.all:
                     # check follow for impossibility
                     dests = set()
-                    relevant = self.all_completed if follow.success_only else self.all_done
+                    relevant = set()
+                    if follow.success:
+                        relevant = self.all_completed
+                    if follow.failure:
+                        relevant = relevant.union(self.all_failed)
                     for m in follow.intersection(relevant):
                         dests.add(self.destinations[m])
                     if len(dests) > 1:
                         self.fail_unreachable(msg_id)
                         return False
                 if targets:
                     # check blacklist+targets for impossibility
                     targets.difference_update(blacklist)
                     if not targets or not targets.intersection(self.targets):
                         self.fail_unreachable(msg_id)
                         return False
                 return False
         else:
             indices = None
             
         self.submit_task(msg_id, raw_msg, targets, follow, timeout, indices)
         return True
             
     @logged
     def save_unmet(self, msg_id, raw_msg, targets, after, follow, timeout):
         """Save a message for later submission when its dependencies are met."""
         self.depending[msg_id] = [raw_msg,targets,after,follow,timeout]
         # track the ids in follow or after, but not those already finished
         for dep_id in after.union(follow).difference(self.all_done):
             if dep_id not in self.graph:
                 self.graph[dep_id] = set()
             self.graph[dep_id].add(msg_id)
     
     @logged
     def submit_task(self, msg_id, raw_msg, targets, follow, timeout, indices=None):
         """Submit a task to any of a subset of our targets."""
         if indices:
             loads = [self.loads[i] for i in indices]
         else:
             loads = self.loads
         idx = self.scheme(loads)
         if indices:
             idx = indices[idx]
         target = self.targets[idx]
         # print (target, map(str, msg[:3]))
         self.engine_stream.send(target, flags=zmq.SNDMORE, copy=False)
         self.engine_stream.send_multipart(raw_msg, copy=False)
         self.add_job(idx)
         self.pending[target][msg_id] = (raw_msg, targets, MET, follow, timeout)
         content = dict(msg_id=msg_id, engine_id=target)
         self.session.send(self.mon_stream, 'task_destination', content=content, 
                         ident=['tracktask',self.session.session])
     
     #-----------------------------------------------------------------------
     # Result Handling
     #-----------------------------------------------------------------------
     @logged
     def dispatch_result(self, raw_msg):
         """dispatch method for result replies"""
         try:
             idents,msg = self.session.feed_identities(raw_msg, copy=False)
             msg = self.session.unpack_message(msg, content=False, copy=False)
         except:
             self.log.error("task::Invaid result: %s"%raw_msg, exc_info=True)
             return
         
         header = msg['header']
         if header.get('dependencies_met', True):
             success = (header['status'] == 'ok')
             self.handle_result(idents, msg['parent_header'], raw_msg, success)
             # send to Hub monitor
             self.mon_stream.send_multipart(['outtask']+raw_msg, copy=False)
         else:
             self.handle_unmet_dependency(idents, msg['parent_header'])
         
     @logged
     def handle_result(self, idents, parent, raw_msg, success=True):
         """handle a real task result, either success or failure"""
         # first, relay result to client
         engine = idents[0]
         client = idents[1]
         # swap_ids for XREP-XREP mirror
         raw_msg[:2] = [client,engine]
         # print (map(str, raw_msg[:4]))
         self.client_stream.send_multipart(raw_msg, copy=False)
         # now, update our data structures
         msg_id = parent['msg_id']
         self.blacklist.pop(msg_id, None)
         self.pending[engine].pop(msg_id)
         if success:
             self.completed[engine].add(msg_id)
             self.all_completed.add(msg_id)
         else:
             self.failed[engine].add(msg_id)
             self.all_failed.add(msg_id)
         self.all_done.add(msg_id)
         self.destinations[msg_id] = engine
         
         self.update_graph(msg_id, success)
         
     @logged
     def handle_unmet_dependency(self, idents, parent):
         """handle an unmet dependency"""
         engine = idents[0]
         msg_id = parent['msg_id']
         
         if msg_id not in self.blacklist:
             self.blacklist[msg_id] = set()
         self.blacklist[msg_id].add(engine)
         
         args = self.pending[engine].pop(msg_id)
         raw,targets,after,follow,timeout = args
         
         if self.blacklist[msg_id] == targets:
             self.depending[msg_id] = args
             return self.fail_unreachable(msg_id)
         
         elif not self.maybe_run(msg_id, *args):
             # resubmit failed, put it back in our dependency tree
             self.save_unmet(msg_id, *args)
         
     
     @logged
     def update_graph(self, dep_id, success=True):
         """dep_id just finished. Update our dependency
         graph and submit any jobs that just became runable."""
         # print ("\n\n***********")
         # pprint (dep_id)
         # pprint (self.graph)
         # pprint (self.depending)
         # pprint (self.all_completed)
         # pprint (self.all_failed)
         # print ("\n\n***********\n\n")
         if dep_id not in self.graph:
             return
         jobs = self.graph.pop(dep_id)
         
         for msg_id in jobs:
             raw_msg, targets, after, follow, timeout = self.depending[msg_id]
-            # if dep_id in after:
-            #     if after.all and (success or not after.success_only):
-            #         after.remove(dep_id)
             
-            if after.unreachable(self.all_failed) or follow.unreachable(self.all_failed):
+            if after.unreachable(self.all_completed, self.all_failed) or follow.unreachable(self.all_completed, self.all_failed):
                 self.fail_unreachable(msg_id)
             
             elif after.check(self.all_completed, self.all_failed): # time deps met, maybe run
                 if self.maybe_run(msg_id, raw_msg, targets, MET, follow, timeout):
                     
                     self.depending.pop(msg_id)
                     for mid in follow.union(after):
                         if mid in self.graph:
                             self.graph[mid].remove(msg_id)
     
     #----------------------------------------------------------------------
     # methods to be overridden by subclasses
     #----------------------------------------------------------------------
     
     def add_job(self, idx):
         """Called after self.targets[idx] just got the job with header.
         Override with subclasses.  The default ordering is simple LRU.
         The default loads are the number of outstanding jobs."""
         self.loads[idx] += 1
         for lis in (self.targets, self.loads):
             lis.append(lis.pop(idx))
             
     
     def finish_job(self, idx):
         """Called after self.targets[idx] just finished a job.
         Override with subclasses."""
         self.loads[idx] -= 1
     
 
 
 def launch_scheduler(in_addr, out_addr, mon_addr, not_addr, config=None,logname='ZMQ', 
                             log_addr=None, loglevel=logging.DEBUG, scheme='lru',
                             identity=b'task'):
     from zmq.eventloop import ioloop
     from zmq.eventloop.zmqstream import ZMQStream
     
     ctx = zmq.Context()
     loop = ioloop.IOLoop()
     print (in_addr, out_addr, mon_addr, not_addr)
     ins = ZMQStream(ctx.socket(zmq.XREP),loop)
     ins.setsockopt(zmq.IDENTITY, identity)
     ins.bind(in_addr)
     
     outs = ZMQStream(ctx.socket(zmq.XREP),loop)
     outs.setsockopt(zmq.IDENTITY, identity)
     outs.bind(out_addr)
     mons = ZMQStream(ctx.socket(zmq.PUB),loop)
     mons.connect(mon_addr)
     nots = ZMQStream(ctx.socket(zmq.SUB),loop)
     nots.setsockopt(zmq.SUBSCRIBE, '')
     nots.connect(not_addr)
     
     scheme = globals().get(scheme, None)
     # setup logging
     if log_addr:
         connect_logger(logname, ctx, log_addr, root="scheduler", loglevel=loglevel)
     else:
         local_logger(logname, loglevel)
     
     scheduler = TaskScheduler(client_stream=ins, engine_stream=outs,
                             mon_stream=mons, notifier_stream=nots,
                             scheme=scheme, loop=loop, logname=logname,
                             config=config)
     scheduler.start()
     try:
         loop.start()
     except KeyboardInterrupt:
         print ("interrupted, exiting...", file=sys.__stderr__)
 

IPython/zmq/parallel/streamkernel.py

@@ -1,493 +1,489 b''
 #!/usr/bin/env python
 """
 Kernel adapted from kernel.py to use ZMQ Streams
 """
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2010-2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 # Standard library imports.
 from __future__ import print_function
 
 import sys
 import time
 
 from code import CommandCompiler
 from datetime import datetime
 from pprint import pprint
 from signal import SIGTERM, SIGKILL
 
 # System library imports.
 import zmq
 from zmq.eventloop import ioloop, zmqstream
 
 # Local imports.
 from IPython.core import ultratb
 from IPython.utils.traitlets import Instance, List, Int, Dict, Set, Str
 from IPython.zmq.completer import KernelCompleter
 from IPython.zmq.iostream import OutStream
 from IPython.zmq.displayhook import DisplayHook
 
 from . import heartmonitor
 from .client import Client
 from .error import wrap_exception
 from .factory import SessionFactory
 from .streamsession import StreamSession
 from .util import serialize_object, unpack_apply_message, ISO8601, Namespace
 
 def printer(*args):
     pprint(args, stream=sys.__stdout__)
 
 
 class _Passer:
     """Empty class that implements `send()` that does nothing."""
     def send(self, *args, **kwargs):
         pass
     send_multipart = send
     
 
 #-----------------------------------------------------------------------------
 # Main kernel class
 #-----------------------------------------------------------------------------
 
 class Kernel(SessionFactory):
 
     #---------------------------------------------------------------------------
     # Kernel interface
     #---------------------------------------------------------------------------
     
     # kwargs:
     int_id = Int(-1, config=True)
     user_ns = Dict(config=True)
     exec_lines = List(config=True)
     
     control_stream = Instance(zmqstream.ZMQStream)
     task_stream = Instance(zmqstream.ZMQStream)
     iopub_stream = Instance(zmqstream.ZMQStream)
     client = Instance('IPython.zmq.parallel.client.Client')
     
     # internals
     shell_streams = List()
     compiler = Instance(CommandCompiler, (), {})
     completer = Instance(KernelCompleter)
     
     aborted = Set()
     shell_handlers = Dict()
     control_handlers = Dict()
     
     def _set_prefix(self):
         self.prefix = "engine.%s"%self.int_id
     
     def _connect_completer(self):
         self.completer = KernelCompleter(self.user_ns)
     
     def __init__(self, **kwargs):
         super(Kernel, self).__init__(**kwargs)
         self._set_prefix()
         self._connect_completer()
         
         self.on_trait_change(self._set_prefix, 'id')
         self.on_trait_change(self._connect_completer, 'user_ns')
         
         # Build dict of handlers for message types
         for msg_type in ['execute_request', 'complete_request', 'apply_request', 
                 'clear_request']:
             self.shell_handlers[msg_type] = getattr(self, msg_type)
         
         for msg_type in ['shutdown_request', 'abort_request']+self.shell_handlers.keys():
             self.control_handlers[msg_type] = getattr(self, msg_type)
         
         self._initial_exec_lines()
     
     def _wrap_exception(self, method=None):
         e_info = dict(engine_uuid=self.ident, engine_id=self.int_id, method=method)
         content=wrap_exception(e_info)
         return content
     
     def _initial_exec_lines(self):
         s = _Passer()
         content = dict(silent=True, user_variable=[],user_expressions=[])
         for line in self.exec_lines:
             self.log.debug("executing initialization: %s"%line)
             content.update({'code':line})
             msg = self.session.msg('execute_request', content)
             self.execute_request(s, [], msg)
         
         
     #-------------------- control handlers -----------------------------
     def abort_queues(self):
         for stream in self.shell_streams:
             if stream:
                 self.abort_queue(stream)
     
     def abort_queue(self, stream):
         while True:
             try:
                 msg = self.session.recv(stream, zmq.NOBLOCK,content=True)
             except zmq.ZMQError as e:
                 if e.errno == zmq.EAGAIN:
                     break
                 else:
                     return
             else:
                 if msg is None:
                     return
                 else:
                     idents,msg = msg
                 
                 # assert self.reply_socketly_socket.rcvmore(), "Unexpected missing message part."
                 # msg = self.reply_socket.recv_json()
             self.log.info("Aborting:")
             self.log.info(str(msg))
             msg_type = msg['msg_type']
             reply_type = msg_type.split('_')[0] + '_reply'
             # reply_msg = self.session.msg(reply_type, {'status' : 'aborted'}, msg)
             # self.reply_socket.send(ident,zmq.SNDMORE)
             # self.reply_socket.send_json(reply_msg)
             reply_msg = self.session.send(stream, reply_type, 
                         content={'status' : 'aborted'}, parent=msg, ident=idents)[0]
             self.log.debug(str(reply_msg))
             # We need to wait a bit for requests to come in. This can probably
             # be set shorter for true asynchronous clients.
             time.sleep(0.05)
     
     def abort_request(self, stream, ident, parent):
         """abort a specifig msg by id"""
         msg_ids = parent['content'].get('msg_ids', None)
         if isinstance(msg_ids, basestring):
             msg_ids = [msg_ids]
         if not msg_ids:
             self.abort_queues()
         for mid in msg_ids:
             self.aborted.add(str(mid))
         
         content = dict(status='ok')
         reply_msg = self.session.send(stream, 'abort_reply', content=content, 
-                parent=parent, ident=ident)[0]
+                parent=parent, ident=ident)
         self.log.debug(str(reply_msg))
     
     def shutdown_request(self, stream, ident, parent):
         """kill ourself.  This should really be handled in an external process"""
         try:
             self.abort_queues()
         except:
             content = self._wrap_exception('shutdown')
         else:
             content = dict(parent['content'])
             content['status'] = 'ok'
         msg = self.session.send(stream, 'shutdown_reply',
                                 content=content, parent=parent, ident=ident)
-        # msg = self.session.send(self.pub_socket, 'shutdown_reply',
-        #                         content, parent, ident)
-        # print >> sys.__stdout__, msg
-        # time.sleep(0.2)
+        self.log.debug(str(msg))
         dc = ioloop.DelayedCallback(lambda : sys.exit(0), 1000, self.loop)
         dc.start()
     
     def dispatch_control(self, msg):
         idents,msg = self.session.feed_identities(msg, copy=False)
         try:
             msg = self.session.unpack_message(msg, content=True, copy=False)
         except:
             self.log.error("Invalid Message", exc_info=True)
             return
         
         header = msg['header']
         msg_id = header['msg_id']
         
         handler = self.control_handlers.get(msg['msg_type'], None)
         if handler is None:
             self.log.error("UNKNOWN CONTROL MESSAGE TYPE: %r"%msg['msg_type'])
         else:
             handler(self.control_stream, idents, msg)
     
 
     #-------------------- queue helpers ------------------------------
     
     def check_dependencies(self, dependencies):
         if not dependencies:
             return True
         if len(dependencies) == 2 and dependencies[0] in 'any all'.split():
             anyorall = dependencies[0]
             dependencies = dependencies[1]
         else:
             anyorall = 'all'
         results = self.client.get_results(dependencies,status_only=True)
         if results['status'] != 'ok':
             return False
         
         if anyorall == 'any':
             if not results['completed']:
                 return False
         else:
             if results['pending']:
                 return False
         
         return True
     
     def check_aborted(self, msg_id):
         return msg_id in self.aborted
     
     #-------------------- queue handlers -----------------------------
     
     def clear_request(self, stream, idents, parent):
         """Clear our namespace."""
         self.user_ns = {}
         msg = self.session.send(stream, 'clear_reply', ident=idents, parent=parent, 
                 content = dict(status='ok'))
         self._initial_exec_lines()
     
     def execute_request(self, stream, ident, parent):
         self.log.debug('execute request %s'%parent)
         try:
             code = parent[u'content'][u'code']
         except:
             self.log.error("Got bad msg: %s"%parent, exc_info=True)
             return
         self.session.send(self.iopub_stream, u'pyin', {u'code':code},parent=parent,
                             ident='%s.pyin'%self.prefix)
         started = datetime.now().strftime(ISO8601)
         try:
             comp_code = self.compiler(code, '<zmq-kernel>')
             # allow for not overriding displayhook
             if hasattr(sys.displayhook, 'set_parent'):
                 sys.displayhook.set_parent(parent)
                 sys.stdout.set_parent(parent)
                 sys.stderr.set_parent(parent)
             exec comp_code in self.user_ns, self.user_ns
         except:
             exc_content = self._wrap_exception('execute')
             # exc_msg = self.session.msg(u'pyerr', exc_content, parent)
             self.session.send(self.iopub_stream, u'pyerr', exc_content, parent=parent,
                             ident='%s.pyerr'%self.prefix)
             reply_content = exc_content
         else:
             reply_content = {'status' : 'ok'}
         
         reply_msg = self.session.send(stream, u'execute_reply', reply_content, parent=parent, 
                     ident=ident, subheader = dict(started=started))
         self.log.debug(str(reply_msg))
         if reply_msg['content']['status'] == u'error':
             self.abort_queues()
 
     def complete_request(self, stream, ident, parent):
         matches = {'matches' : self.complete(parent),
                    'status' : 'ok'}
         completion_msg = self.session.send(stream, 'complete_reply',
                                            matches, parent, ident)
         # print >> sys.__stdout__, completion_msg
 
     def complete(self, msg):
         return self.completer.complete(msg.content.line, msg.content.text)
     
     def apply_request(self, stream, ident, parent):
         # flush previous reply, so this request won't block it
         stream.flush(zmq.POLLOUT)
         
         try:
             content = parent[u'content']
             bufs = parent[u'buffers']
             msg_id = parent['header']['msg_id']
-            bound = content.get('bound', False)
+            # bound = parent['header'].get('bound', False)
         except:
             self.log.error("Got bad msg: %s"%parent, exc_info=True)
             return
         # pyin_msg = self.session.msg(u'pyin',{u'code':code}, parent=parent)
         # self.iopub_stream.send(pyin_msg)
         # self.session.send(self.iopub_stream, u'pyin', {u'code':code},parent=parent)
         sub = {'dependencies_met' : True, 'engine' : self.ident,
                 'started': datetime.now().strftime(ISO8601)}
         try:
             # allow for not overriding displayhook
             if hasattr(sys.displayhook, 'set_parent'):
                 sys.displayhook.set_parent(parent)
                 sys.stdout.set_parent(parent)
                 sys.stderr.set_parent(parent)
             # exec "f(*args,**kwargs)" in self.user_ns, self.user_ns
             working = self.user_ns
             # suffix = 
             prefix = "_"+str(msg_id).replace("-","")+"_"
-            # if bound:
-            #     
-            # else:
-            #     working = dict()
-            #     suffix = prefix = "_" # prevent keyword collisions with lambda
+            
             f,args,kwargs = unpack_apply_message(bufs, working, copy=False)
-            if bound:
-                bound_ns = Namespace(working)
-                args = [bound_ns]+list(args)
-            # if f.fun
+            # if bound:
+            #     bound_ns = Namespace(working)
+            #     args = [bound_ns]+list(args)
+
             fname = getattr(f, '__name__', 'f')
             
             fname = prefix+"f"
             argname = prefix+"args"
             kwargname = prefix+"kwargs"
             resultname = prefix+"result"
             
             ns = { fname : f, argname : args, kwargname : kwargs , resultname : None }
             # print ns
             working.update(ns)
             code = "%s=%s(*%s,**%s)"%(resultname, fname, argname, kwargname)
             try:
                 exec code in working,working
                 result = working.get(resultname)
             finally:
                 for key in ns.iterkeys():
                     working.pop(key)
-            if bound:
-                working.update(bound_ns)
+            # if bound:
+            #     working.update(bound_ns)
             
             packed_result,buf = serialize_object(result)
             result_buf = [packed_result]+buf
         except:
             exc_content = self._wrap_exception('apply')
             # exc_msg = self.session.msg(u'pyerr', exc_content, parent)
             self.session.send(self.iopub_stream, u'pyerr', exc_content, parent=parent,
                                 ident='%s.pyerr'%self.prefix)
             reply_content = exc_content
             result_buf = []
             
             if exc_content['ename'] == 'UnmetDependency':
                 sub['dependencies_met'] = False
         else:
             reply_content = {'status' : 'ok'}
         
         # put 'ok'/'error' status in header, for scheduler introspection:
         sub['status'] = reply_content['status']
         
         reply_msg = self.session.send(stream, u'apply_reply', reply_content, 
                     parent=parent, ident=ident,buffers=result_buf, subheader=sub)
 
-        # if reply_msg['content']['status'] == u'error':
-        #     self.abort_queues()
+        # flush i/o
+        # should this be before reply_msg is sent, like in the single-kernel code, 
+        # or should nothing get in the way of real results?
+        sys.stdout.flush()
+        sys.stderr.flush()
     
     def dispatch_queue(self, stream, msg):
         self.control_stream.flush()
         idents,msg = self.session.feed_identities(msg, copy=False)
         try:
             msg = self.session.unpack_message(msg, content=True, copy=False)
         except:
             self.log.error("Invalid Message", exc_info=True)
             return
             
         
         header = msg['header']
         msg_id = header['msg_id']
         if self.check_aborted(msg_id):
             self.aborted.remove(msg_id)
             # is it safe to assume a msg_id will not be resubmitted?
             reply_type = msg['msg_type'].split('_')[0] + '_reply'
             reply_msg = self.session.send(stream, reply_type, 
                         content={'status' : 'aborted'}, parent=msg, ident=idents)
             return
         handler = self.shell_handlers.get(msg['msg_type'], None)
         if handler is None:
             self.log.error("UNKNOWN MESSAGE TYPE: %r"%msg['msg_type'])
         else:
             handler(stream, idents, msg)
     
     def start(self):
         #### stream mode:
         if self.control_stream:
             self.control_stream.on_recv(self.dispatch_control, copy=False)
             self.control_stream.on_err(printer)
         
         def make_dispatcher(stream):
             def dispatcher(msg):
                 return self.dispatch_queue(stream, msg)
             return dispatcher
         
         for s in self.shell_streams:
             s.on_recv(make_dispatcher(s), copy=False)
             s.on_err(printer)
         
         if self.iopub_stream:
             self.iopub_stream.on_err(printer)
         
         #### while True mode:
         # while True:
         #     idle = True
         #     try:
         #         msg = self.shell_stream.socket.recv_multipart(
         #                     zmq.NOBLOCK, copy=False)
         #     except zmq.ZMQError, e:
         #         if e.errno != zmq.EAGAIN:
         #             raise e
         #     else:
         #         idle=False
         #         self.dispatch_queue(self.shell_stream, msg)
         #             
         #     if not self.task_stream.empty():
         #         idle=False
         #         msg = self.task_stream.recv_multipart()
         #         self.dispatch_queue(self.task_stream, msg)
         #     if idle:
         #         # don't busywait
         #         time.sleep(1e-3)
 
 def make_kernel(int_id, identity, control_addr, shell_addrs, iopub_addr, hb_addrs, 
                 client_addr=None, loop=None, context=None, key=None,
                 out_stream_factory=OutStream, display_hook_factory=DisplayHook):
     """NO LONGER IN USE"""
     # create loop, context, and session:
     if loop is None:
         loop = ioloop.IOLoop.instance()
     if context is None:
         context = zmq.Context()
     c = context
     session = StreamSession(key=key)
     # print (session.key)
     # print (control_addr, shell_addrs, iopub_addr, hb_addrs)
     
     # create Control Stream
     control_stream = zmqstream.ZMQStream(c.socket(zmq.PAIR), loop)
     control_stream.setsockopt(zmq.IDENTITY, identity)
     control_stream.connect(control_addr)
     
     # create Shell Streams (MUX, Task, etc.):
     shell_streams = []
     for addr in shell_addrs:
         stream = zmqstream.ZMQStream(c.socket(zmq.PAIR), loop)
         stream.setsockopt(zmq.IDENTITY, identity)
         stream.connect(addr)
         shell_streams.append(stream)
     
     # create iopub stream:
     iopub_stream = zmqstream.ZMQStream(c.socket(zmq.PUB), loop)
     iopub_stream.setsockopt(zmq.IDENTITY, identity)
     iopub_stream.connect(iopub_addr)
     
     # Redirect input streams and set a display hook.
     if out_stream_factory:
         sys.stdout = out_stream_factory(session, iopub_stream, u'stdout')
         sys.stdout.topic = 'engine.%i.stdout'%int_id
         sys.stderr = out_stream_factory(session, iopub_stream, u'stderr')
         sys.stderr.topic = 'engine.%i.stderr'%int_id
     if display_hook_factory:
         sys.displayhook = display_hook_factory(session, iopub_stream)
         sys.displayhook.topic = 'engine.%i.pyout'%int_id
     
     
     # launch heartbeat
     heart = heartmonitor.Heart(*map(str, hb_addrs), heart_id=identity)
     heart.start()
     
     # create (optional) Client
     if client_addr:
         client = Client(client_addr, username=identity)
     else:
         client = None
     
     kernel = Kernel(id=int_id, session=session, control_stream=control_stream,
             shell_streams=shell_streams, iopub_stream=iopub_stream, 
             client=client, loop=loop)
     kernel.start()
     return loop, c, kernel
 

IPython/zmq/parallel/tests/__init__.py

@@ -1,48 +1,69 b''
 """toplevel setup/teardown for parallel tests."""
 
+#-------------------------------------------------------------------------------
+#  Copyright (C) 2011  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-------------------------------------------------------------------------------
+
+#-------------------------------------------------------------------------------
+# Imports
+#-------------------------------------------------------------------------------
+
 import tempfile
 import time
 from subprocess import Popen, PIPE, STDOUT
 
 from IPython.zmq.parallel import client
 
 processes = []
 blackhole = tempfile.TemporaryFile()
 
 # nose setup/teardown
 
 def setup():
     cp = Popen('ipcontrollerz --profile iptest -r --log-level 10 --log-to-file'.split(), stdout=blackhole, stderr=STDOUT)
     processes.append(cp)
     time.sleep(.5)
-    add_engine()
+    add_engines(1)
     c = client.Client(profile='iptest')
     while not c.ids:
         time.sleep(.1)
         c.spin()
+    c.close()
 
-def add_engine(profile='iptest'):
+def add_engines(n=1, profile='iptest'):
+    rc = client.Client(profile=profile)
+    base = len(rc)
+    eps = []
+    for i in range(n):
     ep = Popen(['ipenginez']+ ['--profile', profile, '--log-level', '10', '--log-to-file'], stdout=blackhole, stderr=STDOUT)
     # ep.start()
     processes.append(ep)
-    return ep
+        eps.append(ep)
+    while len(rc) < base+n:
+        time.sleep(.1)
+        rc.spin()
+    rc.close()
+    return eps
 
 def teardown():
     time.sleep(1)
     while processes:
         p = processes.pop()
         if p.poll() is None:
             try:
                 p.terminate()
             except Exception, e:
                 print e
                 pass
         if p.poll() is None:
             time.sleep(.25)
         if p.poll() is None:
             try:
                 print 'killing'
                 p.kill()
             except:
                 print "couldn't shutdown process: ", p
     

IPython/zmq/parallel/tests/clienttest.py

@@ -1,106 +1,119 b''
+"""base class for parallel client tests"""
+
+#-------------------------------------------------------------------------------
+#  Copyright (C) 2011  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-------------------------------------------------------------------------------
+
 import sys
 import tempfile
 import time
 from signal import SIGINT
 from multiprocessing import Process
 
 from nose import SkipTest
 
+import zmq
 from zmq.tests import BaseZMQTestCase
 
 from IPython.external.decorator import decorator
 
 from IPython.zmq.parallel import error
 from IPython.zmq.parallel.client import Client
 from IPython.zmq.parallel.ipcluster import launch_process
 from IPython.zmq.parallel.entry_point import select_random_ports
-from IPython.zmq.parallel.tests import processes,add_engine
+from IPython.zmq.parallel.tests import processes,add_engines
 
 # simple tasks for use in apply tests
 
 def segfault():
     """this will segfault"""
     import ctypes
     ctypes.memset(-1,0,1)
 
 def wait(n):
     """sleep for a time"""
     import time
     time.sleep(n)
     return n
 
 def raiser(eclass):
     """raise an exception"""
     raise eclass()
 
 # test decorator for skipping tests when libraries are unavailable
 def skip_without(*names):
     """skip a test if some names are not importable"""
     @decorator
     def skip_without_names(f, *args, **kwargs):
         """decorator to skip tests in the absence of numpy."""
         for name in names:
             try:
                 __import__(name)
             except ImportError:
                 raise SkipTest
         return f(*args, **kwargs)
     return skip_without_names
 
-
 class ClusterTestCase(BaseZMQTestCase):
     
     def add_engines(self, n=1, block=True):
         """add multiple engines to our cluster"""
-        for i in range(n):
-            self.engines.append(add_engine())
+        self.engines.extend(add_engines(n))
         if block:
             self.wait_on_engines()
     
     def wait_on_engines(self, timeout=5):
         """wait for our engines to connect."""
         n = len(self.engines)+self.base_engine_count
         tic = time.time()
         while time.time()-tic < timeout and len(self.client.ids) < n:
             time.sleep(0.1)
         
         assert not len(self.client.ids) < n, "waiting for engines timed out"
     
     def connect_client(self):
         """connect a client with my Context, and track its sockets for cleanup"""
         c = Client(profile='iptest',context=self.context)
-        
-        # for name in filter(lambda n:n.endswith('socket'), dir(c)):
-        #     self.sockets.append(getattr(c, name))
+        for name in filter(lambda n:n.endswith('socket'), dir(c)):
+            s = getattr(c, name)
+            s.setsockopt(zmq.LINGER, 0)
+            self.sockets.append(s)
         return c
     
     def assertRaisesRemote(self, etype, f, *args, **kwargs):
         try:
             try:
                 f(*args, **kwargs)
             except error.CompositeError as e:
                 e.raise_exception()
         except error.RemoteError as e:
             self.assertEquals(etype.__name__, e.ename, "Should have raised %r, but raised %r"%(e.ename, etype.__name__))
         else:
             self.fail("should have raised a RemoteError")
             
     def setUp(self):
         BaseZMQTestCase.setUp(self)
         self.client = self.connect_client()
         self.base_engine_count=len(self.client.ids)
         self.engines=[]
     
     def tearDown(self):
-        
+        # self.client.clear(block=True)
         # close fds:
         for e in filter(lambda e: e.poll() is not None, processes):
             processes.remove(e)
         
+        # allow flushing of incoming messages to prevent crash on socket close
+        self.client.wait(timeout=2)
+        # time.sleep(2)
+        self.client.spin()
         self.client.close()
         BaseZMQTestCase.tearDown(self)
-        # this will be superfluous when pyzmq merges PR #88
-        self.context.term()
+        # this will be redundant when pyzmq merges PR #88
+        # self.context.term()
         # print tempfile.TemporaryFile().fileno(),
         # sys.stdout.flush()
         
  No newline at end of file

IPython/zmq/parallel/tests/test_client.py

@@ -1,262 +1,147 b''
+"""Tests for parallel client.py"""
+
+#-------------------------------------------------------------------------------
+#  Copyright (C) 2011  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-------------------------------------------------------------------------------
+
+#-------------------------------------------------------------------------------
+# Imports
+#-------------------------------------------------------------------------------
+
 import time
 from tempfile import mktemp
 
 import zmq
 
 from IPython.zmq.parallel import client as clientmod
 from IPython.zmq.parallel import error
 from IPython.zmq.parallel.asyncresult import AsyncResult, AsyncHubResult
 from IPython.zmq.parallel.view import LoadBalancedView, DirectView
 
-from clienttest import ClusterTestCase, segfault, wait
+from clienttest import ClusterTestCase, segfault, wait, add_engines
+
+def setup():
+    add_engines(4)
 
 class TestClient(ClusterTestCase):
     
     def test_ids(self):
         n = len(self.client.ids)
         self.add_engines(3)
         self.assertEquals(len(self.client.ids), n+3)
     
-    def test_segfault_task(self):
-        """test graceful handling of engine death (balanced)"""
-        self.add_engines(1)
-        ar = self.client.apply(segfault, block=False)
-        self.assertRaisesRemote(error.EngineError, ar.get)
-        eid = ar.engine_id
-        while eid in self.client.ids:
-            time.sleep(.01)
-            self.client.spin()
-    
-    def test_segfault_mux(self):
-        """test graceful handling of engine death (direct)"""
-        self.add_engines(1)
-        eid = self.client.ids[-1]
-        ar = self.client[eid].apply_async(segfault)
-        self.assertRaisesRemote(error.EngineError, ar.get)
-        eid = ar.engine_id
-        while eid in self.client.ids:
-            time.sleep(.01)
-            self.client.spin()
-    
     def test_view_indexing(self):
         """test index access for views"""
         self.add_engines(2)
         targets = self.client._build_targets('all')[-1]
         v = self.client[:]
         self.assertEquals(v.targets, targets)
         t = self.client.ids[2]
         v = self.client[t]
         self.assert_(isinstance(v, DirectView))
         self.assertEquals(v.targets, t)
         t = self.client.ids[2:4]
         v = self.client[t]
         self.assert_(isinstance(v, DirectView))
         self.assertEquals(v.targets, t)
         v = self.client[::2]
         self.assert_(isinstance(v, DirectView))
         self.assertEquals(v.targets, targets[::2])
         v = self.client[1::3]
         self.assert_(isinstance(v, DirectView))
         self.assertEquals(v.targets, targets[1::3])
         v = self.client[:-3]
         self.assert_(isinstance(v, DirectView))
         self.assertEquals(v.targets, targets[:-3])
         v = self.client[-1]
         self.assert_(isinstance(v, DirectView))
         self.assertEquals(v.targets, targets[-1])
         self.assertRaises(TypeError, lambda : self.client[None])
     
     def test_view_cache(self):
         """test that multiple view requests return the same object"""
         v = self.client[:2]
         v2 =self.client[:2]
         self.assertTrue(v is v2)
-        v = self.client.view()
-        v2 = self.client.view(balanced=True)
+        v = self.client.load_balanced_view()
+        v2 = self.client.load_balanced_view(targets=None)
         self.assertTrue(v is v2)
     
     def test_targets(self):
         """test various valid targets arguments"""
         build = self.client._build_targets
         ids = self.client.ids
         idents,targets = build(None)
         self.assertEquals(ids, targets)
     
     def test_clear(self):
         """test clear behavior"""
-        self.add_engines(2)
-        self.client.block=True
-        self.client.push(dict(a=5))
-        self.client.pull('a')
+        # self.add_engines(2)
+        v = self.client[:]
+        v.block=True
+        v.push(dict(a=5))
+        v.pull('a')
         id0 = self.client.ids[-1]
         self.client.clear(targets=id0)
-        self.client.pull('a', targets=self.client.ids[:-1])
-        self.assertRaisesRemote(NameError, self.client.pull, 'a')
-        self.client.clear()
+        self.client[:-1].pull('a')
+        self.assertRaisesRemote(NameError, self.client[id0].get, 'a')
+        self.client.clear(block=True)
         for i in self.client.ids:
-            self.assertRaisesRemote(NameError, self.client.pull, 'a', targets=i)
-            
-    
-    def test_push_pull(self):
-        """test pushing and pulling"""
-        data = dict(a=10, b=1.05, c=range(10), d={'e':(1,2),'f':'hi'})
-        t = self.client.ids[-1]
-        self.add_engines(2)
-        push = self.client.push
-        pull = self.client.pull
-        self.client.block=True
-        nengines = len(self.client)
-        push({'data':data}, targets=t)
-        d = pull('data', targets=t)
-        self.assertEquals(d, data)
-        push({'data':data})
-        d = pull('data')
-        self.assertEquals(d, nengines*[data])
-        ar = push({'data':data}, block=False)
-        self.assertTrue(isinstance(ar, AsyncResult))
-        r = ar.get()
-        ar = pull('data', block=False)
-        self.assertTrue(isinstance(ar, AsyncResult))
-        r = ar.get()
-        self.assertEquals(r, nengines*[data])
-        push(dict(a=10,b=20))
-        r = pull(('a','b'))
-        self.assertEquals(r, nengines*[[10,20]])
-    
-    def test_push_pull_function(self):
-        "test pushing and pulling functions"
-        def testf(x):
-            return 2.0*x
-        
-        self.add_engines(4)
-        t = self.client.ids[-1]
-        self.client.block=True
-        push = self.client.push
-        pull = self.client.pull
-        execute = self.client.execute
-        push({'testf':testf}, targets=t)
-        r = pull('testf', targets=t)
-        self.assertEqual(r(1.0), testf(1.0))
-        execute('r = testf(10)', targets=t)
-        r = pull('r', targets=t)
-        self.assertEquals(r, testf(10))
-        ar = push({'testf':testf}, block=False)
-        ar.get()
-        ar = pull('testf', block=False)
-        rlist = ar.get()
-        for r in rlist:
-            self.assertEqual(r(1.0), testf(1.0))
-        execute("def g(x): return x*x", targets=t)
-        r = pull(('testf','g'),targets=t)
-        self.assertEquals((r[0](10),r[1](10)), (testf(10), 100))
-    
-    def test_push_function_globals(self):
-        """test that pushed functions have access to globals"""
-        def geta():
-            return a
-        self.add_engines(1)
-        v = self.client[-1]
-        v.block=True
-        v['f'] = geta
-        self.assertRaisesRemote(NameError, v.execute, 'b=f()')
-        v.execute('a=5')
-        v.execute('b=f()')
-        self.assertEquals(v['b'], 5)
-    
-    def test_push_function_defaults(self):
-        """test that pushed functions preserve default args"""
-        def echo(a=10):
-            return a
-        self.add_engines(1)
-        v = self.client[-1]
-        v.block=True
-        v['f'] = echo
-        v.execute('b=f()')
-        self.assertEquals(v['b'], 10)
+            # print i
+            self.assertRaisesRemote(NameError, self.client[i].get, 'a')
 
     def test_get_result(self):
         """test getting results from the Hub."""
         c = clientmod.Client(profile='iptest')
-        self.add_engines(1)
+        # self.add_engines(1)
         t = c.ids[-1]
-        ar = c.apply(wait, (1,), block=False, targets=t)
+        ar = c[t].apply_async(wait, 1)
         # give the monitor time to notice the message
         time.sleep(.25)
         ahr = self.client.get_result(ar.msg_ids)
         self.assertTrue(isinstance(ahr, AsyncHubResult))
         self.assertEquals(ahr.get(), ar.get())
         ar2 = self.client.get_result(ar.msg_ids)
         self.assertFalse(isinstance(ar2, AsyncHubResult))
+        c.close()
     
     def test_ids_list(self):
         """test client.ids"""
-        self.add_engines(2)
+        # self.add_engines(2)
         ids = self.client.ids
         self.assertEquals(ids, self.client._ids)
         self.assertFalse(ids is self.client._ids)
         ids.remove(ids[-1])
         self.assertNotEquals(ids, self.client._ids)
     
-    def test_run_newline(self):
-        """test that run appends newline to files"""
-        tmpfile = mktemp()
-        with open(tmpfile, 'w') as f:
-            f.write("""def g():
-                return 5
-                """)
-        v = self.client[-1]
-        v.run(tmpfile, block=True)
-        self.assertEquals(v.apply_sync(lambda : g()), 5)
-
-    def test_apply_tracked(self):
-        """test tracking for apply"""
-        # self.add_engines(1)
-        t = self.client.ids[-1]
-        self.client.block=False
-        def echo(n=1024*1024, **kwargs):
-            return self.client.apply(lambda x: x, args=('x'*n,), targets=t, **kwargs)
-        ar = echo(1)
-        self.assertTrue(ar._tracker is None)
-        self.assertTrue(ar.sent)
-        ar = echo(track=True)
-        self.assertTrue(isinstance(ar._tracker, zmq.MessageTracker))
-        self.assertEquals(ar.sent, ar._tracker.done)
-        ar._tracker.wait()
-        self.assertTrue(ar.sent)
-        
-    def test_push_tracked(self):
-        t = self.client.ids[-1]
-        ns = dict(x='x'*1024*1024)
-        ar = self.client.push(ns, targets=t, block=False)
-        self.assertTrue(ar._tracker is None)
-        self.assertTrue(ar.sent)
-        
-        ar = self.client.push(ns, targets=t, block=False, track=True)
-        self.assertTrue(isinstance(ar._tracker, zmq.MessageTracker))
-        self.assertEquals(ar.sent, ar._tracker.done)
-        ar._tracker.wait()
-        self.assertTrue(ar.sent)
-        ar.get()
-        
-    def test_scatter_tracked(self):
-        t = self.client.ids
-        x='x'*1024*1024
-        ar = self.client.scatter('x', x, targets=t, block=False)
-        self.assertTrue(ar._tracker is None)
-        self.assertTrue(ar.sent)
-        
-        ar = self.client.scatter('x', x, targets=t, block=False, track=True)
-        self.assertTrue(isinstance(ar._tracker, zmq.MessageTracker))
-        self.assertEquals(ar.sent, ar._tracker.done)
-        ar._tracker.wait()
-        self.assertTrue(ar.sent)
-        ar.get()
-    
-    def test_remote_reference(self):
-        v = self.client[-1]
-        v['a'] = 123
-        ra = clientmod.Reference('a')
-        b = v.apply_sync(lambda x: x, ra)
-        self.assertEquals(b, 123)
+    def test_queue_status(self):
+        # self.addEngine(4)
+        ids = self.client.ids
+        id0 = ids[0]
+        qs = self.client.queue_status(targets=id0)
+        self.assertTrue(isinstance(qs, dict))
+        self.assertEquals(sorted(qs.keys()), ['completed', 'queue', 'tasks'])
+        allqs = self.client.queue_status()
+        self.assertTrue(isinstance(allqs, dict))
+        self.assertEquals(sorted(allqs.keys()), self.client.ids)
+        for eid,qs in allqs.items():
+            self.assertTrue(isinstance(qs, dict))
+            self.assertEquals(sorted(qs.keys()), ['completed', 'queue', 'tasks'])
+
+    def test_shutdown(self):
+        # self.addEngine(4)
+        ids = self.client.ids
+        id0 = ids[0]
+        self.client.shutdown(id0, block=True)
+        while id0 in self.client.ids:
+            time.sleep(0.1)
+            self.client.spin()
 
+        self.assertRaises(IndexError, lambda : self.client[id0])
 
+    def test_result_status(self):
+        pass
+        # to be written

IPython/zmq/parallel/tests/test_newserialized.py

@@ -1,87 +1,108 b''
 """test serialization with newserialized"""
 
+#-------------------------------------------------------------------------------
+#  Copyright (C) 2011  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-------------------------------------------------------------------------------
+
+#-------------------------------------------------------------------------------
+# Imports
+#-------------------------------------------------------------------------------
+
 from unittest import TestCase
 
 from IPython.testing.parametric import parametric
 from IPython.utils import newserialized as ns
 from IPython.utils.pickleutil import can, uncan, CannedObject, CannedFunction
 from IPython.zmq.parallel.tests.clienttest import skip_without
 
 
 class CanningTestCase(TestCase):
     def test_canning(self):
         d = dict(a=5,b=6)
         cd = can(d)
         self.assertTrue(isinstance(cd, dict))
 
     def test_canned_function(self):
         f = lambda : 7
         cf = can(f)
         self.assertTrue(isinstance(cf, CannedFunction))
     
     @parametric
     def test_can_roundtrip(cls):
         objs = [
             dict(),
             set(),
             list(),
             ['a',1,['a',1],u'e'],
         ]
         return map(cls.run_roundtrip, objs)
     
     @classmethod
     def run_roundtrip(self, obj):
         o = uncan(can(obj))
         assert o == obj, "failed assertion: %r == %r"%(o,obj)
     
     def test_serialized_interfaces(self):
 
         us = {'a':10, 'b':range(10)}
         s = ns.serialize(us)
         uus = ns.unserialize(s)
         self.assertTrue(isinstance(s, ns.SerializeIt))
         self.assertEquals(uus, us)
 
     def test_pickle_serialized(self):
         obj = {'a':1.45345, 'b':'asdfsdf', 'c':10000L}
         original = ns.UnSerialized(obj)
         originalSer = ns.SerializeIt(original)
         firstData = originalSer.getData()
         firstTD = originalSer.getTypeDescriptor()
         firstMD = originalSer.getMetadata()
         self.assertEquals(firstTD, 'pickle')
         self.assertEquals(firstMD, {})
         unSerialized = ns.UnSerializeIt(originalSer)
         secondObj = unSerialized.getObject()
         for k, v in secondObj.iteritems():
             self.assertEquals(obj[k], v)
         secondSer = ns.SerializeIt(ns.UnSerialized(secondObj))
         self.assertEquals(firstData, secondSer.getData())
         self.assertEquals(firstTD, secondSer.getTypeDescriptor() )
         self.assertEquals(firstMD, secondSer.getMetadata())
     
     @skip_without('numpy')
     def test_ndarray_serialized(self):
         import numpy
         a = numpy.linspace(0.0, 1.0, 1000)
         unSer1 = ns.UnSerialized(a)
         ser1 = ns.SerializeIt(unSer1)
         td = ser1.getTypeDescriptor()
         self.assertEquals(td, 'ndarray')
         md = ser1.getMetadata()
         self.assertEquals(md['shape'], a.shape)
         self.assertEquals(md['dtype'], a.dtype.str)
         buff = ser1.getData()
         self.assertEquals(buff, numpy.getbuffer(a))
         s = ns.Serialized(buff, td, md)
         final = ns.unserialize(s)
         self.assertEquals(numpy.getbuffer(a), numpy.getbuffer(final))
         self.assertTrue((a==final).all())
         self.assertEquals(a.dtype.str, final.dtype.str)
         self.assertEquals(a.shape, final.shape)
         # test non-copying:
         a[2] = 1e9
         self.assertTrue((a==final).all())
         
+    def test_uncan_function_globals(self):
+        """test that uncanning a module function restores it into its module"""
+        from re import search
+        cf = can(search)
+        csearch = uncan(cf)
+        self.assertEqual(csearch.__module__, search.__module__)
+        self.assertNotEqual(csearch('asd', 'asdf'), None)
+        csearch = uncan(cf, dict(a=5))
+        self.assertEqual(csearch.__module__, search.__module__)
+        self.assertNotEqual(csearch('asd', 'asdf'), None)
         
         
  No newline at end of file

IPython/zmq/parallel/tests/test_streamsession.py

@@ -1,99 +1,111 b''
+"""test building messages with streamsession"""
+
+#-------------------------------------------------------------------------------
+#  Copyright (C) 2011  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-------------------------------------------------------------------------------
+
+#-------------------------------------------------------------------------------
+# Imports
+#-------------------------------------------------------------------------------
 
 import os
 import uuid
 import zmq
 
 from zmq.tests import BaseZMQTestCase
 from zmq.eventloop.zmqstream import ZMQStream
 # from IPython.zmq.tests import SessionTestCase
 from IPython.zmq.parallel import streamsession as ss
 
 class SessionTestCase(BaseZMQTestCase):
     
     def setUp(self):
         BaseZMQTestCase.setUp(self)
         self.session = ss.StreamSession()
 
 class TestSession(SessionTestCase):
     
     def test_msg(self):
         """message format"""
         msg = self.session.msg('execute')
         thekeys = set('header msg_id parent_header msg_type content'.split())
         s = set(msg.keys())
         self.assertEquals(s, thekeys)
         self.assertTrue(isinstance(msg['content'],dict))
         self.assertTrue(isinstance(msg['header'],dict))
         self.assertTrue(isinstance(msg['parent_header'],dict))
         self.assertEquals(msg['msg_type'], 'execute')
         
             
     
     def test_args(self):
         """initialization arguments for StreamSession"""
         s = self.session
         self.assertTrue(s.pack is ss.default_packer)
         self.assertTrue(s.unpack is ss.default_unpacker)
         self.assertEquals(s.username, os.environ.get('USER', 'username'))
         
         s = ss.StreamSession(username=None)
         self.assertEquals(s.username, os.environ.get('USER', 'username'))
         
         self.assertRaises(TypeError, ss.StreamSession, packer='hi')
         self.assertRaises(TypeError, ss.StreamSession, unpacker='hi')
         u = str(uuid.uuid4())
         s = ss.StreamSession(username='carrot', session=u)
         self.assertEquals(s.session, u)
         self.assertEquals(s.username, 'carrot')
         
     def test_tracking(self):
         """test tracking messages"""
         a,b = self.create_bound_pair(zmq.PAIR, zmq.PAIR)
         s = self.session
         stream = ZMQStream(a)
         msg = s.send(a, 'hello', track=False)
         self.assertTrue(msg['tracker'] is None)
         msg = s.send(a, 'hello', track=True)
         self.assertTrue(isinstance(msg['tracker'], zmq.MessageTracker))
         M = zmq.Message(b'hi there', track=True)
         msg = s.send(a, 'hello', buffers=[M], track=True)
         t = msg['tracker']
         self.assertTrue(isinstance(t, zmq.MessageTracker))
         self.assertRaises(zmq.NotDone, t.wait, .1)
         del M
         t.wait(1) # this will raise
         
         
     # def test_rekey(self):
     #     """rekeying dict around json str keys"""
     #     d = {'0': uuid.uuid4(), 0:uuid.uuid4()}
     #     self.assertRaises(KeyError, ss.rekey, d)
     #     
     #     d = {'0': uuid.uuid4(), 1:uuid.uuid4(), 'asdf':uuid.uuid4()}
     #     d2 = {0:d['0'],1:d[1],'asdf':d['asdf']}
     #     rd = ss.rekey(d)
     #     self.assertEquals(d2,rd)
     #     
     #     d = {'1.5':uuid.uuid4(),'1':uuid.uuid4()}
     #     d2 = {1.5:d['1.5'],1:d['1']}
     #     rd = ss.rekey(d)
     #     self.assertEquals(d2,rd)
     #     
     #     d = {'1.0':uuid.uuid4(),'1':uuid.uuid4()}
     #     self.assertRaises(KeyError, ss.rekey, d)
     # 
     def test_unique_msg_ids(self):
         """test that messages receive unique ids"""
         ids = set()
         for i in range(2**12):
             h = self.session.msg_header('test')
             msg_id = h['msg_id']
             self.assertTrue(msg_id not in ids)
             ids.add(msg_id)
     
     def test_feed_identities(self):
         """scrub the front for zmq IDENTITIES"""
         theids = "engine client other".split()
         content = dict(code='whoda',stuff=object())
         themsg = self.session.msg('execute',content=content)
         pmsg = theids

IPython/zmq/parallel/util.py

@@ -1,318 +1,354 b''
 """some generic utilities for dealing with classes, urls, and serialization"""
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2010-2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import re
 import socket
 
 try:
     import cPickle
     pickle = cPickle
 except:
     cPickle = None
     import pickle
 
 
 from IPython.utils.pickleutil import can, uncan, canSequence, uncanSequence
 from IPython.utils.newserialized import serialize, unserialize
 
 ISO8601="%Y-%m-%dT%H:%M:%S.%f"
 
 #-----------------------------------------------------------------------------
 # Classes
 #-----------------------------------------------------------------------------
 
 class Namespace(dict):
     """Subclass of dict for attribute access to keys."""
     
     def __getattr__(self, key):
         """getattr aliased to getitem"""
         if key in self.iterkeys():
             return self[key]
         else:
             raise NameError(key)
 
     def __setattr__(self, key, value):
         """setattr aliased to setitem, with strict"""
         if hasattr(dict, key):
             raise KeyError("Cannot override dict keys %r"%key)
         self[key] = value
     
 
 class ReverseDict(dict):
     """simple double-keyed subset of dict methods."""
     
     def __init__(self, *args, **kwargs):
         dict.__init__(self, *args, **kwargs)
         self._reverse = dict()
         for key, value in self.iteritems():
             self._reverse[value] = key
     
     def __getitem__(self, key):
         try:
             return dict.__getitem__(self, key)
         except KeyError:
             return self._reverse[key]
     
     def __setitem__(self, key, value):
         if key in self._reverse:
             raise KeyError("Can't have key %r on both sides!"%key)
         dict.__setitem__(self, key, value)
         self._reverse[value] = key
     
     def pop(self, key):
         value = dict.pop(self, key)
         self._reverse.pop(value)
         return value
     
     def get(self, key, default=None):
         try:
             return self[key]
         except KeyError:
             return default
 
 #-----------------------------------------------------------------------------
 # Functions
 #-----------------------------------------------------------------------------
 
 def validate_url(url):
     """validate a url for zeromq"""
     if not isinstance(url, basestring):
         raise TypeError("url must be a string, not %r"%type(url))
     url = url.lower()
     
     proto_addr = url.split('://')
     assert len(proto_addr) == 2, 'Invalid url: %r'%url
     proto, addr = proto_addr
     assert proto in ['tcp','pgm','epgm','ipc','inproc'], "Invalid protocol: %r"%proto
     
     # domain pattern adapted from http://www.regexlib.com/REDetails.aspx?regexp_id=391
     # author: Remi Sabourin
     pat = re.compile(r'^([\w\d]([\w\d\-]{0,61}[\w\d])?\.)*[\w\d]([\w\d\-]{0,61}[\w\d])?$')
     
     if proto == 'tcp':
         lis = addr.split(':')
         assert len(lis) == 2, 'Invalid url: %r'%url
         addr,s_port = lis
         try:
             port = int(s_port)
         except ValueError:
             raise AssertionError("Invalid port %r in url: %r"%(port, url))
         
         assert addr == '*' or pat.match(addr) is not None, 'Invalid url: %r'%url
         
     else:
         # only validate tcp urls currently
         pass
     
     return True
 
 
 def validate_url_container(container):
     """validate a potentially nested collection of urls."""
     if isinstance(container, basestring):
         url = container
         return validate_url(url)
     elif isinstance(container, dict):
         container = container.itervalues()
     
     for element in container:
         validate_url_container(element)
 
 
 def split_url(url):
     """split a zmq url (tcp://ip:port) into ('tcp','ip','port')."""
     proto_addr = url.split('://')
     assert len(proto_addr) == 2, 'Invalid url: %r'%url
     proto, addr = proto_addr
     lis = addr.split(':')
     assert len(lis) == 2, 'Invalid url: %r'%url
     addr,s_port = lis
     return proto,addr,s_port
     
 def disambiguate_ip_address(ip, location=None):
     """turn multi-ip interfaces '0.0.0.0' and '*' into connectable
     ones, based on the location (default interpretation of location is localhost)."""
     if ip in ('0.0.0.0', '*'):
         external_ips = socket.gethostbyname_ex(socket.gethostname())[2]
         if location is None or location in external_ips:
             ip='127.0.0.1'
         elif location:
             return location
     return ip
 
 def disambiguate_url(url, location=None):
     """turn multi-ip interfaces '0.0.0.0' and '*' into connectable
     ones, based on the location (default interpretation is localhost).
     
     This is for zeromq urls, such as tcp://*:10101."""
     try:
         proto,ip,port = split_url(url)
     except AssertionError:
         # probably not tcp url; could be ipc, etc.
         return url
     
     ip = disambiguate_ip_address(ip,location)
     
     return "%s://%s:%s"%(proto,ip,port)
 
 
 def rekey(dikt):
     """Rekey a dict that has been forced to use str keys where there should be
     ints by json.  This belongs in the jsonutil added by fperez."""
     for k in dikt.iterkeys():
         if isinstance(k, str):
             ik=fk=None
             try:
                 ik = int(k)
             except ValueError:
                 try:
                     fk = float(k)
                 except ValueError:
                     continue
             if ik is not None:
                 nk = ik
             else:
                 nk = fk
             if nk in dikt:
                 raise KeyError("already have key %r"%nk)
             dikt[nk] = dikt.pop(k)
     return dikt
 
 def serialize_object(obj, threshold=64e-6):
     """Serialize an object into a list of sendable buffers.
     
     Parameters
     ----------
     
     obj : object
         The object to be serialized
     threshold : float
         The threshold for not double-pickling the content.
         
     
     Returns
     -------
     ('pmd', [bufs]) :
         where pmd is the pickled metadata wrapper,
         bufs is a list of data buffers
     """
     databuffers = []
     if isinstance(obj, (list, tuple)):
         clist = canSequence(obj)
         slist = map(serialize, clist)
         for s in slist:
             if s.typeDescriptor in ('buffer', 'ndarray') or s.getDataSize() > threshold:
                 databuffers.append(s.getData())
                 s.data = None
         return pickle.dumps(slist,-1), databuffers
     elif isinstance(obj, dict):
         sobj = {}
         for k in sorted(obj.iterkeys()):
             s = serialize(can(obj[k]))
             if s.typeDescriptor in ('buffer', 'ndarray') or s.getDataSize() > threshold:
                 databuffers.append(s.getData())
                 s.data = None
             sobj[k] = s
         return pickle.dumps(sobj,-1),databuffers
     else:
         s = serialize(can(obj))
         if s.typeDescriptor in ('buffer', 'ndarray') or s.getDataSize() > threshold:
             databuffers.append(s.getData())
             s.data = None
         return pickle.dumps(s,-1),databuffers
             
         
 def unserialize_object(bufs):
     """reconstruct an object serialized by serialize_object from data buffers."""
     bufs = list(bufs)
     sobj = pickle.loads(bufs.pop(0))
     if isinstance(sobj, (list, tuple)):
         for s in sobj:
             if s.data is None:
                 s.data = bufs.pop(0)
         return uncanSequence(map(unserialize, sobj)), bufs
     elif isinstance(sobj, dict):
         newobj = {}
         for k in sorted(sobj.iterkeys()):
             s = sobj[k]
             if s.data is None:
                 s.data = bufs.pop(0)
             newobj[k] = uncan(unserialize(s))
         return newobj, bufs
     else:
         if sobj.data is None:
             sobj.data = bufs.pop(0)
         return uncan(unserialize(sobj)), bufs
 
 def pack_apply_message(f, args, kwargs, threshold=64e-6):
     """pack up a function, args, and kwargs to be sent over the wire
     as a series of buffers. Any object whose data is larger than `threshold`
     will not have their data copied (currently only numpy arrays support zero-copy)"""
     msg = [pickle.dumps(can(f),-1)]
     databuffers = [] # for large objects
     sargs, bufs = serialize_object(args,threshold)
     msg.append(sargs)
     databuffers.extend(bufs)
     skwargs, bufs = serialize_object(kwargs,threshold)
     msg.append(skwargs)
     databuffers.extend(bufs)
     msg.extend(databuffers)
     return msg
 
 def unpack_apply_message(bufs, g=None, copy=True):
     """unpack f,args,kwargs from buffers packed by pack_apply_message()
     Returns: original f,args,kwargs"""
     bufs = list(bufs) # allow us to pop
     assert len(bufs) >= 3, "not enough buffers!"
     if not copy:
         for i in range(3):
             bufs[i] = bufs[i].bytes
     cf = pickle.loads(bufs.pop(0))
     sargs = list(pickle.loads(bufs.pop(0)))
     skwargs = dict(pickle.loads(bufs.pop(0)))
     # print sargs, skwargs
     f = uncan(cf, g)
     for sa in sargs:
         if sa.data is None:
             m = bufs.pop(0)
             if sa.getTypeDescriptor() in ('buffer', 'ndarray'):
                 if copy:
                     sa.data = buffer(m)
                 else:
                     sa.data = m.buffer
             else:
                 if copy:
                     sa.data = m
                 else:
                     sa.data = m.bytes
     
     args = uncanSequence(map(unserialize, sargs), g)
     kwargs = {}
     for k in sorted(skwargs.iterkeys()):
         sa = skwargs[k]
         if sa.data is None:
             m = bufs.pop(0)
             if sa.getTypeDescriptor() in ('buffer', 'ndarray'):
                 if copy:
                     sa.data = buffer(m)
                 else:
                     sa.data = m.buffer
             else:
                 if copy:
                     sa.data = m
                 else:
                     sa.data = m.bytes
 
         kwargs[k] = uncan(unserialize(sa), g)
     
     return f,args,kwargs
 
+#--------------------------------------------------------------------------
+# helpers for implementing old MEC API via view.apply
+#--------------------------------------------------------------------------
+
+def interactive(f):
+    """decorator for making functions appear as interactively defined.
+    This results in the function being linked to the user_ns as globals()
+    instead of the module globals().
+    """
+    f.__module__ = '__main__'
+    return f
+
+@interactive
+def _push(ns):
+    """helper method for implementing `client.push` via `client.apply`"""
+    globals().update(ns)
+
+@interactive
+def _pull(keys):
+    """helper method for implementing `client.pull` via `client.apply`"""
+    user_ns = globals()
+    if isinstance(keys, (list,tuple, set)):
+        for key in keys:
+            if not user_ns.has_key(key):
+                raise NameError("name '%s' is not defined"%key)
+        return map(user_ns.get, keys)
+    else:
+        if not user_ns.has_key(keys):
+            raise NameError("name '%s' is not defined"%keys)
+        return user_ns.get(keys)
+
+@interactive
+def _execute(code):
+    """helper method for implementing `client.execute` via `client.apply`"""
+    exec code in globals()
+

IPython/zmq/parallel/view.py

@@ -1,670 +1,920 b''
 """Views of remote engines."""
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2010  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
+import warnings
+from contextlib import contextmanager
+
+import zmq
+
 from IPython.testing import decorators as testdec
 from IPython.utils.traitlets import HasTraits, Any, Bool, List, Dict, Set, Int, Instance
 
 from IPython.external.decorator import decorator
 
-from .asyncresult import AsyncResult
-from .dependency import Dependency
+from . import map as Map
+from . import util
+from .asyncresult import AsyncResult, AsyncMapResult
+from .dependency import Dependency, dependent
 from .remotefunction import ParallelFunction, parallel, remote
 
 #-----------------------------------------------------------------------------
 # Decorators
 #-----------------------------------------------------------------------------
 
 @decorator
-def myblock(f, self, *args, **kwargs):
-    """override client.block with self.block during a call"""
-    block = self.client.block
-    self.client.block = self.block
-    try:
-        ret = f(self, *args, **kwargs)
-    finally:
-        self.client.block = block
-    return ret
-
-@decorator
 def save_ids(f, self, *args, **kwargs):
     """Keep our history and outstanding attributes up to date after a method call."""
     n_previous = len(self.client.history)
+    try:
     ret = f(self, *args, **kwargs)
+    finally:
     nmsgs = len(self.client.history) - n_previous
     msg_ids = self.client.history[-nmsgs:]
     self.history.extend(msg_ids)
     map(self.outstanding.add, msg_ids)
     return ret
 
 @decorator
 def sync_results(f, self, *args, **kwargs):
     """sync relevant results from self.client to our results attribute."""
     ret = f(self, *args, **kwargs)
     delta = self.outstanding.difference(self.client.outstanding)
     completed = self.outstanding.intersection(delta)
     self.outstanding = self.outstanding.difference(completed)
     for msg_id in completed:
         self.results[msg_id] = self.client.results[msg_id]
     return ret
 
 @decorator
 def spin_after(f, self, *args, **kwargs):
     """call spin after the method."""
     ret = f(self, *args, **kwargs)
     self.spin()
     return ret
 
 #-----------------------------------------------------------------------------
 # Classes
 #-----------------------------------------------------------------------------
 
 class View(HasTraits):
     """Base View class for more convenint apply(f,*args,**kwargs) syntax via attributes.
     
     Don't use this class, use subclasses.
+    
+    Methods
+    -------
+    
+    spin
+        flushes incoming results and registration state changes
+        control methods spin, and requesting `ids` also ensures up to date
+    
+    wait
+        wait on one or more msg_ids
+    
+    execution methods
+        apply
+        legacy: execute, run
+    
+    data movement
+        push, pull, scatter, gather
+    
+    query methods
+        get_result, queue_status, purge_results, result_status
+    
+    control methods
+        abort, shutdown
+    
     """
     block=Bool(False)
-    bound=Bool(False)
-    track=Bool(False)
+    track=Bool(True)
     history=List()
     outstanding = Set()
     results = Dict()
     client = Instance('IPython.zmq.parallel.client.Client')
     
+    _socket = Instance('zmq.Socket')
     _ntargets = Int(1)
-    _balanced = Bool(False)
-    _default_names = List(['block', 'bound', 'track'])
+    _flag_names = List(['block', 'track'])
     _targets = Any()
+    _idents = Any()
     
-    def __init__(self, client=None, targets=None):
-        super(View, self).__init__(client=client)
-        self._targets = targets
+    def __init__(self, client=None, socket=None, targets=None):
+        super(View, self).__init__(client=client, _socket=socket)
         self._ntargets = 1 if isinstance(targets, (int,type(None))) else len(targets)
         self.block = client.block
         
-        for name in self._default_names:
+        self._idents, self._targets = self.client._build_targets(targets)
+        if targets is None or isinstance(targets, int):
+            self._targets = targets
+        for name in self._flag_names:
+            # set flags, if they haven't been set yet
             setattr(self, name, getattr(self, name, None))
         
         assert not self.__class__ is View, "Don't use base View objects, use subclasses"
         
 
     def __repr__(self):
         strtargets = str(self._targets)
         if len(strtargets) > 16:
             strtargets = strtargets[:12]+'...]'
         return "<%s %s>"%(self.__class__.__name__, strtargets)
 
     @property
     def targets(self):
         return self._targets
 
     @targets.setter
     def targets(self, value):
         raise AttributeError("Cannot set View `targets` after construction!")
     
-    @property
-    def balanced(self):
-        return self._balanced
-
-    @balanced.setter
-    def balanced(self, value):
-        raise AttributeError("Cannot set View `balanced` after construction!")
-    
-    def _defaults(self, *excludes):
-        """return dict of our default attributes, excluding names given."""
-        d = dict(balanced=self._balanced, targets=self._targets)
-        for name in self._default_names:
-            if name not in excludes:
-                d[name] = getattr(self, name)
-        return d
-    
     def set_flags(self, **kwargs):
         """set my attribute flags by keyword.
         
-        A View is a wrapper for the Client's apply method, but
-        with attributes that specify keyword arguments, those attributes
-        can be set by keyword argument with this method.
+        Views determine behavior with a few attributes (`block`, `track`, etc.).
+        These attributes can be set all at once by name with this method.
         
         Parameters
         ----------
         
         block : bool
             whether to wait for results
-        bound : bool
-            whether to pass the client's Namespace as the first argument
-            to functions called via `apply`.
         track : bool
             whether to create a MessageTracker to allow the user to 
             safely edit after arrays and buffers during non-copying
             sends.
         """
-        for key in kwargs:
-            if key not in self._default_names:
-                raise KeyError("Invalid name: %r"%key)
-        for name in ('block', 'bound'):
-            if name in kwargs:
-                setattr(self, name, kwargs[name])
+        for name, value in kwargs.iteritems():
+            if name not in self._flag_names:
+                raise KeyError("Invalid name: %r"%name)
+            else:
+                setattr(self, name, value)
+    
+    @contextmanager
+    def temp_flags(self, **kwargs):
+        """temporarily set flags, for use in `with` statements.
+        
+        See set_flags for permanent setting of flags
+        
+        Examples
+        --------
+        
+        >>> view.track=False
+        ...
+        >>> with view.temp_flags(track=True):
+        ...    ar = view.apply(dostuff, my_big_array)
+        ...    ar.tracker.wait() # wait for send to finish
+        >>> view.track
+        False
+        
+        """
+        # preflight: save flags, and set temporaries
+        saved_flags = {}
+        for f in self._flag_names:
+            saved_flags[f] = getattr(self, f)
+        self.set_flags(**kwargs)
+        # yield to the with-statement block
+        yield
+        # postflight: restore saved flags
+        self.set_flags(**saved_flags)
+        
     
     #----------------------------------------------------------------
-    # wrappers for client methods:
+    # apply
     #----------------------------------------------------------------
-    @sync_results
-    def spin(self):
-        """spin the client, and sync"""
-        self.client.spin()
     
     @sync_results
     @save_ids
+    def _really_apply(self, f, args, kwargs, block=None, **options):
+        """wrapper for client.send_apply_message"""
+        raise NotImplementedError("Implement in subclasses")
+    
     def apply(self, f, *args, **kwargs):
         """calls f(*args, **kwargs) on remote engines, returning the result.
         
-        This method sets all of `client.apply`'s keyword arguments via this
-        View's attributes.
+        This method sets all apply flags via this View's attributes.
         
         if self.block is False:
             returns AsyncResult
         else:
             returns actual result of f(*args, **kwargs)
         """
-        return self.client.apply(f, args, kwargs, **self._defaults())
+        return self._really_apply(f, args, kwargs)
 
-    @save_ids
     def apply_async(self, f, *args, **kwargs):
         """calls f(*args, **kwargs) on remote engines in a nonblocking manner.
         
         returns AsyncResult
         """
-        d = self._defaults('block', 'bound')
-        return self.client.apply(f,args,kwargs, block=False, bound=False, **d)
+        return self._really_apply(f, args, kwargs, block=False)
 
     @spin_after
-    @save_ids
     def apply_sync(self, f, *args, **kwargs):
         """calls f(*args, **kwargs) on remote engines in a blocking manner,
          returning the result.
         
         returns: actual result of f(*args, **kwargs)
         """
-        d = self._defaults('block', 'bound', 'track')
-        return self.client.apply(f,args,kwargs, block=True, bound=False, **d)
+        return self._really_apply(f, args, kwargs, block=True)
 
-    # @sync_results
-    # @save_ids
-    # def apply_bound(self, f, *args, **kwargs):
-    #     """calls f(*args, **kwargs) bound to engine namespace(s).
-    #     
-    #     if self.block is False:
-    #         returns msg_id
-    #     else:
-    #         returns actual result of f(*args, **kwargs)
-    #     
-    #     This method has access to the targets' namespace via globals()
-    #     
-    #     """
-    #     d = self._defaults('bound')
-    #     return self.client.apply(f, args, kwargs, bound=True, **d)
-    # 
+    #----------------------------------------------------------------
+    # wrappers for client and control methods
+    #----------------------------------------------------------------
     @sync_results
-    @save_ids
-    def apply_async_bound(self, f, *args, **kwargs):
-        """calls f(*args, **kwargs) bound to engine namespace(s) 
-        in a nonblocking manner.
-        
-        The first argument to `f` will be the Engine's Namespace
-        
-        returns: AsyncResult
+    def spin(self):
+        """spin the client, and sync"""
+        self.client.spin()
         
-        """
-        d = self._defaults('block', 'bound')
-        return self.client.apply(f, args, kwargs, block=False, bound=True, **d)
+    @sync_results
+    def wait(self, jobs=None, timeout=-1):
+        """waits on one or more `jobs`, for up to `timeout` seconds.
 
-    @spin_after
-    @save_ids
-    def apply_sync_bound(self, f, *args, **kwargs):
-        """calls f(*args, **kwargs) bound to engine namespace(s), waiting for the result.
+        Parameters
+        ----------
         
-        The first argument to `f` will be the Engine's Namespace
+        jobs : int, str, or list of ints and/or strs, or one or more AsyncResult objects
+                ints are indices to self.history
+                strs are msg_ids
+                default: wait on all outstanding messages
+        timeout : float
+                a time in seconds, after which to give up.
+                default is -1, which means no timeout
         
-        returns: actual result of f(*args, **kwargs)
+        Returns
+        -------
         
+        True : when all msg_ids are done
+        False : timeout reached, some msg_ids still outstanding
         """
-        d = self._defaults('block', 'bound')
-        return self.client.apply(f, args, kwargs, block=True, bound=True, **d)
+        if jobs is None:
+            jobs = self.history
+        return self.client.wait(jobs, timeout)
     
     def abort(self, jobs=None, block=None):
         """Abort jobs on my engines.
         
         Parameters
         ----------
         
         jobs : None, str, list of strs, optional
             if None: abort all jobs.
             else: abort specific msg_id(s).
         """
         block = block if block is not None else self.block
         return self.client.abort(jobs=jobs, targets=self._targets, block=block)
 
     def queue_status(self, verbose=False):
         """Fetch the Queue status of my engines"""
         return self.client.queue_status(targets=self._targets, verbose=verbose)
     
     def purge_results(self, jobs=[], targets=[]):
         """Instruct the controller to forget specific results."""
         if targets is None or targets == 'all':
             targets = self._targets
         return self.client.purge_results(jobs=jobs, targets=targets)
     
     @spin_after
     def get_result(self, indices_or_msg_ids=None):
         """return one or more results, specified by history index or msg_id.
         
         See client.get_result for details.
         
         """
         
         if indices_or_msg_ids is None:
             indices_or_msg_ids = -1
         if isinstance(indices_or_msg_ids, int):
             indices_or_msg_ids = self.history[indices_or_msg_ids]
         elif isinstance(indices_or_msg_ids, (list,tuple,set)):
             indices_or_msg_ids = list(indices_or_msg_ids)
             for i,index in enumerate(indices_or_msg_ids):
                 if isinstance(index, int):
                     indices_or_msg_ids[i] = self.history[index]
         return self.client.get_result(indices_or_msg_ids)
     
     #-------------------------------------------------------------------
     # Map
     #-------------------------------------------------------------------
     
     def map(self, f, *sequences, **kwargs):
         """override in subclasses"""
         raise NotImplementedError
     
     def map_async(self, f, *sequences, **kwargs):
         """Parallel version of builtin `map`, using this view's engines.
         
         This is equivalent to map(...block=False)
         
         See `self.map` for details.
         """
         if 'block' in kwargs:
             raise TypeError("map_async doesn't take a `block` keyword argument.")
         kwargs['block'] = False
         return self.map(f,*sequences,**kwargs)
     
     def map_sync(self, f, *sequences, **kwargs):
         """Parallel version of builtin `map`, using this view's engines.
         
         This is equivalent to map(...block=True)
         
         See `self.map` for details.
         """
         if 'block' in kwargs:
             raise TypeError("map_sync doesn't take a `block` keyword argument.")
         kwargs['block'] = True
         return self.map(f,*sequences,**kwargs)
     
     def imap(self, f, *sequences, **kwargs):
         """Parallel version of `itertools.imap`.
         
         See `self.map` for details.
+        
         """
         
         return iter(self.map_async(f,*sequences, **kwargs))
     
     #-------------------------------------------------------------------
     # Decorators
     #-------------------------------------------------------------------
     
-    def remote(self, bound=False, block=True):
+    def remote(self, block=True, **flags):
         """Decorator for making a RemoteFunction"""
-        return remote(self.client, bound=bound, targets=self._targets, block=block, balanced=self._balanced)
+        block = self.block if block is None else block
+        return remote(self, block=block, **flags)
     
-    def parallel(self, dist='b', bound=False, block=None):
+    def parallel(self, dist='b', block=None, **flags):
         """Decorator for making a ParallelFunction"""
         block = self.block if block is None else block
-        return parallel(self.client, bound=bound, targets=self._targets, block=block, balanced=self._balanced)
+        return parallel(self, dist=dist, block=block, **flags)
 
 @testdec.skip_doctest
 class DirectView(View):
     """Direct Multiplexer View of one or more engines.
     
     These are created via indexed access to a client:
     
     >>> dv_1 = client[1]
     >>> dv_all = client[:]
     >>> dv_even = client[::2]
     >>> dv_some = client[1:3]
     
     This object provides dictionary access to engine namespaces:
     
     # push a=5:
     >>> dv['a'] = 5 
     # pull 'foo':
     >>> db['foo']
     
     """
     
-    def __init__(self, client=None, targets=None):
-        super(DirectView, self).__init__(client=client, targets=targets)
-        self._balanced = False
+    def __init__(self, client=None, socket=None, targets=None):
+        super(DirectView, self).__init__(client=client, socket=socket, targets=targets)
     
-    @spin_after
+    
+    @sync_results
     @save_ids
+    def _really_apply(self, f, args=None, kwargs=None, block=None, track=None):
+        """calls f(*args, **kwargs) on remote engines, returning the result.
+        
+        This method sets all of `apply`'s flags via this View's attributes.
+        
+        Parameters
+        ----------
+        
+        f : callable
+        
+        args : list [default: empty]
+        
+        kwargs : dict [default: empty]
+        
+        block : bool [default: self.block]
+            whether to block 
+        track : bool [default: self.track]
+            whether to ask zmq to track the message, for safe non-copying sends
+        
+        Returns
+        -------
+        
+        if self.block is False:
+            returns AsyncResult
+        else:
+            returns actual result of f(*args, **kwargs) on the engine(s)
+            This will be a list of self.targets is also a list (even length 1), or
+            the single result if self.targets is an integer engine id
+        """
+        args = [] if args is None else args
+        kwargs = {} if kwargs is None else kwargs
+        block = self.block if block is None else block
+        track = self.track if track is None else track
+        msg_ids = []
+        trackers = []
+        for ident in self._idents:
+            msg = self.client.send_apply_message(self._socket, f, args, kwargs, track=track,
+                                    ident=ident)
+            if track:
+                trackers.append(msg['tracker'])
+            msg_ids.append(msg['msg_id'])
+        tracker = None if track is False else zmq.MessageTracker(*trackers)
+        ar = AsyncResult(self.client, msg_ids, fname=f.__name__, targets=self._targets, tracker=tracker)
+        if block:
+            try:
+                return ar.get()
+            except KeyboardInterrupt:
+                pass
+        return ar
+    
+    @spin_after
     def map(self, f, *sequences, **kwargs):
-        """view.map(f, *sequences, block=self.block, bound=self.bound) => list|AsyncMapResult
+        """view.map(f, *sequences, block=self.block) => list|AsyncMapResult
         
         Parallel version of builtin `map`, using this View's `targets`.
         
         There will be one task per target, so work will be chunked
         if the sequences are longer than `targets`.  
         
         Results can be iterated as they are ready, but will become available in chunks.
         
         Parameters
         ----------
         
         f : callable
             function to be mapped
         *sequences: one or more sequences of matching length
             the sequences to be distributed and passed to `f`
         block : bool
             whether to wait for the result or not [default self.block]
-        bound : bool
-            whether to pass the client's Namespace as the first argument to `f`
         
         Returns
         -------
         
         if block=False:
             AsyncMapResult
                 An object like AsyncResult, but which reassembles the sequence of results
                 into a single list. AsyncMapResults can be iterated through before all
                 results are complete.
         else:
             list
                 the result of map(f,*sequences)
         """
         
-        block = kwargs.get('block', self.block)
-        bound = kwargs.get('bound', self.bound)
+        block = kwargs.pop('block', self.block)
         for k in kwargs.keys():
-            if k not in ['block', 'bound']:
+            if k not in ['block', 'track']:
                 raise TypeError("invalid keyword arg, %r"%k)
         
         assert len(sequences) > 0, "must have some sequences to map onto!"
-        pf = ParallelFunction(self.client, f, block=block, bound=bound,
-                        targets=self._targets, balanced=False)
+        pf = ParallelFunction(self, f, block=block, **kwargs)
         return pf.map(*sequences)
     
-    @sync_results
-    @save_ids
     def execute(self, code, block=None):
-        """execute some code on my targets."""
+        """Executes `code` on `targets` in blocking or nonblocking manner.
         
-        block = block if block is not None else self.block
+        ``execute`` is always `bound` (affects engine namespace)
         
-        return self.client.execute(code, block=block, targets=self._targets)
+        Parameters
+        ----------
     
-    @sync_results
-    @save_ids
-    def run(self, fname, block=None):
-        """execute the code in a file on my targets."""
+        code : str
+                the code string to be executed
+        block : bool
+                whether or not to wait until done to return
+                default: self.block
+        """
+        return self._really_apply(util._execute, args=(code,), block=block)
         
-        block = block if block is not None else self.block
+    def run(self, filename, block=None):
+        """Execute contents of `filename` on my engine(s). 
+        
+        This simply reads the contents of the file and calls `execute`.
+        
+        Parameters
+        ----------
+        
+        filename : str
+                The path to the file
+        targets : int/str/list of ints/strs
+                the engines on which to execute
+                default : all
+        block : bool
+                whether or not to wait until done
+                default: self.block
         
-        return self.client.run(fname, block=block, targets=self._targets)
+        """
+        with open(filename, 'r') as f:
+            # add newline in case of trailing indented whitespace
+            # which will cause SyntaxError
+            code = f.read()+'\n'
+        return self.execute(code, block=block)
     
     def update(self, ns):
-        """update remote namespace with dict `ns`"""
-        return self.client.push(ns, targets=self._targets, block=self.block)
+        """update remote namespace with dict `ns`
     
-    def push(self, ns, block=None):
-        """update remote namespace with dict `ns`"""
+        See `push` for details.
+        """
+        return self.push(ns, block=self.block, track=self.track)
         
-        block = block if block is not None else self.block
+    def push(self, ns, block=None, track=None):
+        """update remote namespace with dict `ns`
         
-        return self.client.push(ns, targets=self._targets, block=block)
+        Parameters
+        ----------
+        
+        ns : dict
+            dict of keys with which to update engine namespace(s)
+        block : bool [default : self.block]
+            whether to wait to be notified of engine receipt
+        
+        """
+        
+        block = block if block is not None else self.block
+        track = track if track is not None else self.track
+        # applier = self.apply_sync if block else self.apply_async
+        if not isinstance(ns, dict):
+            raise TypeError("Must be a dict, not %s"%type(ns))
+        return self._really_apply(util._push, (ns,),block=block, track=track)
 
     def get(self, key_s):
         """get object(s) by `key_s` from remote namespace
-        will return one object if it is a key.
-        It also takes a list of keys, and will return a list of objects."""
+        
+        see `pull` for details.
+        """
         # block = block if block is not None else self.block
-        return self.client.pull(key_s, block=True, targets=self._targets)
+        return self.pull(key_s, block=True)
+    
+    def pull(self, names, block=True):
+        """get object(s) by `name` from remote namespace
     
-    @sync_results
-    @save_ids
-    def pull(self, key_s, block=True):
-        """get object(s) by `key_s` from remote namespace
         will return one object if it is a key.
-        It also takes a list of keys, and will return a list of objects."""
+        can also take a list of keys, in which case it will return a list of objects.
+        """
         block = block if block is not None else self.block
-        return self.client.pull(key_s, block=block, targets=self._targets)
+        applier = self.apply_sync if block else self.apply_async
+        if isinstance(names, basestring):
+            pass
+        elif isinstance(names, (list,tuple,set)):
+            for key in names:
+                if not isinstance(key, basestring):
+                    raise TypeError("keys must be str, not type %r"%type(key))
+        else:
+            raise TypeError("names must be strs, not %r"%names)
+        return applier(util._pull, names)
     
-    def scatter(self, key, seq, dist='b', flatten=False, block=None):
+    def scatter(self, key, seq, dist='b', flatten=False, block=None, track=None):
         """
         Partition a Python sequence and send the partitions to a set of engines.
         """
         block = block if block is not None else self.block
+        track = track if track is not None else self.track
+        targets = self._targets
+        mapObject = Map.dists[dist]()
+        nparts = len(targets)
+        msg_ids = []
+        trackers = []
+        for index, engineid in enumerate(targets):
+            push = self.client[engineid].push
+            partition = mapObject.getPartition(seq, index, nparts)
+            if flatten and len(partition) == 1:
+                r = push({key: partition[0]}, block=False, track=track)
+            else:
+                r = push({key: partition},block=False, track=track)
+            msg_ids.extend(r.msg_ids)
+            if track:
+                trackers.append(r._tracker)
         
-        return self.client.scatter(key, seq, dist=dist, flatten=flatten,
-                    targets=self._targets, block=block)
+        if track:
+            tracker = zmq.MessageTracker(*trackers)
+        else:
+            tracker = None
+        
+        r = AsyncResult(self.client, msg_ids, fname='scatter', targets=targets, tracker=tracker)
+        if block:
+            r.wait()
+        else:
+            return r
     
     @sync_results
     @save_ids
     def gather(self, key, dist='b', block=None):
         """
         Gather a partitioned sequence on a set of engines as a single local seq.
         """
         block = block if block is not None else self.block
+        mapObject = Map.dists[dist]()
+        msg_ids = []
+        for index, engineid in enumerate(self._targets):
         
-        return self.client.gather(key, dist=dist, targets=self._targets, block=block)
+            msg_ids.extend(self.client[engineid].pull(key, block=False).msg_ids)
+        
+        r = AsyncMapResult(self.client, msg_ids, mapObject, fname='gather')
+        
+        if block:
+            try:
+                return r.get()
+            except KeyboardInterrupt:
+                pass
+        return r
     
     def __getitem__(self, key):
         return self.get(key)
     
     def __setitem__(self,key, value):
         self.update({key:value})
     
     def clear(self, block=False):
         """Clear the remote namespaces on my engines."""
         block = block if block is not None else self.block
         return self.client.clear(targets=self._targets, block=block)
     
     def kill(self, block=True):
         """Kill my engines."""
         block = block if block is not None else self.block
         return self.client.kill(targets=self._targets, block=block)
     
     #----------------------------------------
     # activate for %px,%autopx magics
     #----------------------------------------
     def activate(self):
         """Make this `View` active for parallel magic commands.
         
         IPython has a magic command syntax to work with `MultiEngineClient` objects.
         In a given IPython session there is a single active one.  While
         there can be many `Views` created and used by the user, 
         there is only one active one.  The active `View` is used whenever 
         the magic commands %px and %autopx are used.
         
         The activate() method is called on a given `View` to make it 
         active.  Once this has been done, the magic commands can be used.
         """
         
         try:
             # This is injected into __builtins__.
             ip = get_ipython()
         except NameError:
             print "The IPython parallel magics (%result, %px, %autopx) only work within IPython."
         else:
             pmagic = ip.plugin_manager.get_plugin('parallelmagic')
             if pmagic is not None:
                 pmagic.active_multiengine_client = self
             else:
                 print "You must first load the parallelmagic extension " \
                       "by doing '%load_ext parallelmagic'"
 
 
 @testdec.skip_doctest
 class LoadBalancedView(View):
     """An load-balancing View that only executes via the Task scheduler.
     
     Load-balanced views can be created with the client's `view` method:
     
-    >>> v = client.view(balanced=True)
+    >>> v = client.load_balanced_view()
     
     or targets can be specified, to restrict the potential destinations:
     
-    >>> v = client.view([1,3],balanced=True)
+    >>> v = client.client.load_balanced_view(([1,3])
     
     which would restrict loadbalancing to between engines 1 and 3.
     
     """
     
-    _default_names = ['block', 'bound', 'follow', 'after', 'timeout']
+    _flag_names = ['block', 'track', 'follow', 'after', 'timeout']
     
-    def __init__(self, client=None, targets=None):
-        super(LoadBalancedView, self).__init__(client=client, targets=targets)
+    def __init__(self, client=None, socket=None, targets=None):
+        super(LoadBalancedView, self).__init__(client=client, socket=socket, targets=targets)
         self._ntargets = 1
-        self._balanced = True
+        self._task_scheme=client._task_scheme
+        if targets is None:
+            self._targets = None
+            self._idents=[]
     
     def _validate_dependency(self, dep):
         """validate a dependency.
         
         For use in `set_flags`.
         """
         if dep is None or isinstance(dep, (str, AsyncResult, Dependency)):
             return True
         elif isinstance(dep, (list,set, tuple)):
             for d in dep:
-                if not isinstance(d, str, AsyncResult):
+                if not isinstance(d, (str, AsyncResult)):
                     return False
         elif isinstance(dep, dict):
             if set(dep.keys()) != set(Dependency().as_dict().keys()):
                 return False
             if not isinstance(dep['msg_ids'], list):
                 return False
             for d in dep['msg_ids']:
                 if not isinstance(d, str):
                     return False
         else:
             return False
                 
+        return True
+                
+    def _render_dependency(self, dep):
+        """helper for building jsonable dependencies from various input forms."""
+        if isinstance(dep, Dependency):
+            return dep.as_dict()
+        elif isinstance(dep, AsyncResult):
+            return dep.msg_ids
+        elif dep is None:
+            return []
+        else:
+            # pass to Dependency constructor
+            return list(Dependency(dep))
+
     def set_flags(self, **kwargs):
         """set my attribute flags by keyword.
         
         A View is a wrapper for the Client's apply method, but with attributes
         that specify keyword arguments, those attributes can be set by keyword
         argument with this method.
         
         Parameters
         ----------
         
         block : bool
             whether to wait for results
-        bound : bool
-            whether to pass the client's Namespace as the first argument
-            to functions called via `apply`.
         track : bool
             whether to create a MessageTracker to allow the user to 
             safely edit after arrays and buffers during non-copying
             sends.
-        follow : Dependency, list, msg_id, AsyncResult
-            the location dependencies of tasks
-        after : Dependency, list, msg_id, AsyncResult
-            the time dependencies of tasks
-        timeout : int,None
-            the timeout to be used for tasks
+        #
+        after : Dependency or collection of msg_ids
+            Only for load-balanced execution (targets=None)
+            Specify a list of msg_ids as a time-based dependency.
+            This job will only be run *after* the dependencies
+            have been met.
+
+        follow : Dependency or collection of msg_ids
+            Only for load-balanced execution (targets=None)
+            Specify a list of msg_ids as a location-based dependency.
+            This job will only be run on an engine where this dependency
+            is met.
+
+        timeout : float/int or None
+            Only for load-balanced execution (targets=None)
+            Specify an amount of time (in seconds) for the scheduler to
+            wait for dependencies to be met before failing with a
+            DependencyTimeout.
         """
         
         super(LoadBalancedView, self).set_flags(**kwargs)
         for name in ('follow', 'after'):
             if name in kwargs:
                 value = kwargs[name]
                 if self._validate_dependency(value):
                     setattr(self, name, value)
                 else:
                     raise ValueError("Invalid dependency: %r"%value)
         if 'timeout' in kwargs:
             t = kwargs['timeout']
-            if not isinstance(t, (int, long, float, None)):
+            if not isinstance(t, (int, long, float, type(None))):
                 raise TypeError("Invalid type for timeout: %r"%type(t))
             if t is not None:
                 if t < 0:
                     raise ValueError("Invalid timeout: %s"%t)
             self.timeout = t
                 
+    @sync_results
+    @save_ids
+    def _really_apply(self, f, args=None, kwargs=None, block=None, track=None,
+                                        after=None, follow=None, timeout=None):
+        """calls f(*args, **kwargs) on a remote engine, returning the result.
+        
+        This method temporarily sets all of `apply`'s flags for a single call.
+        
+        Parameters
+        ----------
+        
+        f : callable
+        
+        args : list [default: empty]
+        
+        kwargs : dict [default: empty]
+        
+        block : bool [default: self.block]
+            whether to block 
+        track : bool [default: self.track]
+            whether to ask zmq to track the message, for safe non-copying sends
+            
+        !!!!!! TODO: THE REST HERE  !!!!
+        
+        Returns
+        -------
+        
+        if self.block is False:
+            returns AsyncResult
+        else:
+            returns actual result of f(*args, **kwargs) on the engine(s)
+            This will be a list of self.targets is also a list (even length 1), or
+            the single result if self.targets is an integer engine id
+        """
+
+        # validate whether we can run
+        if self._socket.closed:
+            msg = "Task farming is disabled"
+            if self._task_scheme == 'pure':
+                msg += " because the pure ZMQ scheduler cannot handle"
+                msg += " disappearing engines."
+            raise RuntimeError(msg)
+        
+        if self._task_scheme == 'pure':
+            # pure zmq scheme doesn't support dependencies
+            msg = "Pure ZMQ scheduler doesn't support dependencies"
+            if (follow or after):
+                # hard fail on DAG dependencies
+                raise RuntimeError(msg)
+            if isinstance(f, dependent):
+                # soft warn on functional dependencies
+                warnings.warn(msg, RuntimeWarning)
+        
+        # build args
+        args = [] if args is None else args
+        kwargs = {} if kwargs is None else kwargs
+        block = self.block if block is None else block
+        track = self.track if track is None else track
+        after = self.after if after is None else after
+        follow = self.follow if follow is None else follow
+        timeout = self.timeout if timeout is None else timeout
+        after = self._render_dependency(after)
+        follow = self._render_dependency(follow)
+        subheader = dict(after=after, follow=follow, timeout=timeout, targets=self._idents)
+        
+        msg = self.client.send_apply_message(self._socket, f, args, kwargs, track=track,
+                                subheader=subheader)
+        tracker = None if track is False else msg['tracker']
+        
+        ar = AsyncResult(self.client, msg['msg_id'], fname=f.__name__, targets=None, tracker=tracker)
+        
+        if block:
+            try:
+                return ar.get()
+            except KeyboardInterrupt:
+                pass
+        return ar
+        
     @spin_after
     @save_ids
     def map(self, f, *sequences, **kwargs):
-        """view.map(f, *sequences, block=self.block, bound=self.bound, chunk_size=1) => list|AsyncMapResult
+        """view.map(f, *sequences, block=self.block, chunksize=1) => list|AsyncMapResult
         
         Parallel version of builtin `map`, load-balanced by this View.
         
-        `block`, `bound`, and `chunk_size` can be specified by keyword only.
+        `block`, and `chunksize` can be specified by keyword only.
         
-        Each `chunk_size` elements will be a separate task, and will be
+        Each `chunksize` elements will be a separate task, and will be
         load-balanced. This lets individual elements be available for iteration
         as soon as they arrive.
         
         Parameters
         ----------
         
         f : callable
             function to be mapped
         *sequences: one or more sequences of matching length
             the sequences to be distributed and passed to `f`
         block : bool
             whether to wait for the result or not [default self.block]
-        bound : bool
-            whether to pass the client's Namespace as the first argument to `f`
         track : bool
             whether to create a MessageTracker to allow the user to 
             safely edit after arrays and buffers during non-copying
             sends.
-        chunk_size : int
+        chunksize : int
             how many elements should be in each task [default 1]
         
         Returns
         -------
         
         if block=False:
             AsyncMapResult
                 An object like AsyncResult, but which reassembles the sequence of results
                 into a single list. AsyncMapResults can be iterated through before all
                 results are complete.
             else:
                 the result of map(f,*sequences)
         
         """
         
         # default
         block = kwargs.get('block', self.block)
-        bound = kwargs.get('bound', self.bound)
-        chunk_size = kwargs.get('chunk_size', 1)
+        chunksize = kwargs.get('chunksize', 1)
         
         keyset = set(kwargs.keys())
-        extra_keys = keyset.difference_update(set(['block', 'bound', 'chunk_size']))
+        extra_keys = keyset.difference_update(set(['block', 'chunksize']))
         if extra_keys:
             raise TypeError("Invalid kwargs: %s"%list(extra_keys))
         
         assert len(sequences) > 0, "must have some sequences to map onto!"
         
-        pf = ParallelFunction(self.client, f, block=block, bound=bound, 
-                                targets=self._targets, balanced=True,
-                                chunk_size=chunk_size)
+        pf = ParallelFunction(self, f, block=block,  chunksize=chunksize)
         return pf.map(*sequences)
     
 __all__ = ['LoadBalancedView', 'DirectView']
  No newline at end of file

docs/examples/newparallel/dagdeps.py

@@ -1,119 +1,120 b''
 """Example for generating an arbitrary DAG as a dependency map.
 
 This demo uses networkx to generate the graph.
 
 Authors
 -------
 * MinRK
 """
 import networkx as nx
 from random import randint, random
 from IPython.zmq.parallel import client as cmod
 
 def randomwait():
     import time
     from random import random
     time.sleep(random())
     return time.time()
 
 
 def random_dag(nodes, edges):
     """Generate a random Directed Acyclic Graph (DAG) with a given number of nodes and edges."""
     G = nx.DiGraph()
     for i in range(nodes):
         G.add_node(i)
     while edges > 0:
         a = randint(0,nodes-1)
         b=a
         while b==a:
             b = randint(0,nodes-1)
         G.add_edge(a,b)
         if nx.is_directed_acyclic_graph(G):
             edges -= 1
         else:
             # we closed a loop!
             G.remove_edge(a,b)
     return G
 
 def add_children(G, parent, level, n=2):
     """Add children recursively to a binary tree."""
     if level == 0:
         return
     for i in range(n):
         child = parent+str(i)
         G.add_node(child)
         G.add_edge(parent,child)
         add_children(G, child, level-1, n)
 
 def make_bintree(levels):
     """Make a symmetrical binary tree with @levels"""
     G = nx.DiGraph()
     root = '0'
     G.add_node(root)
     add_children(G, root, levels, 2)
     return G
 
-def submit_jobs(client, G, jobs):
+def submit_jobs(view, G, jobs):
     """Submit jobs via client where G describes the time dependencies."""
     results = {}
     for node in nx.topological_sort(G):
-        deps = [ results[n] for n in G.predecessors(node) ]
-        results[node] = client.apply(jobs[node], after=deps)
+        with view.temp_flags(after=[ results[n] for n in G.predecessors(node) ]):
+            results[node] = view.apply(jobs[node])
     return results
 
 def validate_tree(G, results):
     """Validate that jobs executed after their dependencies."""
     for node in G:
         started = results[node].metadata.started
         for parent in G.predecessors(node):
             finished = results[parent].metadata.completed
             assert started > finished, "%s should have happened after %s"%(node, parent)
 
 def main(nodes, edges):
     """Generate a random graph, submit jobs, then validate that the
     dependency order was enforced.
     Finally, plot the graph, with time on the x-axis, and
     in-degree on the y (just for spread).  All arrows must
     point at least slightly to the right if the graph is valid.
     """
-    import pylab
+    from matplotlib import pyplot as plt
     from matplotlib.dates import date2num
     from matplotlib.cm import gist_rainbow
     print "building DAG"
     G = random_dag(nodes, edges)
     jobs = {}
     pos = {}
     colors = {}
     for node in G:
         jobs[node] = randomwait
     
     client = cmod.Client()
+    view = client.load_balanced_view()
     print "submitting %i tasks with %i dependencies"%(nodes,edges)
-    results = submit_jobs(client, G, jobs)
+    results = submit_jobs(view, G, jobs)
     print "waiting for results"
-    client.barrier()
+    view.wait()
     print "done"
     for node in G:
         md = results[node].metadata
         start = date2num(md.started)
         runtime = date2num(md.completed) - start
         pos[node] = (start, runtime)
         colors[node] = md.engine_id
     validate_tree(G, results)
     nx.draw(G, pos, node_list=colors.keys(), node_color=colors.values(), cmap=gist_rainbow,
             with_labels=False)
     x,y = zip(*pos.values())
     xmin,ymin = map(min, (x,y))
     xmax,ymax = map(max, (x,y))
     xscale = xmax-xmin
     yscale = ymax-ymin
-    pylab.xlim(xmin-xscale*.1,xmax+xscale*.1)
-    pylab.ylim(ymin-yscale*.1,ymax+yscale*.1)
+    plt.xlim(xmin-xscale*.1,xmax+xscale*.1)
+    plt.ylim(ymin-yscale*.1,ymax+yscale*.1)
     return G,results
 
 if __name__ == '__main__':
-    import pylab
+    from matplotlib import pyplot as plt
     # main(5,10)
     main(32,96)
-    pylab.show()
+    plt.show()
     
  No newline at end of file

docs/examples/newparallel/demo/dependencies.py

@@ -1,118 +1,130 b''
 from IPython.zmq.parallel import error
 from IPython.zmq.parallel.dependency import Dependency
 from IPython.zmq.parallel.client import *
 
 client = Client()
 
 # this will only run on machines that can import numpy:
 @require('numpy')
 def norm(A):
     from numpy.linalg import norm
     return norm(A,2)
 
 def checkpid(pid):
     """return the pid of the engine"""
     import os
     return os.getpid() == pid
 
 def checkhostname(host):
     import socket
     return socket.gethostname() == host
 
 def getpid():
     import os
     return os.getpid()
 
 pid0 = client[0].apply_sync(getpid)
 
 # this will depend on the pid being that of target 0:
 @depend(checkpid, pid0)
 def getpid2():
     import os
     return os.getpid()
 
-view = client[None]
+view = client.load_balanced_view()
 view.block=True
 
 # will run on anything:
 pids1 = [ view.apply(getpid) for i in range(len(client.ids)) ]
 print pids1
 # will only run on e0:
 pids2 = [ view.apply(getpid2) for i in range(len(client.ids)) ]
 print pids2
 
 print "now test some dependency behaviors"
 
 def wait(t):
     import time
     time.sleep(t)
     return t
 
 # fail after some time:
 def wait_and_fail(t):
     import time
     time.sleep(t)
     return 1/0
 
 successes = [ view.apply_async(wait, 1).msg_ids[0] for i in range(len(client.ids)) ]
 failures = [ view.apply_async(wait_and_fail, 1).msg_ids[0] for i in range(len(client.ids)) ]
 
 mixed = [failures[0],successes[0]]
-d1a = Dependency(mixed, mode='any', success_only=False) # yes
-d1b = Dependency(mixed, mode='any', success_only=True) # yes
-d2a = Dependency(mixed, mode='all', success_only=False) # yes after / no follow
-d2b = Dependency(mixed, mode='all', success_only=True) # no
-d3 = Dependency(failures, mode='any', success_only=True) # no
-d4 = Dependency(failures, mode='any', success_only=False) # yes
-d5 = Dependency(failures, mode='all', success_only=False) # yes after / no follow
-d6 = Dependency(successes, mode='all', success_only=False) # yes after / no follow
-
-client.block = False
-
-r1a = client.apply(getpid, after=d1a)
-r1b = client.apply(getpid, follow=d1b)
-r2a = client.apply(getpid, after=d2b, follow=d2a)
-r2b = client.apply(getpid, after=d2a, follow=d2b)
-r3 = client.apply(getpid, after=d3)
-r4a = client.apply(getpid, after=d4)
-r4b = client.apply(getpid, follow=d4)
-r4c = client.apply(getpid, after=d3, follow=d4)
-r5 = client.apply(getpid, after=d5)
-r5b = client.apply(getpid, follow=d5, after=d3)
-r6 = client.apply(getpid, follow=d6)
-r6b = client.apply(getpid, after=d6, follow=d2b)
+d1a = Dependency(mixed, all=False, failure=True) # yes
+d1b = Dependency(mixed, all=False) # yes
+d2a = Dependency(mixed, all=True, failure=True) # yes after / no follow
+d2b = Dependency(mixed, all=True) # no
+d3 = Dependency(failures, all=False) # no
+d4 = Dependency(failures, all=False, failure=True) # yes
+d5 = Dependency(failures, all=True, failure=True) # yes after / no follow
+d6 = Dependency(successes, all=True, failure=True) # yes after / no follow
+
+view.block = False
+flags = view.temp_flags
+with flags(after=d1a):
+    r1a = view.apply(getpid)
+with flags(follow=d1b):
+    r1b = view.apply(getpid)
+with flags(after=d2b, follow=d2a):
+    r2a = view.apply(getpid)
+with flags(after=d2a, follow=d2b):
+    r2b = view.apply(getpid)
+with flags(after=d3):
+    r3 = view.apply(getpid)
+with flags(after=d4):
+    r4a = view.apply(getpid)
+with flags(follow=d4):
+    r4b = view.apply(getpid)
+with flags(after=d3, follow=d4):
+    r4c = view.apply(getpid)
+with flags(after=d5):
+    r5 = view.apply(getpid)
+with flags(follow=d5, after=d3):
+    r5b = view.apply(getpid)
+with flags(follow=d6):
+    r6 = view.apply(getpid)
+with flags(after=d6, follow=d2b):
+    r6b = view.apply(getpid)
 
 def should_fail(f):
     try:
         f()
     except error.KernelError:
         pass
     else:
         print 'should have raised'
         # raise Exception("should have raised")
 
 # print r1a.msg_ids
 r1a.get()
 # print r1b.msg_ids
 r1b.get()
 # print r2a.msg_ids
 should_fail(r2a.get)
 # print r2b.msg_ids
 should_fail(r2b.get)
 # print r3.msg_ids
 should_fail(r3.get)
 # print r4a.msg_ids
 r4a.get()
 # print r4b.msg_ids
 r4b.get()
 # print r4c.msg_ids
 should_fail(r4c.get)
 # print r5.msg_ids
 r5.get()
 # print r5b.msg_ids
 should_fail(r5b.get)
 # print r6.msg_ids
 should_fail(r6.get) # assuming > 1 engine
 # print r6b.msg_ids
 should_fail(r6b.get)
 print 'done'

docs/examples/newparallel/demo/map.py

@@ -1,36 +1,37 b''
 from IPython.zmq.parallel.client import *
 
 client = Client()
+view = client[:]
 
-@remote(client, block=True)
[e-mail](block=True)
 def square(a):
     """return square of a number"""
     return a*a
 
 squares = map(square, range(42))
 
 # but that blocked between each result; not exactly useful
 
 square.block = False
 
 arlist = map(square, range(42))
 # submitted very fast
 
 # wait for the results:
 squares2 = [ r.get() for r in arlist ]
 
 # now the more convenient @parallel decorator, which has a map method:
 
-@parallel(client, block=False)
[e-mail](block=False)
 def psquare(a):
     """return square of a number"""
     return a*a
 
 # this chunks the data into n-negines jobs, not 42 jobs:
 ar = psquare.map(range(42))
 
 # wait for the results to be done:
 squares3 = ar.get()
 
 print squares == squares2, squares3==squares
 # True
  No newline at end of file

docs/examples/newparallel/demo/views.py

@@ -1,15 +1,15 b''
 from IPython.zmq.parallel.client import *
 
 client = Client()
 
 for id in client.ids:
-    client.push(dict(ids=id*id), targets=id)
+    client[id].push(dict(ids=id*id))
 
-rns = client[0]
-rns['a'] = 5
+v = client[0]
+v['a'] = 5
 
-print rns['a']
+print v['a']
 
 remotes = client[:]
 
 print remotes['ids']
  No newline at end of file

docs/examples/newparallel/mcdriver.py

@@ -1,144 +1,144 b''
 #!/usr/bin/env python
 """Run a Monte-Carlo options pricer in parallel."""
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import sys
 import time
 from IPython.zmq.parallel import client
 import numpy as np
 from mcpricer import price_options
 from matplotlib import pyplot as plt
 
 #-----------------------------------------------------------------------------
 # Setup parameters for the run
 #-----------------------------------------------------------------------------
 
 def ask_question(text, the_type, default):
     s = '%s [%r]: ' % (text, the_type(default))
     result = raw_input(s)
     if result:
         return the_type(result)
     else:
         return the_type(default)
 
 cluster_profile = ask_question("Cluster profile", str, "default")
 price = ask_question("Initial price", float, 100.0)
 rate = ask_question("Interest rate", float, 0.05)
 days = ask_question("Days to expiration", int, 260)
 paths = ask_question("Number of MC paths", int, 10000)
 n_strikes = ask_question("Number of strike values", int, 5)
 min_strike = ask_question("Min strike price", float, 90.0)
 max_strike = ask_question("Max strike price", float, 110.0)
 n_sigmas = ask_question("Number of volatility values", int, 5)
 min_sigma = ask_question("Min volatility", float, 0.1)
 max_sigma = ask_question("Max volatility", float, 0.4)
 
 strike_vals = np.linspace(min_strike, max_strike, n_strikes)
 sigma_vals = np.linspace(min_sigma, max_sigma, n_sigmas)
 
 #-----------------------------------------------------------------------------
 # Setup for parallel calculation
 #-----------------------------------------------------------------------------
 
 # The Client is used to setup the calculation and works with all
 # engines.
 c = client.Client(profile=cluster_profile)
 
 # A LoadBalancedView is an interface to the engines that provides dynamic load 
 # balancing at the expense of not knowing which engine will execute the code.
-view = c.view()
+view = c.load_balanced_view()
 
 # Initialize the common code on the engines. This Python module has the
 # price_options function that prices the options.
 
 #-----------------------------------------------------------------------------
 # Perform parallel calculation
 #-----------------------------------------------------------------------------
 
 print "Running parallel calculation over strike prices and volatilities..."
 print "Strike prices: ", strike_vals
 print "Volatilities: ", sigma_vals
 sys.stdout.flush()
 
 # Submit tasks to the TaskClient for each (strike, sigma) pair as a MapTask.
 t1 = time.time()
 async_results = []
 for strike in strike_vals:
     for sigma in sigma_vals:
         ar = view.apply_async(price_options, price, strike, sigma, rate, days, paths)
         async_results.append(ar)
 
 print "Submitted tasks: ", len(async_results)
 sys.stdout.flush()
 
 # Block until all tasks are completed.
-c.barrier(async_results)
+c.wait(async_results)
 t2 = time.time()
 t = t2-t1
 
 print "Parallel calculation completed, time = %s s" % t
 print "Collecting results..."
 
 # Get the results using TaskClient.get_task_result.
 results = [ar.get() for ar in async_results]
 
 # Assemble the result into a structured NumPy array.
 prices = np.empty(n_strikes*n_sigmas,
     dtype=[('ecall',float),('eput',float),('acall',float),('aput',float)]
 )
 
 for i, price in enumerate(results):
     prices[i] = tuple(price)
     
 prices.shape = (n_strikes, n_sigmas)
 strike_mesh, sigma_mesh = np.meshgrid(strike_vals, sigma_vals)
 
 print "Results are available: strike_mesh, sigma_mesh, prices"
 print "To plot results type 'plot_options(sigma_mesh, strike_mesh, prices)'"
 
 #-----------------------------------------------------------------------------
 # Utilities
 #-----------------------------------------------------------------------------
 
 def plot_options(sigma_mesh, strike_mesh, prices):
     """
     Make a contour plot of the option price in (sigma, strike) space.
     """
     plt.figure(1)
     
     plt.subplot(221)
     plt.contourf(sigma_mesh, strike_mesh, prices['ecall'])
     plt.axis('tight')
     plt.colorbar()
     plt.title('European Call')
     plt.ylabel("Strike Price")
 
     plt.subplot(222)
     plt.contourf(sigma_mesh, strike_mesh, prices['acall'])
     plt.axis('tight')
     plt.colorbar()
     plt.title("Asian Call")
 
     plt.subplot(223)
     plt.contourf(sigma_mesh, strike_mesh, prices['eput'])
     plt.axis('tight')
     plt.colorbar()
     plt.title("European Put")
     plt.xlabel("Volatility")
     plt.ylabel("Strike Price")
 
     plt.subplot(224)
     plt.contourf(sigma_mesh, strike_mesh, prices['aput'])
     plt.axis('tight')
     plt.colorbar()
     plt.title("Asian Put")
     plt.xlabel("Volatility")
 
 
 
     
 
 

docs/examples/newparallel/parallelpi.py

@@ -1,64 +1,64 b''
 """Calculate statistics on the digits of pi in parallel.
 
 This program uses the functions in :file:`pidigits.py` to calculate
 the frequencies of 2 digit sequences in the digits of pi. The
 results are plotted using matplotlib.
 
 To run, text files from http://www.super-computing.org/
 must be installed in the working directory of the IPython engines.
 The actual filenames to be used can be set with the ``filestring``
 variable below.
 
 The dataset we have been using for this is the 200 million digit one here:
 ftp://pi.super-computing.org/.2/pi200m/
 
 and the files used will be downloaded if they are not in the working directory 
 of the IPython engines.
 """ 
 
 from IPython.zmq.parallel import client
 from matplotlib import pyplot as plt
 import numpy as np
 from pidigits import *
 from timeit import default_timer as clock
 
 # Files with digits of pi (10m digits each)
 filestring = 'pi200m.ascii.%(i)02dof20'
 files = [filestring % {'i':i} for i in range(1,16)]
 
 # Connect to the IPython cluster
-c = client.Client(profile='edison')
-c.run('pidigits.py')
+c = client.Client()
+c[:].run('pidigits.py')
 
 # the number of engines
 n = len(c)
 id0 = c.ids[0]
 v = c[:]
-v.set_flags(bound=True,block=True)
+v.block=True
 # fetch the pi-files
 print "downloading %i files of pi"%n
 v.map(fetch_pi_file, files[:n])
 print "done"
 
 # Run 10m digits on 1 engine
 t1 = clock()
 freqs10m = c[id0].apply_sync(compute_two_digit_freqs, files[0])
 t2 = clock()
 digits_per_second1 = 10.0e6/(t2-t1)
 print "Digits per second (1 core, 10m digits):   ", digits_per_second1
 
 
 # Run n*10m digits on all engines
 t1 = clock()
 freqs_all = v.map(compute_two_digit_freqs, files[:n])
 freqs150m = reduce_freqs(freqs_all)
 t2 = clock()
 digits_per_second8 = n*10.0e6/(t2-t1)
 print "Digits per second (%i engines, %i0m digits): "%(n,n), digits_per_second8
 
 print "Speedup: ", digits_per_second8/digits_per_second1
 
 plot_two_digit_freqs(freqs150m)
 plt.title("2 digit sequences in %i0m digits of pi"%n)
 plt.show()
 

docs/examples/newparallel/wave2D/parallelwave-mpi.py

@@ -1,203 +1,205 b''
 #!/usr/bin/env python
 """
 A simple python program of solving a 2D wave equation in parallel.
 Domain partitioning and inter-processor communication
 are done by an object of class MPIRectPartitioner2D
 (which is a subclass of RectPartitioner2D and uses MPI via mpi4py)
 
 An example of running the program is (8 processors, 4x2 partition,
 400x100 grid cells)::
 
    $ ipclusterz start --profile mpi -n 8 # start 8 engines (assuming mpi profile has been configured)
    $ ./parallelwave-mpi.py --grid 400 100 --partition 4 2 --profile mpi
 
 See also parallelwave-mpi, which runs the same program, but uses MPI 
 (via mpi4py) for the inter-engine communication.
 
 Authors
 -------
 
  * Xing Cai
  * Min Ragan-Kelley
 
 """
 
 import sys
 import time
 
 from numpy import exp, zeros, newaxis, sqrt
 
 from IPython.external import argparse
 from IPython.zmq.parallel.client import Client, Reference
 
-def setup_partitioner(ns, index, num_procs, gnum_cells, parts):
+def setup_partitioner(index, num_procs, gnum_cells, parts):
     """create a partitioner in the engine namespace"""
+    global partitioner
     p = MPIRectPartitioner2D(my_id=index, num_procs=num_procs)
     p.redim(global_num_cells=gnum_cells, num_parts=parts)
     p.prepare_communication()
     # put the partitioner into the global namespace:
-    ns.partitioner=p
+    partitioner=p
 
-def setup_solver(ns, *args, **kwargs):
+def setup_solver(*args, **kwargs):
     """create a WaveSolver in the engine namespace"""
-    ns.solver = WaveSolver(*args, **kwargs)
+    global solver
+    solver = WaveSolver(*args, **kwargs)
 
 def wave_saver(u, x, y, t):
     """save the wave log"""
     global u_hist
     global t_hist
     t_hist.append(t)
     u_hist.append(1.0*u)
     
 
 # main program:
 if __name__ == '__main__':
     
     parser = argparse.ArgumentParser()
     paa = parser.add_argument
     paa('--grid', '-g',
         type=int, nargs=2, default=[100,100], dest='grid',
         help="Cells in the grid, e.g. --grid 100 200")
     paa('--partition', '-p',
         type=int, nargs=2, default=None,
         help="Process partition grid, e.g. --partition 4 2 for 4x2")
     paa('-c',
         type=float, default=1.,
         help="Wave speed (I think)")
     paa('-Ly',
         type=float, default=1.,
         help="system size (in y)")
     paa('-Lx',
         type=float, default=1.,
         help="system size (in x)")
     paa('-t', '--tstop',
         type=float, default=1.,
         help="Time units to run")
     paa('--profile', 
         type=unicode, default=u'default',
         help="Specify the ipcluster profile for the client to connect to.")
     paa('--save', 
         action='store_true',
         help="Add this flag to save the time/wave history during the run.")
     paa('--scalar', 
         action='store_true',
         help="Also run with scalar interior implementation, to see vector speedup.")
     
     ns = parser.parse_args()
     # set up arguments
     grid = ns.grid
     partition = ns.partition
     Lx = ns.Lx
     Ly = ns.Ly
     c = ns.c
     tstop = ns.tstop
     if ns.save:
         user_action = wave_saver
     else:
         user_action = None
     
     num_cells = 1.0*(grid[0]-1)*(grid[1]-1)
     final_test = True
     
     # create the Client
     rc = Client(profile=ns.profile)
     num_procs = len(rc.ids)
     
     if partition is None:
         partition = [1,num_procs]
     
     assert partition[0]*partition[1] == num_procs, "can't map partition %s to %i engines"%(partition, num_procs)
     
     view = rc[:]
     print "Running %s system on %s processes until %f"%(grid, partition, tstop)
     
     # functions defining initial/boundary/source conditions
     def I(x,y):
         from numpy import exp
         return 1.5*exp(-100*((x-0.5)**2+(y-0.5)**2))
     def f(x,y,t):
         return 0.0
         # from numpy import exp,sin
         # return 10*exp(-(x - sin(100*t))**2)
     def bc(x,y,t):
         return 0.0
     
     # initial imports, setup rank
     view.execute('\n'.join([
     "from mpi4py import MPI",
     "import numpy",
     "mpi = MPI.COMM_WORLD",
     "my_id = MPI.COMM_WORLD.Get_rank()"]), block=True)
     
     # initialize t_hist/u_hist for saving the state at each step (optional)
     view['t_hist'] = []
     view['u_hist'] = []
     
     # set vector/scalar implementation details
     impl = {}
     impl['ic'] = 'vectorized'
     impl['inner'] = 'scalar'
     impl['bc'] = 'vectorized'
     
     # execute some files so that the classes we need will be defined on the engines:
     view.run('RectPartitioner.py')
     view.run('wavesolver.py')
     
     # setup remote partitioner
     # note that Reference means that the argument passed to setup_partitioner will be the
     # object named 'my_id' in the engine's namespace
-    view.apply_sync_bound(setup_partitioner, Reference('my_id'), num_procs, grid, partition)
+    view.apply_sync(setup_partitioner, Reference('my_id'), num_procs, grid, partition)
     # wait for initial communication to complete
     view.execute('mpi.barrier()')
     # setup remote solvers
-    view.apply_sync_bound(setup_solver, I,f,c,bc,Lx,Ly,partitioner=Reference('partitioner'), dt=0,implementation=impl)
+    view.apply_sync(setup_solver, I,f,c,bc,Lx,Ly,partitioner=Reference('partitioner'), dt=0,implementation=impl)
 
     # lambda for calling solver.solve:
     _solve = lambda *args, **kwargs: solver.solve(*args, **kwargs)
     
     if ns.scalar:
         impl['inner'] = 'scalar'
         # run first with element-wise Python operations for each cell
         t0 = time.time()
         ar = view.apply_async(_solve, tstop, dt=0, verbose=True, final_test=final_test, user_action=user_action)
         if final_test:
             # this sum is performed element-wise as results finish
             s = sum(ar)
             # the L2 norm (RMS) of the result:
             norm = sqrt(s/num_cells)
         else:
             norm = -1
         t1 = time.time()
         print 'scalar inner-version, Wtime=%g, norm=%g'%(t1-t0, norm)
     
     impl['inner'] = 'vectorized'
     # setup new solvers
-    view.apply_sync_bound(setup_solver, I,f,c,bc,Lx,Ly,partitioner=Reference('partitioner'), dt=0,implementation=impl)
+    view.apply_sync(setup_solver, I,f,c,bc,Lx,Ly,partitioner=Reference('partitioner'), dt=0,implementation=impl)
     view.execute('mpi.barrier()')
     
     # run again with numpy vectorized inner-implementation
     t0 = time.time()
     ar = view.apply_async(_solve, tstop, dt=0, verbose=True, final_test=final_test)#, user_action=wave_saver)
     if final_test:
         # this sum is performed element-wise as results finish
         s = sum(ar)
         # the L2 norm (RMS) of the result:
         norm = sqrt(s/num_cells)
     else:
         norm = -1
     t1 = time.time()
     print 'vector inner-version, Wtime=%g, norm=%g'%(t1-t0, norm)
     
     # if ns.save is True, then u_hist stores the history of u as a list
     # If the partion scheme is Nx1, then u can be reconstructed via 'gather':
     if ns.save and partition[-1] == 1:
         import pylab
         view.execute('u_last=u_hist[-1]')
         # map mpi IDs to IPython IDs, which may not match
         ranks = view['my_id']
         targets = range(len(ranks))
         for idx in range(len(ranks)):
             targets[idx] = ranks.index(idx)
         u_last = rc[targets].gather('u_last', block=True)
         pylab.pcolor(u_last)
         pylab.show()

docs/examples/newparallel/wave2D/parallelwave.py

@@ -1,207 +1,209 b''
 #!/usr/bin/env python
 """
 A simple python program of solving a 2D wave equation in parallel.
 Domain partitioning and inter-processor communication
 are done by an object of class ZMQRectPartitioner2D
 (which is a subclass of RectPartitioner2D and uses 0MQ via pyzmq)
 
 An example of running the program is (8 processors, 4x2 partition,
 200x200 grid cells)::
 
    $ ipclusterz start -n 8 # start 8 engines
    $ ./parallelwave.py --grid 200 200 --partition 4 2
 
 See also parallelwave-mpi, which runs the same program, but uses MPI 
 (via mpi4py) for the inter-engine communication.
 
 Authors
 -------
 
  * Xing Cai
  * Min Ragan-Kelley
 
 """
 #
 import sys
 import time
 
 from numpy import exp, zeros, newaxis, sqrt
 
 from IPython.external import argparse
 from IPython.zmq.parallel.client import Client, Reference
 
-def setup_partitioner(ns, comm, addrs, index, num_procs, gnum_cells, parts):
+def setup_partitioner(comm, addrs, index, num_procs, gnum_cells, parts):
     """create a partitioner in the engine namespace"""
+    global partitioner
     p = ZMQRectPartitioner2D(comm, addrs, my_id=index, num_procs=num_procs)
     p.redim(global_num_cells=gnum_cells, num_parts=parts)
     p.prepare_communication()
     # put the partitioner into the global namespace:
-    ns.partitioner=p
+    partitioner=p
 
-def setup_solver(ns, *args, **kwargs):
+def setup_solver(*args, **kwargs):
     """create a WaveSolver in the engine namespace."""
-    ns.solver = WaveSolver(*args, **kwargs)
+    global solver
+    solver = WaveSolver(*args, **kwargs)
 
 def wave_saver(u, x, y, t):
     """save the wave state for each timestep."""
     global u_hist
     global t_hist
     t_hist.append(t)
     u_hist.append(1.0*u)
     
 
 # main program:
 if __name__ == '__main__':
     
     parser = argparse.ArgumentParser()
     paa = parser.add_argument
     paa('--grid', '-g',
         type=int, nargs=2, default=[100,100], dest='grid',
         help="Cells in the grid, e.g. --grid 100 200")
     paa('--partition', '-p',
         type=int, nargs=2, default=None,
         help="Process partition grid, e.g. --partition 4 2 for 4x2")
     paa('-c',
         type=float, default=1.,
         help="Wave speed (I think)")
     paa('-Ly',
         type=float, default=1.,
         help="system size (in y)")
     paa('-Lx',
         type=float, default=1.,
         help="system size (in x)")
     paa('-t', '--tstop',
         type=float, default=1.,
         help="Time units to run")
     paa('--profile', 
         type=unicode, default=u'default',
         help="Specify the ipcluster profile for the client to connect to.")
     paa('--save', 
         action='store_true',
         help="Add this flag to save the time/wave history during the run.")
     paa('--scalar', 
         action='store_true',
         help="Also run with scalar interior implementation, to see vector speedup.")
         
     ns = parser.parse_args()
     # set up arguments
     grid = ns.grid
     partition = ns.partition
     Lx = ns.Lx
     Ly = ns.Ly
     c = ns.c
     tstop = ns.tstop
     if ns.save:
         user_action = wave_saver
     else:
         user_action = None
     
     num_cells = 1.0*(grid[0]-1)*(grid[1]-1)
     final_test = True
     
     # create the Client
     rc = Client(profile=ns.profile)
     num_procs = len(rc.ids)
     
     if partition is None:
         partition = [num_procs,1]
     else:
         num_procs = min(num_procs, partition[0]*partition[1])
     
     assert partition[0]*partition[1] == num_procs, "can't map partition %s to %i engines"%(partition, num_procs)
     
     # construct the View:
     view = rc[:num_procs]
     print "Running %s system on %s processes until %f"%(grid, partition, tstop)
     
     # functions defining initial/boundary/source conditions
     def I(x,y):
         from numpy import exp
         return 1.5*exp(-100*((x-0.5)**2+(y-0.5)**2))
     def f(x,y,t):
         return 0.0
         # from numpy import exp,sin
         # return 10*exp(-(x - sin(100*t))**2)
     def bc(x,y,t):
         return 0.0
     
     # initialize t_hist/u_hist for saving the state at each step (optional)
     view['t_hist'] = []
     view['u_hist'] = []
 
     # set vector/scalar implementation details
     impl = {}
     impl['ic'] = 'vectorized'
     impl['inner'] = 'scalar'
     impl['bc'] = 'vectorized'
     
     # execute some files so that the classes we need will be defined on the engines:
     view.execute('import numpy')
     view.run('communicator.py')
     view.run('RectPartitioner.py')
     view.run('wavesolver.py')
     
     # scatter engine IDs
     view.scatter('my_id', range(num_procs), flatten=True)
     
     # create the engine connectors
     view.execute('com = EngineCommunicator()')
 
     # gather the connection information into a single dict
     ar = view.apply_async(lambda : com.info)
     peers = ar.get_dict()
     # print peers
     # this is a dict, keyed by engine ID, of the connection info for the EngineCommunicators
     
     # setup remote partitioner
     # note that Reference means that the argument passed to setup_partitioner will be the
     # object named 'com' in the engine's namespace
-    view.apply_sync_bound(setup_partitioner, Reference('com'), peers, Reference('my_id'), num_procs, grid, partition)
+    view.apply_sync(setup_partitioner, Reference('com'), peers, Reference('my_id'), num_procs, grid, partition)
     time.sleep(1)
     # convenience lambda to call solver.solve:
     _solve = lambda *args, **kwargs: solver.solve(*args, **kwargs)
 
     if ns.scalar:
         impl['inner'] = 'scalar'
         # setup remote solvers
-        view.apply_sync_bound(setup_solver, I,f,c,bc,Lx,Ly, partitioner=Reference('partitioner'), dt=0,implementation=impl)
+        view.apply_sync(setup_solver, I,f,c,bc,Lx,Ly, partitioner=Reference('partitioner'), dt=0,implementation=impl)
 
         # run first with element-wise Python operations for each cell
         t0 = time.time()
         ar = view.apply_async(_solve, tstop, dt=0, verbose=True, final_test=final_test, user_action=user_action)
         if final_test:
             # this sum is performed element-wise as results finish
             s = sum(ar)
             # the L2 norm (RMS) of the result:
             norm = sqrt(s/num_cells)
         else:
             norm = -1
         t1 = time.time()
         print 'scalar inner-version, Wtime=%g, norm=%g'%(t1-t0, norm)
     
     # run again with faster numpy-vectorized inner implementation:
     impl['inner'] = 'vectorized'
     # setup remote solvers
-    view.apply_sync_bound(setup_solver, I,f,c,bc,Lx,Ly,partitioner=Reference('partitioner'), dt=0,implementation=impl)
+    view.apply_sync(setup_solver, I,f,c,bc,Lx,Ly,partitioner=Reference('partitioner'), dt=0,implementation=impl)
 
     t0 = time.time()
     
     ar = view.apply_async(_solve, tstop, dt=0, verbose=True, final_test=final_test)#, user_action=wave_saver)
     if final_test:
         # this sum is performed element-wise as results finish
         s = sum(ar)
         # the L2 norm (RMS) of the result:
         norm = sqrt(s/num_cells)
     else:
         norm = -1
     t1 = time.time()
     print 'vector inner-version, Wtime=%g, norm=%g'%(t1-t0, norm)
     
     # if ns.save is True, then u_hist stores the history of u as a list
     # If the partion scheme is Nx1, then u can be reconstructed via 'gather':
     if ns.save and partition[-1] == 1:
         import pylab
         view.execute('u_last=u_hist[-1]')
         u_last = view.gather('u_last', block=True)
         pylab.pcolor(u_last)
         pylab.show()
  No newline at end of file

docs/source/index.txt

@@ -1,34 +1,34 b''
 =====================
 IPython Documentation
 =====================
 
 .. htmlonly::
 
    :Release: |release|
    :Date: |today|
 
 Welcome to the official IPython documentation.
 
 Contents
 ========
 
 .. toctree::
    :maxdepth: 1
 
    overview.txt
    whatsnew/index.txt
    install/index.txt
    interactive/index.txt
-   parallel/index.txt
+   .. parallel/index.txt
    parallelz/index.txt
    config/index.txt
    development/index.txt
    api/index.txt
    faq.txt
    about/index.txt
 
 .. htmlonly::
    * :ref:`genindex`
    * :ref:`modindex`
    * :ref:`search`
 

docs/source/parallelz/dag_dependencies.txt

@@ -1,172 +1,173 b''
 .. _dag_dependencies:
 
 ================
 DAG Dependencies
 ================
 
 Often, parallel workflow is described in terms of a `Directed Acyclic Graph
 <http://en.wikipedia.org/wiki/Directed_acyclic_graph>`_ or DAG.  A popular library
 for working with Graphs is NetworkX_.  Here, we will walk through a demo mapping
 a nx DAG to task dependencies.
 
 The full script that runs this demo can be found in
 :file:`docs/examples/newparallel/dagdeps.py`.
 
 Why are DAGs good for task dependencies?
 ----------------------------------------
 
 The 'G' in DAG is 'Graph'. A Graph is a collection of **nodes** and **edges** that connect
 the nodes. For our purposes, each node would be a task, and each edge would be a
 dependency. The 'D' in DAG stands for 'Directed'. This means that each edge has a
 direction associated with it. So we can interpret the edge (a,b) as meaning that b depends
 on a, whereas the edge (b,a) would mean a depends on b. The 'A' is 'Acyclic', meaning that
 there must not be any closed loops in the graph. This is important for dependencies,
 because if a loop were closed, then a task could ultimately depend on itself, and never be
 able to run. If your workflow can be described as a DAG, then it is impossible for your
 dependencies to cause a deadlock.
 
 A Sample DAG
 ------------
 
 Here, we have a very simple 5-node DAG:
 
 .. figure:: simpledag.*
 
 With NetworkX, an arrow is just a fattened bit on the edge. Here, we can see that task 0
 depends on nothing, and can run immediately. 1 and 2 depend on 0; 3 depends on
 1 and 2; and 4 depends only on 1.
 
 A possible sequence of events for this workflow:
 
 0. Task 0 can run right away
 1. 0 finishes, so 1,2 can start
 2. 1 finishes, 3 is still waiting on 2, but 4 can start right away
 3. 2 finishes, and 3 can finally start
 
 
 Further, taking failures into account, assuming all dependencies are run with the default
-`success_only=True`, the following cases would occur for each node's failure:
+`success=True,failure=False`, the following cases would occur for each node's failure:
 
 0. fails: all other tasks fail as Impossible
 1. 2 can still succeed, but 3,4 are unreachable
 2. 3 becomes unreachable, but 4 is unaffected
 3. and 4. are terminal, and can have no effect on other nodes
 
 The code to generate the simple DAG:
 
 .. sourcecode:: python
 
     import networkx as nx
     
     G = nx.DiGraph()
     
     # add 5 nodes, labeled 0-4:
     map(G.add_node, range(5))
     # 1,2 depend on 0:
     G.add_edge(0,1)
     G.add_edge(0,2)
     # 3 depends on 1,2
     G.add_edge(1,3)
     G.add_edge(2,3)
     # 4 depends on 1
     G.add_edge(1,4)
     
     # now draw the graph:
     pos = { 0 : (0,0), 1 : (1,1), 2 : (-1,1),
             3 : (0,2), 4 : (2,2)}
     nx.draw(G, pos, edge_color='r')
 
 
 For demonstration purposes, we have a function that generates a random DAG with a given
 number of nodes and edges.
 
 .. literalinclude:: ../../examples/newparallel/dagdeps.py
     :language: python
     :lines: 20-36
 
 So first, we start with a graph of 32 nodes, with 128 edges:
 
 .. sourcecode:: ipython
 
     In [2]: G = random_dag(32,128)
 
 Now, we need to build our dict of jobs corresponding to the nodes on the graph:
 
 .. sourcecode:: ipython
 
     In [3]: jobs = {}
     
     # in reality, each job would presumably be different
     # randomwait is just a function that sleeps for a random interval
     In [4]: for node in G:
        ...:     jobs[node] = randomwait 
 
 Once we have a dict of jobs matching the nodes on the graph, we can start submitting jobs,
 and linking up the dependencies. Since we don't know a job's msg_id until it is submitted,
 which is necessary for building dependencies, it is critical that we don't submit any jobs
 before other jobs it may depend on. Fortunately, NetworkX provides a
 :meth:`topological_sort` method which ensures exactly this. It presents an iterable, that
 guarantees that when you arrive at a node, you have already visited all the nodes it
 on which it depends:
 
 .. sourcecode:: ipython
 
-    In [5]: c = client.Client()
+    In [5]: rc = client.Client()
+    In [5]: view = rc.load_balanced_view()
     
     In [6]: results = {}
     
     In [7]: for node in G.topological_sort():
        ...:     # get list of AsyncResult objects from nodes
        ...:     # leading into this one as dependencies
        ...:     deps = [ results[n] for n in G.predecessors(node) ]
        ...:     # submit and store AsyncResult object
-       ...:     results[node] = client.apply(jobs[node], after=deps, block=False)
+       ...:     results[node] = view.apply_with_flags(jobs[node], after=deps, block=False)
 
 Now that we have submitted all the jobs, we can wait for the results:
 
 .. sourcecode:: ipython
 
-    In [8]: [ r.get() for r in results.values() ]
+    In [8]: view.wait(results.values())
 
 Now, at least we know that all the jobs ran and did not fail (``r.get()`` would have
 raised an error if a task failed).  But we don't know that the ordering was properly
 respected.  For this, we can use the :attr:`metadata` attribute of each AsyncResult.
 
 These objects store a variety of metadata about each task, including various timestamps.
 We can validate that the dependencies were respected by checking that each task was
 started after all of its predecessors were completed:
 
 .. literalinclude:: ../../examples/newparallel/dagdeps.py
     :language: python
     :lines: 64-70
 
 We can also validate the graph visually. By drawing the graph with each node's x-position
 as its start time, all arrows must be pointing to the right if dependencies were respected.
 For spreading, the y-position will be the runtime of the task, so long tasks
 will be at the top, and quick, small tasks will be at the bottom.
 
 .. sourcecode:: ipython
 
     In [10]: from matplotlib.dates import date2num
     
     In [11]: from matplotlib.cm import gist_rainbow
     
     In [12]: pos = {}; colors = {}
     
     In [12]: for node in G:
         ...:    md = results[node].metadata
         ...:    start = date2num(md.started)
         ...:    runtime = date2num(md.completed) - start
         ...:    pos[node] = (start, runtime)
         ...:    colors[node] = md.engine_id
     
     In [13]: nx.draw(G, pos, node_list=colors.keys(), node_color=colors.values(),
         ...:    cmap=gist_rainbow)
 
 .. figure:: dagdeps.*
 
     Time started on x, runtime on y, and color-coded by engine-id (in this case there
     were four engines). Edges denote dependencies.
 
 
 .. _NetworkX: http://networkx.lanl.gov/

docs/source/parallelz/index.txt

@@ -1,21 +1,22 b''
 .. _parallelz_index:
 
 ==========================================
 Using IPython for parallel computing (ZMQ)
 ==========================================
 
 .. toctree::
    :maxdepth: 2
 
    parallel_intro.txt
    parallel_process.txt
    parallel_multiengine.txt
    parallel_task.txt
    parallel_mpi.txt
    parallel_security.txt
    parallel_winhpc.txt
    parallel_demos.txt
    dag_dependencies.txt
    parallel_details.txt
+   parallel_transition.txt
 
 

docs/source/parallelz/parallel_demos.txt

@@ -1,284 +1,284 b''
 =================
 Parallel examples
 =================
 
 .. note::
 
     Performance numbers from ``IPython.kernel``, not newparallel.
 
 In this section we describe two more involved examples of using an IPython
 cluster to perform a parallel computation. In these examples, we will be using
 IPython's "pylab" mode, which enables interactive plotting using the
 Matplotlib package. IPython can be started in this mode by typing::
 
     ipython --pylab
 
 at the system command line.
 
 150 million digits of pi
 ========================
 
 In this example we would like to study the distribution of digits in the
 number pi (in base 10). While it is not known if pi is a normal number (a
 number is normal in base 10 if 0-9 occur with equal likelihood) numerical
 investigations suggest that it is. We will begin with a serial calculation on
 10,000 digits of pi and then perform a parallel calculation involving 150
 million digits.
 
 In both the serial and parallel calculation we will be using functions defined
 in the :file:`pidigits.py` file, which is available in the
 :file:`docs/examples/newparallel` directory of the IPython source distribution.
 These functions provide basic facilities for working with the digits of pi and
 can be loaded into IPython by putting :file:`pidigits.py` in your current
 working directory and then doing:
 
 .. sourcecode:: ipython
 
     In [1]: run pidigits.py
 
 Serial calculation
 ------------------
 
 For the serial calculation, we will use `SymPy <http://www.sympy.org>`_ to
 calculate 10,000 digits of pi and then look at the frequencies of the digits
 0-9. Out of 10,000 digits, we expect each digit to occur 1,000 times. While
 SymPy is capable of calculating many more digits of pi, our purpose here is to
 set the stage for the much larger parallel calculation.
 
 In this example, we use two functions from :file:`pidigits.py`: 
 :func:`one_digit_freqs` (which calculates how many times each digit occurs)
 and :func:`plot_one_digit_freqs` (which uses Matplotlib to plot the result).
 Here is an interactive IPython session that uses these functions with
 SymPy:
 
 .. sourcecode:: ipython
 
     In [7]: import sympy
 
     In [8]: pi = sympy.pi.evalf(40)
 
     In [9]: pi
     Out[9]: 3.141592653589793238462643383279502884197
 
     In [10]: pi = sympy.pi.evalf(10000)
 
     In [11]: digits = (d for d in str(pi)[2:])  # create a sequence of digits
 
     In [12]: run pidigits.py  # load one_digit_freqs/plot_one_digit_freqs
 
     In [13]: freqs = one_digit_freqs(digits)
 
     In [14]: plot_one_digit_freqs(freqs)
     Out[14]: [<matplotlib.lines.Line2D object at 0x18a55290>]
 
 The resulting plot of the single digit counts shows that each digit occurs
 approximately 1,000 times, but that with only 10,000 digits the
 statistical fluctuations are still rather large:
 
 .. image:: ../parallel/single_digits.*
 
 It is clear that to reduce the relative fluctuations in the counts, we need
 to look at many more digits of pi. That brings us to the parallel calculation.
 
 Parallel calculation
 --------------------
 
 Calculating many digits of pi is a challenging computational problem in itself. 
 Because we want to focus on the distribution of digits in this example, we
 will use pre-computed digit of pi from the website of Professor Yasumasa
 Kanada at the University of Tokyo (http://www.super-computing.org). These
 digits come in a set of text files (ftp://pi.super-computing.org/.2/pi200m/)
 that each have 10 million digits of pi.
 
 For the parallel calculation, we have copied these files to the local hard
 drives of the compute nodes. A total of 15 of these files will be used, for a
 total of 150 million digits of pi. To make things a little more interesting we
 will calculate the frequencies of all 2 digits sequences (00-99) and then plot
 the result using a 2D matrix in Matplotlib.
 
 The overall idea of the calculation is simple: each IPython engine will
 compute the two digit counts for the digits in a single file. Then in a final
 step the counts from each engine will be added up. To perform this
 calculation, we will need two top-level functions from :file:`pidigits.py`:
 
 .. literalinclude:: ../../examples/newparallel/pidigits.py
    :language: python
    :lines: 41-56
 
 We will also use the :func:`plot_two_digit_freqs` function to plot the
 results. The code to run this calculation in parallel is contained in
 :file:`docs/examples/newparallel/parallelpi.py`. This code can be run in parallel
 using IPython by following these steps:
 
 1. Use :command:`ipclusterz` to start 15 engines. We used an 8 core (2 quad 
    core CPUs) cluster with hyperthreading enabled which makes the 8 cores 
    looks like 16 (1 controller + 15 engines) in the OS. However, the maximum
    speedup we can observe is still only 8x.
 2. With the file :file:`parallelpi.py` in your current working directory, open
    up IPython in pylab mode and type ``run parallelpi.py``.  This will download
    the pi files via ftp the first time you run it, if they are not
    present in the Engines' working directory.
 
 When run on our 8 core cluster, we observe a speedup of 7.7x. This is slightly
 less than linear scaling (8x) because the controller is also running on one of
 the cores.
 
 To emphasize the interactive nature of IPython, we now show how the
 calculation can also be run by simply typing the commands from
 :file:`parallelpi.py` interactively into IPython:
 
 .. sourcecode:: ipython
 
     In [1]: from IPython.zmq.parallel import client
 
     # The Client allows us to use the engines interactively.
     # We simply pass Client the name of the cluster profile we
     # are using.
     In [2]: c = client.Client(profile='mycluster')
-    In [3]: view = c.view(balanced=True)
+    In [3]: view = c.load_balanced_view()
 
     In [3]: c.ids
     Out[3]: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14]
 
     In [4]: run pidigits.py
 
     In [5]: filestring = 'pi200m.ascii.%(i)02dof20'
 
     # Create the list of files to process.
     In [6]: files = [filestring % {'i':i} for i in range(1,16)]
 
     In [7]: files
     Out[7]: 
     ['pi200m.ascii.01of20',
      'pi200m.ascii.02of20',
      'pi200m.ascii.03of20',
      'pi200m.ascii.04of20',
      'pi200m.ascii.05of20',
      'pi200m.ascii.06of20',
      'pi200m.ascii.07of20',
      'pi200m.ascii.08of20',
      'pi200m.ascii.09of20',
      'pi200m.ascii.10of20',
      'pi200m.ascii.11of20',
      'pi200m.ascii.12of20',
      'pi200m.ascii.13of20',
      'pi200m.ascii.14of20',
      'pi200m.ascii.15of20']
     
     # download the data files if they don't already exist:
-    In [8]: c.map(fetch_pi_file, files)
+    In [8]: v.map(fetch_pi_file, files)
     
     # This is the parallel calculation using the Client.map method 
     # which applies compute_two_digit_freqs to each file in files in parallel.
-    In [9]: freqs_all = c.map(compute_two_digit_freqs, files)
+    In [9]: freqs_all = v.map(compute_two_digit_freqs, files)
 
     # Add up the frequencies from each engine.
     In [10]: freqs = reduce_freqs(freqs_all)
 
     In [11]: plot_two_digit_freqs(freqs)
     Out[11]: <matplotlib.image.AxesImage object at 0x18beb110>
 
     In [12]: plt.title('2 digit counts of 150m digits of pi')
     Out[12]: <matplotlib.text.Text object at 0x18d1f9b0>
 
 The resulting plot generated by Matplotlib is shown below. The colors indicate
 which two digit sequences are more (red) or less (blue) likely to occur in the
 first 150 million digits of pi. We clearly see that the sequence "41" is 
 most likely and that "06" and "07" are least likely. Further analysis would
 show that the relative size of the statistical fluctuations have decreased
 compared to the 10,000 digit calculation.
 
 .. image:: ../parallel/two_digit_counts.*
 
 
 Parallel options pricing
 ========================
 
 An option is a financial contract that gives the buyer of the contract the
 right to buy (a "call") or sell (a "put") a secondary asset (a stock for
 example) at a particular date in the future (the expiration date) for a
 pre-agreed upon price (the strike price). For this right, the buyer pays the
 seller a premium (the option price). There are a wide variety of flavors of
 options (American, European, Asian, etc.) that are useful for different
 purposes: hedging against risk, speculation, etc.
 
 Much of modern finance is driven by the need to price these contracts
 accurately based on what is known about the properties (such as volatility) of
 the underlying asset. One method of pricing options is to use a Monte Carlo
 simulation of the underlying asset price. In this example we use this approach
 to price both European and Asian (path dependent) options for various strike
 prices and volatilities.
 
 The code for this example can be found in the :file:`docs/examples/newparallel`
 directory of the IPython source. The function :func:`price_options` in
 :file:`mcpricer.py` implements the basic Monte Carlo pricing algorithm using
 the NumPy package and is shown here:
 
 .. literalinclude:: ../../examples/newparallel/mcpricer.py
    :language: python
 
 To run this code in parallel, we will use IPython's :class:`LoadBalancedView` class,
 which distributes work to the engines using dynamic load balancing. This
 view is a wrapper of the :class:`Client` class shown in
 the previous example. The parallel calculation using :class:`LoadBalancedView` can
 be found in the file :file:`mcpricer.py`. The code in this file creates a
 :class:`TaskClient` instance and then submits a set of tasks using
 :meth:`TaskClient.run` that calculate the option prices for different
 volatilities and strike prices. The results are then plotted as a 2D contour
 plot using Matplotlib.
 
 .. literalinclude:: ../../examples/newparallel/mcdriver.py
    :language: python
 
 To use this code, start an IPython cluster using :command:`ipclusterz`, open
 IPython in the pylab mode with the file :file:`mcdriver.py` in your current
 working directory and then type:
 
 .. sourcecode:: ipython
 
     In [7]: run mcdriver.py
     Submitted tasks:  [0, 1, 2, ...]
 
 Once all the tasks have finished, the results can be plotted using the
 :func:`plot_options` function. Here we make contour plots of the Asian
 call and Asian put options as function of the volatility and strike price:
 
 .. sourcecode:: ipython
 
     In [8]: plot_options(sigma_vals, K_vals, prices['acall'])
 
     In [9]: plt.figure()
     Out[9]: <matplotlib.figure.Figure object at 0x18c178d0>
 
     In [10]: plot_options(sigma_vals, K_vals, prices['aput'])
 
 These results are shown in the two figures below. On a 8 core cluster the
 entire calculation (10 strike prices, 10 volatilities, 100,000 paths for each)
 took 30 seconds in parallel, giving a speedup of 7.7x, which is comparable
 to the speedup observed in our previous example.
 
 .. image:: ../parallel/asian_call.*
 
 .. image:: ../parallel/asian_put.*
 
 Conclusion
 ==========
 
 To conclude these examples, we summarize the key features of IPython's
 parallel architecture that have been demonstrated:
 
 * Serial code can be parallelized often with only a few extra lines of code.
   We have used the :class:`DirectView` and :class:`LoadBalancedView` classes
   for this purpose.
 * The resulting parallel code can be run without ever leaving the IPython's
   interactive shell.
 * Any data computed in parallel can be explored interactively through 
   visualization or further numerical calculations.
 * We have run these examples on a cluster running Windows HPC Server 2008.
   IPython's built in support for the Windows HPC job scheduler makes it
   easy to get started with IPython's parallel capabilities.
 
 .. note::
 
     The newparallel code has never been run on Windows HPC Server, so the last
     conclusion is untested.

docs/source/parallelz/parallel_details.txt

@@ -1,438 +1,493 b''
 .. _parallel_details:
 
 ==========================================
 Details of Parallel Computing with IPython
 ==========================================
 
 .. note::
 
     There are still many sections to fill out
 
 
 Caveats
 =======
 
 First, some caveats about the detailed workings of parallel computing with 0MQ and IPython.
 
 Non-copying sends and numpy arrays
 ----------------------------------
 
 When numpy arrays are passed as arguments to apply or via data-movement methods, they are not
-copied.  This means that you must be careful if you are sending an array that you intend to work on.
-PyZMQ does allow you to track when a message has been sent so you can know when it is safe to edit the buffer, but
-IPython only allows for this.
+copied. This means that you must be careful if you are sending an array that you intend to work
+on. PyZMQ does allow you to track when a message has been sent so you can know when it is safe
+to edit the buffer, but IPython only allows for this.
 
 It is also important to note that the non-copying receive of a message is *read-only*.  That
-means that if you intend to work in-place on an array that you have sent or received, you must copy
-it.  This is true for both numpy arrays sent to engines and numpy arrays retrieved as results.
+means that if you intend to work in-place on an array that you have sent or received, you must
+copy it. This is true for both numpy arrays sent to engines and numpy arrays retrieved as
+results.
 
 The following will fail:
 
 .. sourcecode:: ipython
 
     In [3]: A = numpy.zeros(2)
     
     In [4]: def setter(a):
        ...: a[0]=1
        ...: return a
 
     In [5]: rc[0].apply_sync(setter, A)
     ---------------------------------------------------------------------------
     RemoteError                               Traceback (most recent call last)
     ...
     RemoteError: RuntimeError(array is not writeable)
     Traceback (most recent call last):
       File "/Users/minrk/dev/ip/mine/IPython/zmq/parallel/streamkernel.py", line 329, in apply_request
         exec code in working, working
       File "<string>", line 1, in <module>
       File "<ipython-input-14-736187483856>", line 2, in setter
     RuntimeError: array is not writeable
 
 If you do need to edit the array in-place, just remember to copy the array if it's read-only.
 The :attr:`ndarray.flags.writeable` flag will tell you if you can write to an array.
 
 .. sourcecode:: ipython
 
     In [3]: A = numpy.zeros(2)
     
     In [4]: def setter(a):
        ...:     """only copy read-only arrays"""
        ...:     if not a.flags.writeable:
        ...:         a=a.copy()
        ...:     a[0]=1
        ...:     return a
 
     In [5]: rc[0].apply_sync(setter, A)
     Out[5]: array([ 1.,  0.])
     
     # note that results will also be read-only:
     In [6]: _.flags.writeable
     Out[6]: False
 
+If you want to safely edit an array in-place after *sending* it, you must use the `track=True` flag.  IPython always performs non-copying sends of arrays, which return immediately.  You
+must instruct IPython track those messages *at send time* in order to know for sure that the send has completed.  AsyncResults have a :attr:`sent` property, and :meth:`wait_on_send` method
+for checking and waiting for 0MQ to finish with a buffer.
+
+.. sourcecode:: ipython
+
+    In [5]: A = numpy.random.random((1024,1024))
+    
+    In [6]: view.track=True
+    
+    In [7]: ar = view.apply_async(lambda x: 2*x, A)
+    
+    In [8]: ar.sent
+    Out[8]: False
+    
+    In [9]: ar.wait_on_send() # blocks until sent is True
+
+
 What is sendable?
 -----------------
 
 If IPython doesn't know what to do with an object, it will pickle it. There is a short list of
 objects that are not pickled: ``buffers``, ``str/bytes`` objects, and ``numpy``
 arrays. These are handled specially by IPython in order to prevent the copying of data. Sending
 bytes or numpy arrays will result in exactly zero in-memory copies of your data (unless the data
 is very small).
 
 If you have an object that provides a Python buffer interface, then you can always send that
 buffer without copying - and reconstruct the object on the other side in your own code. It is
 possible that the object reconstruction will become extensible, so you can add your own
 non-copying types, but this does not yet exist.
 
+Closures
+********
+
+Just about anything in Python is pickleable. The one notable exception is objects (generally
+functions) with *closures*. Closures can be a complicated topic, but the basic principal is that
+functions that refer to variables in their parent scope have closures.
+
+An example of a function that uses a closure:
+
+.. sourcecode:: python
+
+    def f(a):
+        def inner():
+            # inner will have a closure
+            return a
+        return echo
+    
+    f1 = f(1)
+    f2 = f(2)
+    f1() # returns 1
+    f2() # returns 2
+
+f1 and f2 will have closures referring to the scope in which `inner` was defined, because they
+use the variable 'a'. As a result, you would not be able to send ``f1`` or ``f2`` with IPython.
+Note that you *would* be able to send `f`. This is only true for interactively defined
+functions (as are often used in decorators), and only when there are variables used inside the
+inner function, that are defined in the outer function. If the names are *not* in the outer
+function, then there will not be a closure, and the generated function will look in
+``globals()`` for the name:
+
+.. sourcecode:: python
+
+    def g(b):
+        # note that `b` is not referenced in inner's scope
+        def inner():
+            # this inner will *not* have a closure
+            return a
+        return echo
+    g1 = g(1)
+    g2 = g(2)
+    g1() # raises NameError on 'a'
+    a=5
+    g2() # returns 5
+
+`g1` and `g2` *will* be sendable with IPython, and will treat the engine's namespace as
+globals().  The :meth:`pull` method is implemented based on this principal.  If we did not
+provide pull, you could implement it yourself with `apply`, by simply returning objects out
+of the global namespace:
+
+.. sourcecode:: ipython
+
+    In [10]: view.apply(lambda : a)
+    
+    # is equivalent to
+    In [11]: view.pull('a')
 
 Running Code
 ============
 
 There are two principal units of execution in Python: strings of Python code (e.g. 'a=5'),
 and Python functions.  IPython is designed around the use of functions via the core
 Client method, called `apply`.
 
 Apply
 -----
 
-The principal method of remote execution is :meth:`apply`, of Client and View objects.  The Client provides the full execution and communication API for engines via its apply method.
+The principal method of remote execution is :meth:`apply`, of View objects. The Client provides
+the full execution and communication API for engines via its low-level
+:meth:`send_apply_message` method.
 
 f : function
     The fuction to be called remotely
 args : tuple/list
     The positional arguments passed to `f`
 kwargs : dict
     The keyword arguments passed to `f`
-bound : bool (default: False)
-    Whether to pass the Engine(s) Namespace as the first argument to `f`.
 block : bool (default: self.block)
     Whether to wait for the result, or return immediately.
     False:
         returns AsyncResult
     True:
         returns actual result(s) of f(*args, **kwargs)
         if multiple targets:
             list of results, matching `targets`
 track : bool
     whether to track non-copying sends.
     [default False]
 
 targets : int,list of ints, 'all', None
     Specify the destination of the job.
     if None:
         Submit via Task queue for load-balancing.
     if 'all':
         Run on all active engines
     if list:
         Run on each specified engine
     if int:
         Run on single engine
     Not eht
 
 balanced : bool, default None
     whether to load-balance.  This will default to True
     if targets is unspecified, or False if targets is specified.
     
     If `balanced` and `targets` are both specified, the task will
     be assigne to *one* of the targets by the scheduler.
     
-The following arguments are only used when balanced is True:
-
 after : Dependency or collection of msg_ids
     Only for load-balanced execution (targets=None)
     Specify a list of msg_ids as a time-based dependency.
     This job will only be run *after* the dependencies
     have been met.
     
 follow : Dependency or collection of msg_ids
     Only for load-balanced execution (targets=None)
     Specify a list of msg_ids as a location-based dependency.
     This job will only be run on an engine where this dependency
     is met.
 
 timeout : float/int or None
     Only for load-balanced execution (targets=None)
     Specify an amount of time (in seconds) for the scheduler to
     wait for dependencies to be met before failing with a
     DependencyTimeout.
 
 execute and run
 ---------------
 
-For executing strings of Python code, Clients also provide an :meth:`execute` and a :meth:`run`
-method, which rather than take functions and arguments, take simple strings. `execute` simply
-takes a string of Python code to execute, and sends it to the Engine(s). `run` is the same as
-`execute`, but for a *file*, rather than a string. It is simply a wrapper that does something
-very similar to ``execute(open(f).read())``.
+For executing strings of Python code, :class:`DirectView`s also provide an :meth:`execute` and a
+:meth:`run` method, which rather than take functions and arguments, take simple strings.
+`execute` simply takes a string of Python code to execute, and sends it to the Engine(s). `run`
+is the same as `execute`, but for a *file*, rather than a string. It is simply a wrapper that
+does something very similar to ``execute(open(f).read())``.
 
 .. note::
 
     TODO: Example
 
 Views
 =====
 
 The principal extension of the :class:`~parallel.client.Client` is the
-:class:`~parallel.view.View` class. The client is a fairly stateless object with respect to
-execution patterns, where you must specify everything about the execution as keywords to each
-call to :meth:`apply`. For users who want to more conveniently specify various options for
-several similar calls, we have the :class:`~parallel.view.View` objects. The basic principle of
-the views is to encapsulate the keyword arguments to :meth:`client.apply` as attributes,
-allowing users to specify them once and apply to any subsequent calls until the attribute is
-changed.
+:class:`~parallel.view.View` class. The client
 
 Two of apply's keyword arguments are set at the construction of the View, and are immutable for
 a given View: `balanced` and `targets`. `balanced` determines whether the View will be a
 :class:`.LoadBalancedView` or a :class:`.DirectView`, and `targets` will be the View's `targets`
 attribute.  Attempts to change this will raise errors.
 
-Views are cached by targets+balanced combinations, so requesting a view multiple times will always return the *same object*, not create a new one:
+Views are cached by targets/class, so requesting a view multiple times will always return the
+*same object*, not create a new one:
 
 .. sourcecode:: ipython
 
-    In [3]: v1 = rc.view([1,2,3], balanced=True)
-    In [4]: v2 = rc.view([1,2,3], balanced=True)
+    In [3]: v1 = rc.load_balanced_view([1,2,3])
+    In [4]: v2 = rc.load_balanced_view([1,2,3])
 
     In [5]: v2 is v1
     Out[5]: True
 
 
-A :class:`View` always uses its `targets` attribute, and it will use its `bound`
-and `block` attributes in its :meth:`apply` method, but the suffixed :meth:`apply_x`
-methods allow overriding `bound` and `block` for a single call.
-
-==================  ==========  ==========
-method              block       bound
-==================  ==========  ==========
-apply               self.block  self.bound
-apply_sync          True        False
-apply_async         False       False
-apply_sync_bound    True        True
-apply_async_bound   False       True
-==================  ==========  ==========
-
 DirectView
 ----------
 
 The :class:`.DirectView` is the class for the IPython :ref:`Multiplexing Interface
 <parallel_multiengine>`.
 
 Creating a DirectView
 *********************
 
 DirectViews can be created in two ways, by index access to a client, or by a client's
 :meth:`view` method.  Index access to a Client works in a few ways.  First, you can create
 DirectViews to single engines simply by accessing the client by engine id:
 
 .. sourcecode:: ipython
 
     In [2]: rc[0]
     Out[2]: <DirectView 0>
 
 You can also create a DirectView with a list of engines:
 
 .. sourcecode:: ipython
 
     In [2]: rc[0,1,2]
     Out[2]: <DirectView [0,1,2]>
 
 Other methods for accessing elements, such as slicing and negative indexing, work by passing
 the index directly to the client's :attr:`ids` list, so:
 
 .. sourcecode:: ipython
 
     # negative index
     In [2]: rc[-1]
     Out[2]: <DirectView 3>
     
     # or slicing:
     In [3]: rc[::2]
     Out[3]: <DirectView [0,2]>
 
 are always the same as:
 
 .. sourcecode:: ipython
 
     In [2]: rc[rc.ids[-1]]
     Out[2]: <DirectView 3>
     
     In [3]: rc[rc.ids[::2]]
     Out[3]: <DirectView [0,2]>
 
 Also note that the slice is evaluated at the time of construction of the DirectView, so the 
 targets will not change over time if engines are added/removed from the cluster.  Requesting
 two views with the same slice at different times will *not* necessarily return the same View
 if the number of engines has changed.
 
 Execution via DirectView
 ************************
 
 The DirectView is the simplest way to work with one or more engines directly (hence the name).  
 
 
 Data movement via DirectView
 ****************************
 
 Since a Python namespace is just a :class:`dict`, :class:`DirectView` objects provide
 dictionary-style access by key and methods such as :meth:`get` and
 :meth:`update` for convenience. This make the remote namespaces of the engines
 appear as a local dictionary. Underneath, these methods call :meth:`apply`:
 
 .. sourcecode:: ipython
 
     In [51]: dview['a']=['foo','bar']
 
     In [52]: dview['a']
     Out[52]: [ ['foo', 'bar'], ['foo', 'bar'], ['foo', 'bar'], ['foo', 'bar'] ]
 
 Scatter and gather
 ------------------
 
 Sometimes it is useful to partition a sequence and push the partitions to
 different engines. In MPI language, this is know as scatter/gather and we
 follow that terminology. However, it is important to remember that in
 IPython's :class:`Client` class, :meth:`scatter` is from the
 interactive IPython session to the engines and :meth:`gather` is from the
 engines back to the interactive IPython session. For scatter/gather operations
 between engines, MPI should be used:
 
 .. sourcecode:: ipython
 
     In [58]: dview.scatter('a',range(16))
     Out[58]: [None,None,None,None]
 
     In [59]: dview['a']
     Out[59]: [ [0, 1, 2, 3], [4, 5, 6, 7], [8, 9, 10, 11], [12, 13, 14, 15] ]
 
     In [60]: dview.gather('a')
     Out[60]: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]
 
 
     
 LoadBalancedView
 ----------------
 
 The :class:`.LoadBalancedView` 
 
 
 Data Movement
 =============
 
 push
 
 pull
 
 Reference
 
 Results
 =======
 
 AsyncResults are the primary class
 
 get_result
 
 results,metadata
 
 Querying the Hub
 ================
 
 The Hub sees all traffic that may pass through the schedulers between engines and clients.
 It does this so that it can track state, allowing multiple clients to retrieve results of
 computations submitted by their peers, as well as persisting the state to a database.
 
 queue_status
 
     You can check the status of the queues of the engines with this command.
 
 result_status
 
 purge_results
 
 Controlling the Engines
 =======================
 
 There are a few actions you can do with Engines that do not involve execution.  These
 messages are sent via the Control socket, and bypass any long queues of waiting execution
 jobs
 
 abort
 
     Sometimes you may want to prevent a job you have submitted from actually running. The method
     for this is :meth:`abort`. It takes a container of msg_ids, and instructs the Engines to not
     run the jobs if they arrive. The jobs will then fail with an AbortedTask error.
 
 clear
 
     You may want to purge the Engine(s) namespace of any data you have left in it.  After
     running `clear`, there will be no names in the Engine's namespace
 
 shutdown
 
     You can also instruct engines (and the Controller) to terminate from a Client.  This 
     can be useful when a job is finished, since you can shutdown all the processes with a
     single command.
 
 Synchronization
 ===============
 
 Since the Client is a synchronous object, events do not automatically trigger in your
 interactive session - you must poll the 0MQ sockets for incoming messages.  Note that 
 this polling *does not* actually make any network requests.  It simply performs a `select`
 operation, to check if messages are already in local memory, waiting to be handled.
 
-The method that handles incoming messages is :meth:`spin`.  This method flushes any waiting messages on the various incoming sockets, and updates the state of the Client.
+The method that handles incoming messages is :meth:`spin`. This method flushes any waiting
+messages on the various incoming sockets, and updates the state of the Client.
 
-If you need to wait for particular results to finish, you can use the :meth:`barrier` method,
+If you need to wait for particular results to finish, you can use the :meth:`wait` method,
 which will call :meth:`spin` until the messages are no longer outstanding. Anything that
 represents a collection of messages, such as a list of msg_ids or one or more AsyncResult
-objects, can be passed as argument to barrier. A timeout can be specified, which will prevent
-the barrier from blocking for more than a specified time, but the default behavior is to wait
+objects, can be passed as argument to wait. A timeout can be specified, which will prevent
+the call from blocking for more than a specified time, but the default behavior is to wait
 forever.
 
 
 
 The client also has an `outstanding` attribute - a ``set`` of msg_ids that are awaiting replies.
-This is the default if barrier is called with no arguments - i.e. barrier on *all* outstanding messages.
+This is the default if wait is called with no arguments - i.e. wait on *all* outstanding
+messages.
 
 
 .. note::
 
-    TODO barrier example
+    TODO wait example
 
 Map
 ===
 
 Many parallel computing problems can be expressed as a `map`, or running a single program with a
 variety of different inputs. Python has a built-in :py-func:`map`, which does exactly this, and
 many parallel execution tools in Python, such as the built-in :py-class:`multiprocessing.Pool`
 object provide implementations of `map`. All View objects provide a :meth:`map` method as well,
 but the load-balanced and direct implementations differ.
 
 Views' map methods can be called on any number of sequences, but they can also take the `block`
 and `bound` keyword arguments, just like :meth:`~client.apply`, but *only as keywords*.
 
 .. sourcecode:: python
 
     dview.map(*sequences, block=None)
 
 
 * iter, map_async, reduce
 
 Decorators and RemoteFunctions
 ==============================
 
 @parallel
 
 @remote
 
 RemoteFunction
 
 ParallelFunction
 
 Dependencies
 ============
 
 @depend
 
 @require
 
 Dependency

docs/source/parallelz/parallel_intro.txt

@@ -1,243 +1,244 b''
 .. _ip1par:
 
 ============================
 Overview and getting started
 ============================
 
 Introduction
 ============
 
 This section gives an overview of IPython's sophisticated and powerful
 architecture for parallel and distributed computing. This architecture
 abstracts out parallelism in a very general way, which enables IPython to
 support many different styles of parallelism including:
 
 * Single program, multiple data (SPMD) parallelism.
 * Multiple program, multiple data (MPMD) parallelism.
 * Message passing using MPI.
 * Task farming.
 * Data parallel.
 * Combinations of these approaches.
 * Custom user defined approaches.
 
 Most importantly, IPython enables all types of parallel applications to
 be developed, executed, debugged and monitored *interactively*. Hence,
 the ``I`` in IPython.  The following are some example usage cases for IPython:
 
 * Quickly parallelize algorithms that are embarrassingly parallel
   using a number of simple approaches.  Many simple things can be
   parallelized interactively in one or two lines of code.
 
 * Steer traditional MPI applications on a supercomputer from an
   IPython session on your laptop.
 
 * Analyze and visualize large datasets (that could be remote and/or
   distributed) interactively using IPython and tools like
   matplotlib/TVTK.
 
 * Develop, test and debug new parallel algorithms 
   (that may use MPI) interactively.
 
 * Tie together multiple MPI jobs running on different systems into
   one giant distributed and parallel system.
 
 * Start a parallel job on your cluster and then have a remote 	
   collaborator connect to it and pull back data into their 
   local IPython session for plotting and analysis.
 
 * Run a set of tasks on a set of CPUs using dynamic load balancing.
 
 Architecture overview
 =====================
 
 The IPython architecture consists of four components:
 
 * The IPython engine.
 * The IPython hub.
 * The IPython schedulers.
 * The controller client.
 
 These components live in the :mod:`IPython.zmq.parallel` package and are
 installed with IPython.  They do, however, have additional dependencies
 that must be installed.  For more information, see our
 :ref:`installation documentation <install_index>`.
 
 .. TODO: include zmq in install_index
 
 IPython engine
 ---------------
 
 The IPython engine is a Python instance that takes Python commands over a
 network connection. Eventually, the IPython engine will be a full IPython
 interpreter, but for now, it is a regular Python interpreter. The engine
 can also handle incoming and outgoing Python objects sent over a network
 connection.  When multiple engines are started, parallel and distributed
 computing becomes possible. An important feature of an IPython engine is
 that it blocks while user code is being executed. Read on for how the
 IPython controller solves this problem to expose a clean asynchronous API
 to the user.
 
 IPython controller
 ------------------
 
 The IPython controller processes provide an interface for working with a set of engines.
 At a general level, the controller is a collection of processes to which IPython engines
 and clients can connect. The controller is composed of a :class:`Hub` and a collection of
 :class:`Schedulers`. These Schedulers are typically run in separate processes but on the
 same machine as the Hub, but can be run anywhere from local threads or on remote machines.
 
 The controller also provides a single point of contact for users who wish to
 utilize the engines connected to the controller. There are different ways of
 working with a controller. In IPython, all of these models are implemented via
-the client's :meth:`.Client.apply` method, with various arguments, or
+the client's :meth:`.View.apply` method, with various arguments, or
 constructing :class:`.View` objects to represent subsets of engines. The two
 primary models for interacting with engines are:
 
 * A **Direct** interface, where engines are addressed explicitly.
 * A **LoadBalanced** interface, where the Scheduler is trusted with assigning work to
   appropriate engines.
 
 Advanced users can readily extend the View models to enable other
 styles of parallelism. 
 
 .. note:: 
 
     A single controller and set of engines can be used with multiple models
     simultaneously. This opens the door for lots of interesting things.
 
 
 The Hub
 *******
 
 The center of an IPython cluster is the Hub. This is the process that keeps
 track of engine connections, schedulers, clients, as well as all task requests and
 results. The primary role of the Hub is to facilitate queries of the cluster state, and
 minimize the necessary information required to establish the many connections involved in
 connecting new clients and engines.
 
 
 Schedulers
 **********
 
 All actions that can be performed on the engine go through a Scheduler. While the engines
 themselves block when user code is run, the schedulers hide that from the user to provide
 a fully asynchronous interface to a set of engines.
 
 
-IPython client
---------------
+IPython client and views
+------------------------
 
-There is one primary object, the :class:`~.parallel.client.Client`, for connecting to a
-controller. For each model, there is a corresponding view. These views allow users to
-interact with a set of engines through the interface. Here are the two default views:
+There is one primary object, the :class:`~.parallel.client.Client`, for connecting to a cluster.
+For each execution model, there is a corresponding :class:`~.parallel.view.View`. These views
+allow users to interact with a set of engines through the interface. Here are the two default
+views:
 
 * The :class:`DirectView` class for explicit addressing.
 * The :class:`LoadBalancedView` class for destination-agnostic scheduling.
 
 Security
 --------
 
 IPython uses ZeroMQ for networking, which has provided many advantages, but
 one of the setbacks is its utter lack of security [ZeroMQ]_. By default, no IPython
 connections are encrypted, but open ports only listen on localhost. The only
 source of security for IPython is via ssh-tunnel. IPython supports both shell
 (`openssh`) and `paramiko` based tunnels for connections.  There is a key necessary
 to submit requests, but due to the lack of encryption, it does not provide
 significant security if loopback traffic is compromised.
 
 In our architecture, the controller is the only process that listens on
 network ports, and is thus the main point of vulnerability. The standard model
 for secure connections is to designate that the controller listen on
 localhost, and use ssh-tunnels to connect clients and/or
 engines.
 
 To connect and authenticate to the controller an engine or client needs
 some information that the controller has stored in a JSON file.
 Thus, the JSON files need to be copied to a location where
 the clients and engines can find them. Typically, this is the
 :file:`~/.ipython/clusterz_default/security` directory on the host where the 
 client/engine is running (which could be a different host than the controller). 
 Once the JSON files are copied over, everything should work fine.
 
 Currently, there are two JSON files that the controller creates:
 
 ipcontroller-engine.json
     This JSON file has the information necessary for an engine to connect
     to a controller.
 
 ipcontroller-client.json
     The client's connection information.  This may not differ from the engine's,
     but since the controller may listen on different ports for clients and
     engines, it is stored separately.
 
 More details of how these JSON files are used are given below.
 
 A detailed description of the security model and its implementation in IPython
 can be found :ref:`here <parallelsecurity>`.
 
 .. warning::
 
     Even at its most secure, the Controller listens on ports on localhost, and
     every time you make a tunnel, you open a localhost port on the connecting
     machine that points to the Controller. If localhost on the Controller's
     machine, or the machine of any client or engine, is untrusted, then your
     Controller is insecure. There is no way around this with ZeroMQ.
 
 
 
 Getting Started
 ===============
 
 To use IPython for parallel computing, you need to start one instance of the
 controller and one or more instances of the engine. Initially, it is best to
 simply start a controller and engines on a single host using the
 :command:`ipclusterz` command. To start a controller and 4 engines on your
 localhost, just do::
 
     $ ipclusterz start -n 4
 
 More details about starting the IPython controller and engines can be found
 :ref:`here <parallel_process>`
 
 Once you have started the IPython controller and one or more engines, you
 are ready to use the engines to do something useful. To make sure
 everything is working correctly, try the following commands:
 
 .. sourcecode:: ipython
 
 	In [1]: from IPython.zmq.parallel import client
 	
 	In [2]: c = client.Client()
 	
 	In [4]: c.ids
 	Out[4]: set([0, 1, 2, 3])
 	
-	In [5]: c.apply(lambda : "Hello, World", targets='all', block=True)
+	In [5]: c[:].apply_sync(lambda : "Hello, World")
 	Out[5]: [ 'Hello, World', 'Hello, World', 'Hello, World', 'Hello, World' ]
 
 
 When a client is created with no arguments, the client tries to find the corresponding
 JSON file in the local `~/.ipython/clusterz_default/security` directory. If it finds it,
 you are set. If you have put the JSON file in a different location or it has a different
 name, create the client like this:
 
 .. sourcecode:: ipython
 
     In [2]: c = client.Client('/path/to/my/ipcontroller-client.json')
 
 Remember, a client needs to be able to see the Hub's ports to connect. So if they are on a
 different machine, you may need to use an ssh server to tunnel access to that machine,
 then you would connect to it with:
 
 .. sourcecode:: ipython
 
     In [2]: c = client.Client(sshserver='myhub.example.com')
 
 Where 'myhub.example.com' is the url or IP address of the machine on
-which the Hub process is running.
+which the Hub process is running (or another machine that has direct access to the Hub's ports).
 
 You are now ready to learn more about the :ref:`Direct
-<parallelmultiengine>` and :ref:`LoadBalanced <paralleltask>` interfaces to the
+<parallel_multiengine>` and :ref:`LoadBalanced <parallel_task>` interfaces to the
 controller.
 
 .. [ZeroMQ] ZeroMQ.  http://www.zeromq.org

docs/source/parallelz/parallel_multiengine.txt

@@ -1,845 +1,799 b''
-.. _parallelmultiengine:
+.. _parallel_multiengine:
 
 ==========================
 IPython's Direct interface
 ==========================
 
 The direct, or multiengine, interface represents one possible way of working with a set of
 IPython engines. The basic idea behind the multiengine interface is that the
 capabilities of each engine are directly and explicitly exposed to the user.
 Thus, in the multiengine interface, each engine is given an id that is used to
 identify the engine and give it work to do. This interface is very intuitive
-and is designed with interactive usage in mind, and is thus the best place for
+and is designed with interactive usage in mind, and is the best place for
 new users of IPython to begin.
 
 Starting the IPython controller and engines
 ===========================================
 
 To follow along with this tutorial, you will need to start the IPython
 controller and four IPython engines. The simplest way of doing this is to use
 the :command:`ipclusterz` command::
 
     $ ipclusterz start -n 4
     
 For more detailed information about starting the controller and engines, see
 our :ref:`introduction <ip1par>` to using IPython for parallel computing.
 
 Creating a ``Client`` instance
 ==============================
 
 The first step is to import the IPython :mod:`IPython.zmq.parallel.client`
 module and then create a :class:`.Client` instance:
 
 .. sourcecode:: ipython
 
     In [1]: from IPython.zmq.parallel import client
     
     In [2]: rc = client.Client()
 
 This form assumes that the default connection information (stored in
 :file:`ipcontroller-client.json` found in :file:`IPYTHON_DIR/clusterz_default/security`) is
 accurate. If the controller was started on a remote machine, you must copy that connection
 file to the client machine, or enter its contents as arguments to the Client constructor:
 
 .. sourcecode:: ipython
 
     # If you have copied the json connector file from the controller:
     In [2]: rc = client.Client('/path/to/ipcontroller-client.json')
     # or to connect with a specific profile you have set up:
     In [3]: rc = client.Client(profile='mpi')
     
 
 To make sure there are engines connected to the controller, users can get a list
 of engine ids:
 
 .. sourcecode:: ipython
 
     In [3]: rc.ids
     Out[3]: [0, 1, 2, 3]
 
 Here we see that there are four engines ready to do work for us.
 
 For direct execution, we will make use of a :class:`DirectView` object, which can be
 constructed via list-access to the client:
 
 .. sourcecode:: ipython
 
     In [4]: dview = rc[:] # use all engines
 
 .. seealso::
 
     For more information, see the in-depth explanation of :ref:`Views <parallel_details>`.
 
 
 Quick and easy parallelism
 ==========================
 
 In many cases, you simply want to apply a Python function to a sequence of
 objects, but *in parallel*. The client interface provides a simple way
 of accomplishing this: using the DirectView's :meth:`~DirectView.map` method.
 
 Parallel map
 ------------
 
 Python's builtin :func:`map` functions allows a function to be applied to a
 sequence element-by-element. This type of code is typically trivial to
 parallelize. In fact, since IPython's interface is all about functions anyway,
 you can just use the builtin :func:`map` with a :class:`RemoteFunction`, or a 
 DirectView's :meth:`map` method:
 
 .. sourcecode:: ipython
 
     In [62]: serial_result = map(lambda x:x**10, range(32))
     
-    In [63]: dview.block = True
-    
-    In [66]: parallel_result = dview.map(lambda x: x**10, range(32))
+    In [63]: parallel_result = dview.map_sync(lambda x: x**10, range(32))
 
     In [67]: serial_result==parallel_result
     Out[67]: True
 
 
 .. note::
 
     The :class:`DirectView`'s version of :meth:`map` does
     not do dynamic load balancing. For a load balanced version, use a
-    :class:`LoadBalancedView`, or a :class:`ParallelFunction` with
-    `balanced=True`.
+    :class:`LoadBalancedView`.
 
 .. seealso::
     
     :meth:`map` is implemented via :class:`ParallelFunction`.
 
 Remote function decorators
 --------------------------
 
 Remote functions are just like normal functions, but when they are called,
 they execute on one or more engines, rather than locally. IPython provides
 two decorators:
 
 .. sourcecode:: ipython
 
-    In [10]: @rc.remote(block=True, targets='all')
+    In [10]: @dview.remote(block=True)
         ...: def getpid():
         ...:     import os
         ...:     return os.getpid()
         ...: 
 
     In [11]: getpid()
     Out[11]: [12345, 12346, 12347, 12348]
 
-A ``@parallel`` decorator creates parallel functions, that break up an element-wise
+The ``@parallel`` decorator creates parallel functions, that break up an element-wise
 operations and distribute them, reconstructing the result.
 
 .. sourcecode:: ipython
 
     In [12]: import numpy as np
     
     In [13]: A = np.random.random((64,48))
     
-    In [14]: @rc.parallel(block=True, targets='all')
+    In [14]: @dview.parallel(block=True)
         ...: def pmul(A,B):
         ...:     return A*B
     
     In [15]: C_local = A*A
     
-    In [16]: C_remote_partial = pmul(A,A)
+    In [16]: C_remote = pmul(A,A)
     
     In [17]: (C_local == C_remote).all()
     Out[17]: True
 
 .. seealso::
 
     See the docstrings for the :func:`parallel` and :func:`remote` decorators for
     options.
 
 Calling Python functions
 ========================
 
 The most basic type of operation that can be performed on the engines is to
 execute Python code or call Python functions. Executing Python code can be
 done in blocking or non-blocking mode (non-blocking is default) using the
-:meth:`execute` method, and calling functions can be done via the
+:meth:`.View.execute` method, and calling functions can be done via the
 :meth:`.View.apply` method.
 
 apply
 -----
 
 The main method for doing remote execution (in fact, all methods that
-communicate with the engines are built on top of it), is :meth:`Client.apply`.
-Ideally, :meth:`apply` would have the signature ``apply(f,*args,**kwargs)``,
-which would call ``f(*args,**kwargs)`` remotely.  However, since :class:`Clients`
-require some more options, they cannot easily provide this interface.
-Instead, they provide the signature:
-
-.. sourcecode:: python
+communicate with the engines are built on top of it), is :meth:`View.apply`.
 
-    c.apply(f, args=None, kwargs=None, bound=True, block=None, targets=None,
-                    after=None, follow=None, timeout=None)
+We strive to provide the cleanest interface we can, so `apply` has the following
+signature:
 
-Where various behavior is controlled via keyword arguments. This means that in the client,
-you must pass `args` as a tuple, and `kwargs` as a dict.
+.. sourcecode:: python
 
-In order to provide the nicer interface, we have :class:`View` classes, which wrap
-:meth:`Client.apply` by using attributes and extra :meth:`apply_x` methods to determine
-the extra keyword arguments.  This means that the views can have the desired pattern:
+    view.apply(f, *args, **kwargs)
 
-.. sourcecode:: python
+There are various ways to call functions with IPython, and these flags are set as
+attributes of the View.  The ``DirectView`` has just two of these flags:
 
-    v.apply(f, *args, **kwargs)
+dv.block : bool
+    whether to wait for the result, or return an :class:`AsyncResult` object
+    immediately
+dv.track : bool
+    whether to instruct pyzmq to track when 
+    This is primarily useful for non-copying sends of numpy arrays that you plan to
+    edit in-place.  You need to know when it becomes safe to edit the buffer
+    without corrupting the message.
 
 
-For instance, performing index-access on a client creates a
-:class:`.DirectView`.
+Creating a view is simple: index-access on a client creates a :class:`.DirectView`.
 
 .. sourcecode:: ipython
 
     In [4]: view = rc[1:3]
     Out[4]: <DirectView [1, 2]>
 
     In [5]: view.apply<tab>
-    view.apply  view.apply_async  view.apply_async_bound  view.apply_sync  view.apply_sync_bound
-
-A :class:`DirectView` always uses its `targets` attribute, and it will use its `bound`
-and `block` attributes in its :meth:`apply` method, but the suffixed :meth:`apply_x`
-methods allow specifying `bound` and `block` via the different methods.
+    view.apply  view.apply_async  view.apply_sync  view.apply_with_flags
 
-==================  ==========  ==========
-method              block       bound
-==================  ==========  ==========
-apply               self.block  self.bound
-apply_sync          True        False
-apply_async         False       False
-apply_sync_bound    True        True
-apply_async_bound   False       True
-==================  ==========  ==========
-
-For explanation of these values, read on.
+For convenience, you can set block temporarily for a single call with the extra sync/async methods.
 
 Blocking execution
 ------------------
 
 In blocking mode, the :class:`.DirectView` object (called ``dview`` in
 these examples) submits the command to the controller, which places the
 command in the engines' queues for execution. The :meth:`apply` call then
 blocks until the engines are done executing the command:
 
 .. sourcecode:: ipython
 
     In [2]: dview = rc[:] # A DirectView of all engines
     In [3]: dview.block=True
     In [4]: dview['a'] = 5
 
     In [5]: dview['b'] = 10
 
-    In [6]: dview.apply_sync(lambda x: a+b+x, 27)
+    In [6]: dview.apply(lambda x: a+b+x, 27)
     Out[6]: [42, 42, 42, 42]
 
-Python commands can be executed on specific engines by calling execute using the ``targets``
-keyword argument in :meth:`client.execute`, or creating a :class:`DirectView` instance by
-index-access to the client:
-
-.. sourcecode:: ipython
-
-    In [6]: rc.execute('c=a+b', targets=[0,2])
-
-    In [7]: rc[1::2].execute('c=a-b') # shorthand for rc.execute('c=a-b',targets=[1,3])
-
-    In [8]: rc[:]['c'] # shorthand for rc.pull('c',targets='all')
-    Out[8]: [15, -5, 15, -5]
-
-.. note::
+You can also select blocking execution on a call-by-call basis with the :meth:`apply_sync`
+method:
 
-    Note that every call to ``rc.<meth>(...,targets=x)`` can be made via
-    ``rc[<x>].<meth>(...)``, which constructs a View object. The only place
-    where this differs in in :meth:`apply`. The :class:`Client` takes many
-    arguments to apply, so it requires `args` and `kwargs` to be passed as
-    individual arguments. Extended options such as `bound`,`targets`, and
-    `block` are controlled by the attributes of the :class:`View` objects, so
-    they can provide the much more convenient
-    :meth:`View.apply(f,*args,**kwargs)`, which simply calls
-    ``f(*args,**kwargs)`` remotely.
+    In [7]: dview.block=False
 
-Bound and unbound execution
----------------------------
+    In [8]: dview.apply_sync(lambda x: a+b+x, 27)
+    Out[8]: [42, 42, 42, 42]
 
-The previous example also shows one of the most important things about the IPython
-engines: they have a persistent user namespaces. The :meth:`apply` method can
-be run in either a bound or unbound manner.
-
-When applying a function in a `bound` manner, the first argument to that function
-will be the Engine's namespace, which is a :class:`Namespace` object, a dictionary
-also providing attribute-access to keys.
-
-In all (unbound and bound) execution
+Python commands can be executed as strings on specific engines by using a View's ``execute``
+method:
 
 .. sourcecode:: ipython
 
-    In [9]: dview['b'] = 5 # assign b to 5 everywhere
-    
-    In [10]: v0 = rc[0]
+    In [6]: rc[::2].execute('c=a+b')
     
-    # multiply b*2 inplace
-    In [12]: v0.apply_sync_bound(lambda ns: ns.b*=2)
+    In [7]: rc[1::2].execute('c=a-b')
     
-    # b is still available in globals during unbound execution
-    In [13]: v0.apply_sync(lambda a: a*b, 3)
-    Out[13]: 30
+    In [8]: rc[:]['c'] # shorthand for rc[:].pull('c', block=True)
+    Out[8]: [15, -5, 15, -5]
 
-`bound=True` specifies that the engine's namespace is to be passed as the first argument when
-the function is called, and the default `bound=False` specifies that the normal behavior, but
-the engine's namespace will be available as the globals() when the function is called.
 
 Non-blocking execution
 ----------------------
 
 In non-blocking mode, :meth:`apply` submits the command to be executed and
 then returns a :class:`AsyncResult` object immediately. The
 :class:`AsyncResult` object gives you a way of getting a result at a later
 time through its :meth:`get` method.
 
 .. Note::
 
     The :class:`AsyncResult` object provides a superset of the interface in 
     :py:class:`multiprocessing.pool.AsyncResult`.  See the 
     `official Python documentation <http://docs.python.org/library/multiprocessing#multiprocessing.pool.AsyncResult>`_
     for more.
 
 
 This allows you to quickly submit long running commands without blocking your
 local Python/IPython session:
 
 .. sourcecode:: ipython
     
     # define our function
     In [6]: def wait(t):
        ...:     import time
        ...:     tic = time.time()
        ...:     time.sleep(t)
        ...:     return time.time()-tic
     
     # In non-blocking mode
     In [7]: ar = dview.apply_async(wait, 2)
 
     # Now block for the result
     In [8]: ar.get()
     Out[8]: [2.0006198883056641, 1.9997570514678955, 1.9996809959411621, 2.0003249645233154]
 
     # Again in non-blocking mode
     In [9]: ar = dview.apply_async(wait, 10)
 
     # Poll to see if the result is ready
     In [10]: ar.ready()
     Out[10]: False
     
     # ask for the result, but wait a maximum of 1 second:
     In [45]: ar.get(1)
     ---------------------------------------------------------------------------
     TimeoutError                              Traceback (most recent call last)
     /home/you/<ipython-input-45-7cd858bbb8e0> in <module>()
     ----> 1 ar.get(1)
 
     /path/to/site-packages/IPython/zmq/parallel/asyncresult.pyc in get(self, timeout)
          62                 raise self._exception
          63         else:
     ---> 64             raise error.TimeoutError("Result not ready.")
          65 
          66     def ready(self):
 
     TimeoutError: Result not ready.
 
 .. Note::
 
     Note the import inside the function. This is a common model, to ensure
-    that the appropriate modules are imported where the task is run.
+    that the appropriate modules are imported where the task is run. You can
+    also manually import modules into the engine(s) namespace(s) via 
+    :meth:`view.execute('import numpy')`.
 
 Often, it is desirable to wait until a set of :class:`AsyncResult` objects
-are done. For this, there is a the method :meth:`barrier`. This method takes a
+are done. For this, there is a the method :meth:`wait`. This method takes a
 tuple of :class:`AsyncResult` objects (or `msg_ids` or indices to the client's History),
 and blocks until all of the associated results are ready:
 
 .. sourcecode:: ipython
 
-    In [72]: rc.block=False
+    In [72]: dview.block=False
 
     # A trivial list of AsyncResults objects
     In [73]: pr_list = [dview.apply_async(wait, 3) for i in range(10)]
 
     # Wait until all of them are done
-    In [74]: rc.barrier(pr_list)
+    In [74]: dview.wait(pr_list)
 
     # Then, their results are ready using get() or the `.r` attribute
     In [75]: pr_list[0].get()
     Out[75]: [2.9982571601867676, 2.9982588291168213, 2.9987530708312988, 2.9990990161895752]
     
 
 
-The ``block`` keyword argument and attributes
----------------------------------------------
+The ``block`` attribute
+-----------------------
 
-Most client methods(like :meth:`apply`) accept
+Many View methods(excluding :meth:`apply`) accept
 ``block`` as a keyword argument. As we have seen above, these
-keyword arguments control the blocking mode. The :class:`Client` class also has 
+keyword arguments control the blocking mode. The :class:`View` class also has 
 a :attr:`block` attribute that controls the default behavior when the keyword
 argument is not provided. Thus the following logic is used for :attr:`block`:
 
 * If no keyword argument is provided, the instance attributes are used.
 * Keyword argument, if provided override the instance attributes for
   the duration of a single call.
   
-DirectView objects also have a ``bound`` attribute, which is used in the same way.
-
 The following examples demonstrate how to use the instance attributes:
 
 .. sourcecode:: ipython
 
-    In [17]: rc.block = False
+    In [17]: dview.block = False
 
-    In [18]: ar = rc.apply(lambda : 10, targets=[0,2])
+    In [18]: ar = dview.apply(lambda : 10)
 
     In [19]: ar.get()
-    Out[19]: [10,10]
+    Out[19]: [10, 10, 10, 10]
 
-    In [21]: rc.block = True
+    In [21]: dview.block = True
 
     # Note targets='all' means all engines
-    In [22]: rc.apply(lambda : 42, targets='all')
+    In [22]: dview.apply(lambda : 42)
     Out[22]: [42, 42, 42, 42]
 
-The :attr:`block`, :attr:`bound`, and :attr:`targets` instance attributes of the
+The :attr:`block` and :attr:`targets` instance attributes of the
 :class:`.DirectView` also determine the behavior of the parallel magic commands.
 
-
 Parallel magic commands
 -----------------------
 
 .. warning::
 
-    The magics have not been changed to work with the zeromq system. ``%px``
-    and ``%autopx`` do work, but ``%result`` does not. %px and %autopx *do
-    not* print stdin/out.
+    The magics have not been changed to work with the zeromq system. The
+    magics do work, but *do not* print stdin/out like they used to in IPython.kernel.
 
 We provide a few IPython magic commands (``%px``, ``%autopx`` and ``%result``)
 that make it more pleasant to execute Python commands on the engines
 interactively. These are simply shortcuts to :meth:`execute` and
 :meth:`get_result` of the :class:`DirectView`. The ``%px`` magic executes a single 
 Python command on the engines specified by the :attr:`targets` attribute of the
 :class:`DirectView` instance:
 
 .. sourcecode:: ipython
 
+    # load the parallel magic extension:
+    In [21]: %load_ext parallelmagic
+    
     # Create a DirectView for all targets
     In [22]: dv = rc[:]
     
     # Make this DirectView active for parallel magic commands
     In [23]: dv.activate()
 
     In [24]: dv.block=True
 
     In [25]: import numpy
 
     In [26]: %px import numpy
     Parallel execution on engines: [0, 1, 2, 3]
 
     In [27]: %px a = numpy.random.rand(2,2)
     Parallel execution on engines: [0, 1, 2, 3]
 
     In [28]: %px ev = numpy.linalg.eigvals(a)
     Parallel execution on engines: [0, 1, 2, 3]
 
     In [28]: dv['ev']
     Out[28]: [ array([ 1.09522024, -0.09645227]), 
                array([ 1.21435496, -0.35546712]), 
                array([ 0.72180653,  0.07133042]), 
                array([  1.46384341e+00,   1.04353244e-04])
              ]
 
 The ``%result`` magic gets the most recent result, or takes an argument
 specifying the index of the result to be requested. It is simply a shortcut to the
 :meth:`get_result` method:
 
 .. sourcecode:: ipython
     
     In [29]: dv.apply_async(lambda : ev)
     
     In [30]: %result
     Out[30]: [ [ 1.28167017  0.14197338],
                 [-0.14093616  1.27877273],
                 [-0.37023573  1.06779409],
                 [ 0.83664764 -0.25602658] ]
 
 The ``%autopx`` magic switches to a mode where everything you type is executed
 on the engines given by the :attr:`targets` attribute:
 
 .. sourcecode:: ipython
 
     In [30]: dv.block=False
 
     In [31]: %autopx
     Auto Parallel Enabled
     Type %autopx to disable
 
     In [32]: max_evals = []
     <IPython.zmq.parallel.asyncresult.AsyncResult object at 0x17b8a70>
 
     In [33]: for i in range(100):
        ....:     a = numpy.random.rand(10,10)
        ....:     a = a+a.transpose()
        ....:     evals = numpy.linalg.eigvals(a)
        ....:     max_evals.append(evals[0].real)
        ....:
        ....:
     <IPython.zmq.parallel.asyncresult.AsyncResult object at 0x17af8f0>
 
     In [34]: %autopx
     Auto Parallel Disabled
 
     In [35]: dv.block=True
 
     In [36]: px ans= "Average max eigenvalue is: %f"%(sum(max_evals)/len(max_evals))
     Parallel execution on engines: [0, 1, 2, 3]
     
     In [37]: dv['ans']
     Out[37]: [ 'Average max eigenvalue is:  10.1387247332',
                'Average max eigenvalue is:  10.2076902286',
                'Average max eigenvalue is:  10.1891484655',
                'Average max eigenvalue is:  10.1158837784',]
 
 
 Moving Python objects around
 ============================
 
 In addition to calling functions and executing code on engines, you can
 transfer Python objects to and from your IPython session and the engines. In
 IPython, these operations are called :meth:`push` (sending an object to the
 engines) and :meth:`pull` (getting an object from the engines). 
 
 Basic push and pull
 -------------------
 
 Here are some examples of how you use :meth:`push` and :meth:`pull`:
 
 .. sourcecode:: ipython
 
-    In [38]: rc.push(dict(a=1.03234,b=3453))
+    In [38]: dview.push(dict(a=1.03234,b=3453))
     Out[38]: [None,None,None,None]
 
-    In [39]: rc.pull('a')
+    In [39]: dview.pull('a')
     Out[39]: [ 1.03234, 1.03234, 1.03234, 1.03234]
 
-    In [40]: rc.pull('b',targets=0)
+    In [40]: rc[0].pull('b')
     Out[40]: 3453
 
-    In [41]: rc.pull(('a','b'))
+    In [41]: dview.pull(('a','b'))
     Out[41]: [ [1.03234, 3453], [1.03234, 3453], [1.03234, 3453], [1.03234, 3453] ]
     
-    # zmq client does not have zip_pull
-    In [42]: rc.zip_pull(('a','b'))
-    Out[42]: [(1.03234, 1.03234, 1.03234, 1.03234), (3453, 3453, 3453, 3453)]
-
-    In [43]: rc.push(dict(c='speed'))
+    In [43]: dview.push(dict(c='speed'))
     Out[43]: [None,None,None,None]
 
 In non-blocking mode :meth:`push` and :meth:`pull` also return
 :class:`AsyncResult` objects:
 
 .. sourcecode:: ipython
 
-    In [47]: rc.block=False
-
-    In [48]: ar = rc.pull('a')
+    In [48]: ar = dview.pull('a', block=False)
 
     In [49]: ar.get()
     Out[49]: [1.03234, 1.03234, 1.03234, 1.03234]
 
 
 Dictionary interface
 --------------------
 
 Since a Python namespace is just a :class:`dict`, :class:`DirectView` objects provide
 dictionary-style access by key and methods such as :meth:`get` and
 :meth:`update` for convenience. This make the remote namespaces of the engines
 appear as a local dictionary. Underneath, these methods call :meth:`apply`:
 
 .. sourcecode:: ipython
 
-    In [50]: dview.block=True
-
     In [51]: dview['a']=['foo','bar']
 
     In [52]: dview['a']
     Out[52]: [ ['foo', 'bar'], ['foo', 'bar'], ['foo', 'bar'], ['foo', 'bar'] ]
 
 Scatter and gather
 ------------------
 
 Sometimes it is useful to partition a sequence and push the partitions to
 different engines. In MPI language, this is know as scatter/gather and we
 follow that terminology. However, it is important to remember that in
 IPython's :class:`Client` class, :meth:`scatter` is from the
 interactive IPython session to the engines and :meth:`gather` is from the
 engines back to the interactive IPython session. For scatter/gather operations
 between engines, MPI should be used:
 
 .. sourcecode:: ipython
 
     In [58]: dview.scatter('a',range(16))
     Out[58]: [None,None,None,None]
 
     In [59]: dview['a']
     Out[59]: [ [0, 1, 2, 3], [4, 5, 6, 7], [8, 9, 10, 11], [12, 13, 14, 15] ]
 
     In [60]: dview.gather('a')
     Out[60]: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]
 
 Other things to look at
 =======================
 
 How to do parallel list comprehensions
 --------------------------------------
 
 In many cases list comprehensions are nicer than using the map function. While
 we don't have fully parallel list comprehensions, it is simple to get the
 basic effect using :meth:`scatter` and :meth:`gather`:
 
 .. sourcecode:: ipython
 
     In [66]: dview.scatter('x',range(64))
 
-    In [67]: px y = [i**10 for i in x]
+    In [67]: %px y = [i**10 for i in x]
     Parallel execution on engines: [0, 1, 2, 3]
     Out[67]: 
 
     In [68]: y = dview.gather('y')
 
     In [69]: print y
     [0, 1, 1024, 59049, 1048576, 9765625, 60466176, 282475249, 1073741824,...]
 
 Parallel exceptions
 -------------------
 
 In the multiengine interface, parallel commands can raise Python exceptions,
 just like serial commands. But, it is a little subtle, because a single
 parallel command can actually raise multiple exceptions (one for each engine
 the command was run on). To express this idea, we have a
 :exc:`CompositeError` exception class that will be raised in most cases. The
 :exc:`CompositeError` class is a special type of exception that wraps one or
 more other types of exceptions. Here is how it works:
 
 .. sourcecode:: ipython
 
     In [76]: dview.block=True
 
     In [77]: dview.execute('1/0')
     ---------------------------------------------------------------------------
     CompositeError                            Traceback (most recent call last)
-    /Users/minrk/<ipython-input-10-5d56b303a66c> in <module>()
-    ----> 1 dview.execute('1/0')
-
-    ...
+    /home/you/<ipython-input-10-15c2c22dec39> in <module>()
+    ----> 1 dview.execute('1/0', block=True)
+
+    /path/to/site-packages/IPython/zmq/parallel/view.py in execute(self, code, block)
+        460                 default: self.block
+        461         """
+    --> 462         return self.apply_with_flags(util._execute, args=(code,), block=block)
+        463 
+        464     def run(self, filename, block=None):
+
+    /home/you/<string> in apply_with_flags(self, f, args, kwargs, block, track)
+
+    /path/to/site-packages/IPython/zmq/parallel/view.py in sync_results(f, self, *args, **kwargs)
+         46 def sync_results(f, self, *args, **kwargs):
+         47     """sync relevant results from self.client to our results attribute."""
+    ---> 48     ret = f(self, *args, **kwargs)
+         49     delta = self.outstanding.difference(self.client.outstanding)
+         50     completed = self.outstanding.intersection(delta)
+
+    /home/you/<string> in apply_with_flags(self, f, args, kwargs, block, track)
+
+    /path/to/site-packages/IPython/zmq/parallel/view.py in save_ids(f, self, *args, **kwargs)
+         35     n_previous = len(self.client.history)
+         36     try:
+    ---> 37         ret = f(self, *args, **kwargs)
+         38     finally:
+         39         nmsgs = len(self.client.history) - n_previous
+
+    /path/to/site-packages/IPython/zmq/parallel/view.py in apply_with_flags(self, f, args, kwargs, block, track)
+        398         if block:
+        399             try:
+    --> 400                 return ar.get()
+        401             except KeyboardInterrupt:
+        402                 pass
     
-    /Users/minrk/dev/ip/mine/IPython/zmq/parallel/client.pyc in apply(self, f, args, kwargs, bound, block, targets, balanced, after, follow, timeout)
-       1012                 raise ValueError(msg)
-       1013         else:
-    -> 1014             return self._apply_direct(f, args, kwargs, **options)
-       1015 
-       1016     def _apply_balanced(self, f, args, kwargs, bound=None, block=None, targets=None,
-
-    /Users/minrk/dev/ip/mine/IPython/zmq/parallel/client.pyc in _apply_direct(self, f, args, kwargs, bound, block, targets)
-       1100         if block:
-       1101             try:
-    -> 1102                 return ar.get()
-       1103             except KeyboardInterrupt:
-       1104                 return ar
-
-    /Users/minrk/dev/ip/mine/IPython/zmq/parallel/asyncresult.pyc in get(self, timeout)
-         78                 return self._result
-         79             else:
-    ---> 80                 raise self._exception
-         81         else:
-         82             raise error.TimeoutError("Result not ready.")
+    /path/to/site-packages/IPython/zmq/parallel/asyncresult.pyc in get(self, timeout)
+         87                 return self._result
+         88             else:
+    ---> 89                 raise self._exception
+         90         else:
+         91             raise error.TimeoutError("Result not ready.")
 
     CompositeError: one or more exceptions from call to method: _execute
     [0:apply]: ZeroDivisionError: integer division or modulo by zero
     [1:apply]: ZeroDivisionError: integer division or modulo by zero
     [2:apply]: ZeroDivisionError: integer division or modulo by zero
     [3:apply]: ZeroDivisionError: integer division or modulo by zero
 
+
 Notice how the error message printed when :exc:`CompositeError` is raised has
 information about the individual exceptions that were raised on each engine.
 If you want, you can even raise one of these original exceptions:
 
 .. sourcecode:: ipython
 
     In [80]: try:
-       ....:     rc.execute('1/0')
+       ....:     dview.execute('1/0')
        ....: except client.CompositeError, e:
        ....:     e.raise_exception()
        ....:     
        ....:     
     ---------------------------------------------------------------------------
     ZeroDivisionError                         Traceback (most recent call last)
 
     /ipython1-client-r3021/docs/examples/<ipython console> in <module>()
 
     /ipython1-client-r3021/ipython1/kernel/error.pyc in raise_exception(self, excid)
         156             raise IndexError("an exception with index %i does not exist"%excid)
         157         else:
     --> 158             raise et, ev, etb
         159 
         160 def collect_exceptions(rlist, method):
 
     ZeroDivisionError: integer division or modulo by zero
 
 If you are working in IPython, you can simple type ``%debug`` after one of
 these :exc:`CompositeError` exceptions is raised, and inspect the exception
 instance:
 
 .. sourcecode:: ipython
 
-    In [81]: rc.execute('1/0')
+    In [81]: dview.execute('1/0')
     ---------------------------------------------------------------------------
     CompositeError                            Traceback (most recent call last)
-    /Users/minrk/<ipython-input-5-b0c7a2b62c52> in <module>()
-    ----> 1 rc.execute('1/0')
-
-    /Users/minrk/<string> in execute(self, code, targets, block)
-
-    /Users/minrk/dev/ip/mine/IPython/zmq/parallel/client.pyc in defaultblock(f, self, *args, **kwargs)
-         88     self.block = block
-         89     try:
-    ---> 90         ret = f(self, *args, **kwargs)
-         91     finally:
-         92         self.block = saveblock
-
-    /Users/minrk/dev/ip/mine/IPython/zmq/parallel/client.pyc in execute(self, code, targets, block)
-        855                 default: self.block
-        856         """
-    --> 857         result = self.apply(_execute, (code,), targets=targets, block=block, bound=True, balanced=False)
-        858         if not block:
-        859             return result
-
-    /Users/minrk/<string> in apply(self, f, args, kwargs, bound, block, targets, balanced, after, follow, timeout)
-
-    /Users/minrk/dev/ip/mine/IPython/zmq/parallel/client.pyc in defaultblock(f, self, *args, **kwargs)
-         88     self.block = block
-         89     try:
-    ---> 90         ret = f(self, *args, **kwargs)
-         91     finally:
-         92         self.block = saveblock
-
-    /Users/minrk/dev/ip/mine/IPython/zmq/parallel/client.pyc in apply(self, f, args, kwargs, bound, block, targets, balanced, after, follow, timeout)
-       1012                 raise ValueError(msg)
-       1013         else:
-    -> 1014             return self._apply_direct(f, args, kwargs, **options)
-       1015 
-       1016     def _apply_balanced(self, f, args, kwargs, bound=None, block=None, targets=None,
-
-    /Users/minrk/dev/ip/mine/IPython/zmq/parallel/client.pyc in _apply_direct(self, f, args, kwargs, bound, block, targets)
-       1100         if block:
-       1101             try:
-    -> 1102                 return ar.get()
-       1103             except KeyboardInterrupt:
-       1104                 return ar
+    /home/you/<ipython-input-10-15c2c22dec39> in <module>()
+    ----> 1 dview.execute('1/0', block=True)
+
+    /path/to/site-packages/IPython/zmq/parallel/view.py in execute(self, code, block)
+        460                 default: self.block
+        461         """
+    --> 462         return self.apply_with_flags(util._execute, args=(code,), block=block)
+        463 
+        464     def run(self, filename, block=None):
+
+    /home/you/<string> in apply_with_flags(self, f, args, kwargs, block, track)
+
+    /path/to/site-packages/IPython/zmq/parallel/view.py in sync_results(f, self, *args, **kwargs)
+         46 def sync_results(f, self, *args, **kwargs):
+         47     """sync relevant results from self.client to our results attribute."""
+    ---> 48     ret = f(self, *args, **kwargs)
+         49     delta = self.outstanding.difference(self.client.outstanding)
+         50     completed = self.outstanding.intersection(delta)
+
+    /home/you/<string> in apply_with_flags(self, f, args, kwargs, block, track)
+
+    /path/to/site-packages/IPython/zmq/parallel/view.py in save_ids(f, self, *args, **kwargs)
+         35     n_previous = len(self.client.history)
+         36     try:
+    ---> 37         ret = f(self, *args, **kwargs)
+         38     finally:
+         39         nmsgs = len(self.client.history) - n_previous
+
+    /path/to/site-packages/IPython/zmq/parallel/view.py in apply_with_flags(self, f, args, kwargs, block, track)
+        398         if block:
+        399             try:
+    --> 400                 return ar.get()
+        401             except KeyboardInterrupt:
+        402                 pass
 
-    /Users/minrk/dev/ip/mine/IPython/zmq/parallel/asyncresult.pyc in get(self, timeout)
-         78                 return self._result
-         79             else:
-    ---> 80                 raise self._exception
-         81         else:
-         82             raise error.TimeoutError("Result not ready.")
+    /path/to/site-packages/IPython/zmq/parallel/asyncresult.pyc in get(self, timeout)
+         87                 return self._result
+         88             else:
+    ---> 89                 raise self._exception
+         90         else:
+         91             raise error.TimeoutError("Result not ready.")
 
     CompositeError: one or more exceptions from call to method: _execute
     [0:apply]: ZeroDivisionError: integer division or modulo by zero
     [1:apply]: ZeroDivisionError: integer division or modulo by zero
     [2:apply]: ZeroDivisionError: integer division or modulo by zero
     [3:apply]: ZeroDivisionError: integer division or modulo by zero
     
     In [82]: %debug
     > /Users/minrk/dev/ip/mine/IPython/zmq/parallel/asyncresult.py(80)get()
          79             else:
     ---> 80                 raise self._exception
          81         else:
     
     
     # With the debugger running, e is the exceptions instance.  We can tab complete
     # on it and see the extra methods that are available.
     ipdb> e.
     e.__class__         e.__getitem__       e.__new__           e.__setstate__      e.args
     e.__delattr__       e.__getslice__      e.__reduce__        e.__str__           e.elist
     e.__dict__          e.__hash__          e.__reduce_ex__     e.__weakref__       e.message
     e.__doc__           e.__init__          e.__repr__          e._get_engine_str   e.print_tracebacks
     e.__getattribute__  e.__module__        e.__setattr__       e._get_traceback    e.raise_exception
     ipdb> e.print_tracebacks()
     [0:apply]: 
     Traceback (most recent call last):
       File "/Users/minrk/dev/ip/mine/IPython/zmq/parallel/streamkernel.py", line 332, in apply_request
         exec code in working, working
       File "<string>", line 1, in <module>
       File "/Users/minrk/dev/ip/mine/IPython/zmq/parallel/client.py", line 69, in _execute
         exec code in globals()
       File "<string>", line 1, in <module>
     ZeroDivisionError: integer division or modulo by zero
 
 
     [1:apply]: 
     Traceback (most recent call last):
       File "/Users/minrk/dev/ip/mine/IPython/zmq/parallel/streamkernel.py", line 332, in apply_request
         exec code in working, working
       File "<string>", line 1, in <module>
       File "/Users/minrk/dev/ip/mine/IPython/zmq/parallel/client.py", line 69, in _execute
         exec code in globals()
       File "<string>", line 1, in <module>
     ZeroDivisionError: integer division or modulo by zero
 
 
     [2:apply]: 
     Traceback (most recent call last):
       File "/Users/minrk/dev/ip/mine/IPython/zmq/parallel/streamkernel.py", line 332, in apply_request
         exec code in working, working
       File "<string>", line 1, in <module>
       File "/Users/minrk/dev/ip/mine/IPython/zmq/parallel/client.py", line 69, in _execute
         exec code in globals()
       File "<string>", line 1, in <module>
     ZeroDivisionError: integer division or modulo by zero
 
 
     [3:apply]: 
     Traceback (most recent call last):
       File "/Users/minrk/dev/ip/mine/IPython/zmq/parallel/streamkernel.py", line 332, in apply_request
         exec code in working, working
       File "<string>", line 1, in <module>
       File "/Users/minrk/dev/ip/mine/IPython/zmq/parallel/client.py", line 69, in _execute
         exec code in globals()
       File "<string>", line 1, in <module>
     ZeroDivisionError: integer division or modulo by zero
     
 
+.. note::
+
+    TODO: The above tracebacks are not up to date
+
 
 All of this same error handling magic even works in non-blocking mode:
 
 .. sourcecode:: ipython
 
-    In [83]: rc.block=False
+    In [83]: dview.block=False
 
-    In [84]: ar = rc.execute('1/0')
+    In [84]: ar = dview.execute('1/0')
 
     In [85]: ar.get()
     ---------------------------------------------------------------------------
     CompositeError                            Traceback (most recent call last)
     /Users/minrk/<ipython-input-3-8531eb3d26fb> in <module>()
     ----> 1 ar.get()
 
     /Users/minrk/dev/ip/mine/IPython/zmq/parallel/asyncresult.pyc in get(self, timeout)
          78                 return self._result
          79             else:
     ---> 80                 raise self._exception
          81         else:
          82             raise error.TimeoutError("Result not ready.")
 
     CompositeError: one or more exceptions from call to method: _execute
     [0:apply]: ZeroDivisionError: integer division or modulo by zero
     [1:apply]: ZeroDivisionError: integer division or modulo by zero
     [2:apply]: ZeroDivisionError: integer division or modulo by zero
     [3:apply]: ZeroDivisionError: integer division or modulo by zero
 

docs/source/parallelz/parallel_task.txt

@@ -1,412 +1,419 b''
-.. _paralleltask:
+.. _parallel_task:
 
 ==========================
 The IPython task interface
 ==========================
 
 The task interface to the cluster presents the engines as a fault tolerant,
 dynamic load-balanced system of workers. Unlike the multiengine interface, in
 the task interface the user have no direct access to individual engines. By
 allowing the IPython scheduler to assign work, this interface is simultaneously
 simpler and more powerful.
 
 Best of all, the user can use both of these interfaces running at the same time
 to take advantage of their respective strengths. When the user can break up
 the user's work into segments that do not depend on previous execution, the
 task interface is ideal. But it also has more power and flexibility, allowing
 the user to guide the distribution of jobs, without having to assign tasks to
 engines explicitly.
 
 Starting the IPython controller and engines
 ===========================================
 
 To follow along with this tutorial, you will need to start the IPython
 controller and four IPython engines. The simplest way of doing this is to use
 the :command:`ipclusterz` command::
 
 	$ ipclusterz start -n 4
 	
 For more detailed information about starting the controller and engines, see
 our :ref:`introduction <ip1par>` to using IPython for parallel computing.
 
 Creating a ``Client`` instance
 ==============================
 
 The first step is to import the IPython :mod:`IPython.zmq.parallel.client`
 module and then create a :class:`.Client` instance, and we will also be using
 a :class:`LoadBalancedView`, here called `lview`:
 
 .. sourcecode:: ipython
 
     In [1]: from IPython.zmq.parallel import client
 
     In [2]: rc = client.Client()
     
 
 This form assumes that the controller was started on localhost with default
 configuration. If not, the location of the controller must be given as an
 argument to the constructor:
 
 .. sourcecode:: ipython
 
     # for a visible LAN controller listening on an external port:
     In [2]: rc = client.Client('tcp://192.168.1.16:10101')
     # or to connect with a specific profile you have set up:
     In [3]: rc = client.Client(profile='mpi')
 
-For load-balanced execution, we will make use of a :class:`LoadBalancedView` object, which can be constructed via the client's :meth:`view` method:
+For load-balanced execution, we will make use of a :class:`LoadBalancedView` object, which can
+be constructed via the client's :meth:`load_balanced_view` method:
 
 .. sourcecode:: ipython
 
-    In [4]: lview = rc.view() # default load-balanced view
+    In [4]: lview = rc.load_balanced_view() # default load-balanced view
 
 .. seealso::
 
     For more information, see the in-depth explanation of :ref:`Views <parallel_details>`.
 
 
 Quick and easy parallelism
 ==========================
 
 In many cases, you simply want to apply a Python function to a sequence of
 objects, but *in parallel*. Like the multiengine interface, these can be
 implemented via the task interface. The exact same tools can perform these
 actions in load-balanced ways as well as multiplexed ways: a parallel version
 of :func:`map` and :func:`@parallel` function decorator. If one specifies the
 argument `balanced=True`, then they are dynamically load balanced. Thus, if the
 execution time per item varies significantly, you should use the versions in
 the task interface.
 
 Parallel map
 ------------
 
 To load-balance :meth:`map`,simply use a LoadBalancedView:
 
 .. sourcecode:: ipython
     
     In [62]: lview.block = True
     
 	In [63]: serial_result = map(lambda x:x**10, range(32))
 
 	In [64]: parallel_result = lview.map(lambda x:x**10, range(32))
 
 	In [65]: serial_result==parallel_result
 	Out[65]: True
 
 Parallel function decorator
 ---------------------------
 
 Parallel functions are just like normal function, but they can be called on
 sequences and *in parallel*. The multiengine interface provides a decorator
 that turns any Python function into a parallel function:
 
 .. sourcecode:: ipython
 
     In [10]: @lview.parallel()
        ....: def f(x):
        ....:     return 10.0*x**4
        ....:
 
     In [11]: f.map(range(32))    # this is done in parallel
     Out[11]: [0.0,10.0,160.0,...]
 
+.. _parallel_dependencies:
+
 Dependencies
 ============
 
 Often, pure atomic load-balancing is too primitive for your work. In these cases, you
 may want to associate some kind of `Dependency` that describes when, where, or whether
 a task can be run.  In IPython, we provide two types of dependencies: 
 `Functional Dependencies`_ and `Graph Dependencies`_
 
 .. note::
 
     It is important to note that the pure ZeroMQ scheduler does not support dependencies,
     and you will see errors or warnings if you try to use dependencies with the pure
     scheduler.
 
 Functional Dependencies
 -----------------------
 
 Functional dependencies are used to determine whether a given engine is capable of running
 a particular task.  This is implemented via a special :class:`Exception` class, 
 :class:`UnmetDependency`, found in `IPython.zmq.parallel.error`.  Its use is very simple: 
 if a task fails with an UnmetDependency exception, then the scheduler, instead of relaying
 the error up to the client like any other error, catches the error, and submits the task
 to a different engine.  This will repeat indefinitely, and a task will never be submitted
 to a given engine a second time.
 
 You can manually raise the :class:`UnmetDependency` yourself, but IPython has provided 
 some decorators for facilitating this behavior.
 
 There are two decorators and a class used for functional dependencies:
 
 .. sourcecode:: ipython
 
     In [9]: from IPython.zmq.parallel.dependency import depend, require, dependent
 
 @require
 ********
 
 The simplest sort of dependency is requiring that a Python module is available. The
 ``@require`` decorator lets you define a function that will only run on engines where names
 you specify are importable:
 
 .. sourcecode:: ipython
 
     In [10]: @require('numpy', 'zmq')
         ...: def myfunc():
         ...:     import numpy,zmq
         ...:     return dostuff()
 
 Now, any time you apply :func:`myfunc`, the task will only run on a machine that has
 numpy and pyzmq available.
 
 @depend
 *******
 
 The ``@depend`` decorator lets you decorate any function with any *other* function to
 evaluate the dependency. The dependency function will be called at the start of the task,
 and if it returns ``False``, then the dependency will be considered unmet, and the task
 will be assigned to another engine. If the dependency returns *anything other than
 ``False``*, the rest of the task will continue.
 
 .. sourcecode:: ipython
 
     In [10]: def platform_specific(plat):
         ...:    import sys
         ...:    return sys.platform == plat
     
     In [11]: @depend(platform_specific, 'darwin')
         ...: def mactask():
         ...:    do_mac_stuff()
 
     In [12]: @depend(platform_specific, 'nt')
         ...: def wintask():
         ...:    do_windows_stuff()
     
 In this case, any time you apply ``mytask``, it will only run on an OSX machine.
 ``@depend`` is just like ``apply``, in that it has a ``@depend(f,*args,**kwargs)``
 signature.
 
 dependents
 **********
 
 You don't have to use the decorators on your tasks, if for instance you may want
 to run tasks with a single function but varying dependencies, you can directly construct
 the :class:`dependent` object that the decorators use:
 
 .. sourcecode::ipython
 
     In [13]: def mytask(*args):
         ...:    dostuff()
     
     In [14]: mactask = dependent(mytask, platform_specific, 'darwin')
     # this is the same as decorating the declaration of mytask with @depend
     # but you can do it again:
     
     In [15]: wintask = dependent(mytask, platform_specific, 'nt')
     
     # in general:
     In [16]: t = dependent(f, g, *dargs, **dkwargs)
     
     # is equivalent to:
     In [17]: @depend(g, *dargs, **dkwargs)
         ...: def t(a,b,c):
         ...:     # contents of f
 
 Graph Dependencies
 ------------------
 
 Sometimes you want to restrict the time and/or location to run a given task as a function
 of the time and/or location of other tasks. This is implemented via a subclass of
 :class:`set`, called a :class:`Dependency`. A Dependency is just a set of `msg_ids`
 corresponding to tasks, and a few attributes to guide how to decide when the Dependency
 has been met.
 
 The switches we provide for interpreting whether a given dependency set has been met:
 
 any|all
     Whether the dependency is considered met if *any* of the dependencies are done, or
     only after *all* of them have finished.  This is set by a Dependency's :attr:`all`
     boolean attribute, which defaults to ``True``.
 
-success_only
-    Whether to consider only tasks that did not raise an error as being fulfilled.
-    Sometimes you want to run a task after another, but only if that task succeeded.  In
-    this case, ``success_only`` should be ``True``.  However sometimes you may not care
-    whether the task succeeds, and always want the second task to run, in which case
-    you should use `success_only=False`.  The default behavior is to only use successes.
+success [default: True]
+    Whether to consider tasks that succeeded as fulfilling dependencies.
+
+failure [default : False]
+    Whether to consider tasks that failed as fulfilling dependencies.
+    using `failure=True,success=False` is useful for setting up cleanup tasks, to be run
+    only when tasks have failed.
+
+Sometimes you want to run a task after another, but only if that task succeeded. In this case,
+``success`` should be ``True`` and ``failure`` should be ``False``. However sometimes you may
+not care whether the task succeeds, and always want the second task to run, in which case you
+should use `success=failure=True`. The default behavior is to only use successes.
 
 There are other switches for interpretation that are made at the *task* level.  These are
 specified via keyword arguments to the client's :meth:`apply` method.
 
 after,follow
     You may want to run a task *after* a given set of dependencies have been run and/or
     run it *where* another set of dependencies are met. To support this, every task has an
     `after` dependency to restrict time, and a `follow` dependency to restrict
     destination.
 
 timeout
     You may also want to set a time-limit for how long the scheduler should wait before a
     task's dependencies are met. This is done via a `timeout`, which defaults to 0, which
     indicates that the task should never timeout. If the timeout is reached, and the
     scheduler still hasn't been able to assign the task to an engine, the task will fail
     with a :class:`DependencyTimeout`.
 
 .. note::
 
     Dependencies only work within the task scheduler. You cannot instruct a load-balanced
     task to run after a job submitted via the MUX interface.
 
-The simplest form of Dependencies is with `all=True,success_only=True`. In these cases,
+The simplest form of Dependencies is with `all=True,success=True,failure=False`. In these cases,
 you can skip using Dependency objects, and just pass msg_ids or AsyncResult objects as the
 `follow` and `after` keywords to :meth:`client.apply`:
 
 .. sourcecode:: ipython
 
     In [14]: client.block=False
 
-    In [15]: ar = client.apply(f, args, kwargs, balanced=True)
+    In [15]: ar = lview.apply(f, args, kwargs)
 
-    In [16]: ar2 = client.apply(f2, balanced=True)
+    In [16]: ar2 = lview.apply(f2)
 
-    In [17]: ar3 = client.apply(f3, after=[ar,ar2], balanced=True)
+    In [17]: ar3 = lview.apply_with_flags(f3, after=[ar,ar2])
 
-    In [17]: ar4 = client.apply(f3, follow=[ar], timeout=2.5, balanced=True)
+    In [17]: ar4 = lview.apply_with_flags(f3, follow=[ar], timeout=2.5)
 
 
 .. seealso::
 
     Some parallel workloads can be described as a `Directed Acyclic Graph
     <http://en.wikipedia.org/wiki/Directed_acyclic_graph>`_, or DAG. See :ref:`DAG
     Dependencies <dag_dependencies>` for an example demonstrating how to use map a NetworkX DAG
     onto task dependencies.
 
 
 
 Impossible Dependencies
 ***********************
 
 The schedulers do perform some analysis on graph dependencies to determine whether they
 are not possible to be met. If the scheduler does discover that a dependency cannot be
 met, then the task will fail with an :class:`ImpossibleDependency` error. This way, if the
 scheduler realized that a task can never be run, it won't sit indefinitely in the
 scheduler clogging the pipeline.
 
 The basic cases that are checked:
 
 * depending on nonexistent messages
 * `follow` dependencies were run on more than one machine and `all=True`
-* any dependencies failed and `all=True,success_only=True`
-* all dependencies failed and `all=False,success_only=True`
+* any dependencies failed and `all=True,success=True,failures=False`
+* all dependencies failed and `all=False,success=True,failure=False`
 
 .. warning::
 
     This analysis has not been proven to be rigorous, so it is likely possible for tasks
     to become impossible to run in obscure situations, so a timeout may be a good choice.
 
 .. _parallel_schedulers:
 
 Schedulers
 ==========
 
 There are a variety of valid ways to determine where jobs should be assigned in a 
 load-balancing situation.  In IPython, we support several standard schemes, and
 even make it easy to define your own.  The scheme can be selected via the ``--scheme``
 argument to :command:`ipcontrollerz`, or in the :attr:`HubFactory.scheme` attribute
 of a controller config object.
 
 The built-in routing schemes:
 
 To select one of these schemes, simply do::
 
     $ ipcontrollerz --scheme <schemename>
     for instance:
     $ ipcontrollerz --scheme lru
 
 lru: Least Recently Used
 
     Always assign work to the least-recently-used engine.  A close relative of
     round-robin, it will be fair with respect to the number of tasks, agnostic 
     with respect to runtime of each task.
 
 plainrandom: Plain Random
 
     Randomly picks an engine on which to run.
 
 twobin: Two-Bin Random
 
     **Requires numpy**
     
      Pick two engines at random, and use the LRU of the two. This is known to be better
     than plain random in many cases, but requires a small amount of computation.
 
 leastload: Least Load
 
     **This is the default scheme**
 
     Always assign tasks to the engine with the fewest outstanding tasks (LRU breaks tie).
 
 weighted: Weighted Two-Bin Random
 
     **Requires numpy**
     
     Pick two engines at random using the number of outstanding tasks as inverse weights,
     and use the one with the lower load.
 
 
 Pure ZMQ Scheduler
 ------------------
 
 For maximum throughput, the 'pure' scheme is not Python at all, but a C-level
 :class:`MonitoredQueue` from PyZMQ, which uses a ZeroMQ ``XREQ`` socket to perform all
 load-balancing. This scheduler does not support any of the advanced features of the Python
 :class:`.Scheduler`.
 
 Disabled features when using the ZMQ Scheduler:
 
 * Engine unregistration
     Task farming will be disabled if an engine unregisters.  
     Further, if an engine is unregistered during computation, the scheduler may not recover.
 * Dependencies
     Since there is no Python logic inside the Scheduler, routing decisions cannot be made
     based on message content.
 * Early destination notification
     The Python schedulers know which engine gets which task, and notify the Hub.  This 
     allows graceful handling of Engines coming and going.  There is no way to know
     where ZeroMQ messages have gone, so there is no way to know what tasks are on which
     engine until they *finish*.  This makes recovery from engine shutdown very difficult.
 
 
 .. note::
 
     TODO: performance comparisons
 
 
 More details
 ============
 
-The :class:`Client` has many more powerful features that allow quite a bit
+The :class:`LoadBalancedView` has many more powerful features that allow quite a bit
 of flexibility in how tasks are defined and run. The next places to look are
 in the following classes:
 
-* :class:`IPython.zmq.parallel.client.Client`
+* :class:`IPython.zmq.parallel.view.LoadBalancedView`
 * :class:`IPython.zmq.parallel.client.AsyncResult`
-* :meth:`IPython.zmq.parallel.client.Client.apply`
+* :meth:`IPython.zmq.parallel.view.LoadBalancedView.apply`
 * :mod:`IPython.zmq.parallel.dependency`
 
 The following is an overview of how to use these classes together:
 
-1. Create a :class:`Client`.
+1. Create a :class:`Client` and :class:`LoadBalancedView`
 2. Define some functions to be run as tasks
 3. Submit your tasks to using the :meth:`apply` method of your
-   :class:`Client` instance, specifying `balanced=True`. This signals
-   the :class:`Client` to entrust the Scheduler with assigning tasks to engines.
-4. Use :meth:`Client.get_results` to get the results of the
+   :class:`LoadBalancedView` instance.
+4. Use :meth:`Client.get_result` to get the results of the
    tasks, or use the :meth:`AsyncResult.get` method of the results to wait
    for and then receive the results.
 
-
 .. seealso::
 
     A demo of :ref:`DAG Dependencies <dag_dependencies>` with NetworkX and IPython.