add ownership to AsyncResult objects

An AsyncResult with owner=True will pop results and metadata, instead of get.
This should result is reduced memory growth on the Client.

AsyncResults created by apply, get_result, etc. have owner=True,
so this should help most cases.

IPython/parallel/client/asyncresult.py

@@ -1,695 +1,703 b''
 """AsyncResult objects for the client"""
 
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 from __future__ import print_function
 
 import sys
 import time
 from datetime import datetime
 
 from zmq import MessageTracker
 
 from IPython.core.display import clear_output, display, display_pretty
 from IPython.external.decorator import decorator
 from IPython.parallel import error
 from IPython.utils.py3compat import string_types
 
-#-----------------------------------------------------------------------------
-# Functions
-#-----------------------------------------------------------------------------
 
 def _raw_text(s):
     display_pretty(s, raw=True)
 
-#-----------------------------------------------------------------------------
-# Classes
-#-----------------------------------------------------------------------------
 
 # global empty tracker that's always done:
 finished_tracker = MessageTracker()
 
 @decorator
 def check_ready(f, self, *args, **kwargs):
     """Call spin() to sync state prior to calling the method."""
     self.wait(0)
     if not self._ready:
         raise error.TimeoutError("result not ready")
     return f(self, *args, **kwargs)
 
 class AsyncResult(object):
     """Class for representing results of non-blocking calls.
 
     Provides the same interface as :py:class:`multiprocessing.pool.AsyncResult`.
     """
 
     msg_ids = None
     _targets = None
     _tracker = None
     _single_result = False
+    owner = False,
 
-    def __init__(self, client, msg_ids, fname='unknown', targets=None, tracker=None):
+    def __init__(self, client, msg_ids, fname='unknown', targets=None, tracker=None,
+        owner=False,
+    ):
         if isinstance(msg_ids, string_types):
             # always a list
             msg_ids = [msg_ids]
             self._single_result = True
         else:
             self._single_result = False
         if tracker is None:
             # default to always done
             tracker = finished_tracker
         self._client = client
         self.msg_ids = msg_ids
         self._fname=fname
         self._targets = targets
         self._tracker = tracker
+        self.owner = owner
         
         self._ready = False
         self._outputs_ready = False
         self._success = None
         self._metadata = [self._client.metadata[id] for id in self.msg_ids]
 
     def __repr__(self):
         if self._ready:
             return "<%s: finished>"%(self.__class__.__name__)
         else:
             return "<%s: %s>"%(self.__class__.__name__,self._fname)
 
 
     def _reconstruct_result(self, res):
         """Reconstruct our result from actual result list (always a list)
 
         Override me in subclasses for turning a list of results
         into the expected form.
         """
         if self._single_result:
             return res[0]
         else:
             return res
 
     def get(self, timeout=-1):
         """Return the result when it arrives.
 
         If `timeout` is not ``None`` and the result does not arrive within
         `timeout` seconds then ``TimeoutError`` is raised. If the
         remote call raised an exception then that exception will be reraised
         by get() inside a `RemoteError`.
         """
         if not self.ready():
             self.wait(timeout)
 
         if self._ready:
             if self._success:
                 return self._result
             else:
                 raise self._exception
         else:
             raise error.TimeoutError("Result not ready.")
 
     def _check_ready(self):
         if not self.ready():
             raise error.TimeoutError("Result not ready.")
     
     def ready(self):
         """Return whether the call has completed."""
         if not self._ready:
             self.wait(0)
         elif not self._outputs_ready:
             self._wait_for_outputs(0)
         
         return self._ready
 
     def wait(self, timeout=-1):
         """Wait until the result is available or until `timeout` seconds pass.
 
         This method always returns None.
         """
         if self._ready:
             self._wait_for_outputs(timeout)
             return
         self._ready = self._client.wait(self.msg_ids, timeout)
         if self._ready:
             try:
                 results = list(map(self._client.results.get, self.msg_ids))
                 self._result = results
                 if self._single_result:
                     r = results[0]
                     if isinstance(r, Exception):
                         raise r
                 else:
                     results = error.collect_exceptions(results, self._fname)
                 self._result = self._reconstruct_result(results)
             except Exception as e:
                 self._exception = e
                 self._success = False
             else:
                 self._success = True
             finally:
                 if timeout is None or timeout < 0:
                     # cutoff infinite wait at 10s
                     timeout = 10
                 self._wait_for_outputs(timeout)
 
+                if self.owner:
+                    
+                    self._metadata = [self._client.metadata.pop(mid) for mid in self.msg_ids]
+                    [self._client.results.pop(mid) for mid in self.msg_ids]
+                
+
 
     def successful(self):
         """Return whether the call completed without raising an exception.
 
         Will raise ``AssertionError`` if the result is not ready.
         """
         assert self.ready()
         return self._success
 
     #----------------------------------------------------------------
     # Extra methods not in mp.pool.AsyncResult
     #----------------------------------------------------------------
 
     def get_dict(self, timeout=-1):
         """Get the results as a dict, keyed by engine_id.
 
         timeout behavior is described in `get()`.
         """
 
         results = self.get(timeout)
         if self._single_result:
             results = [results]
         engine_ids = [ md['engine_id'] for md in self._metadata ]
         
         
         rdict = {}
         for engine_id, result in zip(engine_ids, results):
             if engine_id in rdict:
                 raise ValueError("Cannot build dict, %i jobs ran on engine #%i" % (
                     engine_ids.count(engine_id), engine_id)
                 )
             else:
                 rdict[engine_id] = result
 
         return rdict
 
     @property
     def result(self):
         """result property wrapper for `get(timeout=-1)`."""
         return self.get()
 
     # abbreviated alias:
     r = result
 
     @property
     def metadata(self):
         """property for accessing execution metadata."""
         if self._single_result:
             return self._metadata[0]
         else:
             return self._metadata
 
     @property
     def result_dict(self):
         """result property as a dict."""
         return self.get_dict()
 
     def __dict__(self):
         return self.get_dict(0)
 
     def abort(self):
         """abort my tasks."""
         assert not self.ready(), "Can't abort, I am already done!"
         return self._client.abort(self.msg_ids, targets=self._targets, block=True)
 
     @property
     def sent(self):
         """check whether my messages have been sent."""
         return self._tracker.done
 
     def wait_for_send(self, timeout=-1):
         """wait for pyzmq send to complete.
 
         This is necessary when sending arrays that you intend to edit in-place.
         `timeout` is in seconds, and will raise TimeoutError if it is reached
         before the send completes.
         """
         return self._tracker.wait(timeout)
 
     #-------------------------------------
     # dict-access
     #-------------------------------------
 
     def __getitem__(self, key):
         """getitem returns result value(s) if keyed by int/slice, or metadata if key is str.
         """
         if isinstance(key, int):
             self._check_ready()
             return error.collect_exceptions([self._result[key]], self._fname)[0]
         elif isinstance(key, slice):
             self._check_ready()
             return error.collect_exceptions(self._result[key], self._fname)
         elif isinstance(key, string_types):
             # metadata proxy *does not* require that results are done
             self.wait(0)
             values = [ md[key] for md in self._metadata ]
             if self._single_result:
                 return values[0]
             else:
                 return values
         else:
             raise TypeError("Invalid key type %r, must be 'int','slice', or 'str'"%type(key))
 
     def __getattr__(self, key):
         """getattr maps to getitem for convenient attr access to metadata."""
         try:
             return self.__getitem__(key)
         except (error.TimeoutError, KeyError):
             raise AttributeError("%r object has no attribute %r"%(
                     self.__class__.__name__, key))
 
     # asynchronous iterator:
     def __iter__(self):
         if self._single_result:
             raise TypeError("AsyncResults with a single result are not iterable.")
         try:
             rlist = self.get(0)
         except error.TimeoutError:
             # wait for each result individually
             for msg_id in self.msg_ids:
                 ar = AsyncResult(self._client, msg_id, self._fname)
                 yield ar.get()
         else:
             # already done
             for r in rlist:
                 yield r
     
     def __len__(self):
         return len(self.msg_ids)
     
     #-------------------------------------
     # Sugar methods and attributes
     #-------------------------------------
     
     def timedelta(self, start, end, start_key=min, end_key=max):
         """compute the difference between two sets of timestamps
         
         The default behavior is to use the earliest of the first
         and the latest of the second list, but this can be changed
         by passing a different
         
         Parameters
         ----------
         
         start : one or more datetime objects (e.g. ar.submitted)
         end : one or more datetime objects (e.g. ar.received)
         start_key : callable
             Function to call on `start` to extract the relevant
             entry [defalt: min]
         end_key : callable
             Function to call on `end` to extract the relevant
             entry [default: max]
         
         Returns
         -------
         
         dt : float
             The time elapsed (in seconds) between the two selected timestamps.
         """
         if not isinstance(start, datetime):
             # handle single_result AsyncResults, where ar.stamp is single object,
             # not a list
             start = start_key(start)
         if not isinstance(end, datetime):
             # handle single_result AsyncResults, where ar.stamp is single object,
             # not a list
             end = end_key(end)
         return (end - start).total_seconds()
         
     @property
     def progress(self):
         """the number of tasks which have been completed at this point.
         
         Fractional progress would be given by 1.0 * ar.progress / len(ar)
         """
         self.wait(0)
         return len(self) - len(set(self.msg_ids).intersection(self._client.outstanding))
     
     @property
     def elapsed(self):
         """elapsed time since initial submission"""
         if self.ready():
             return self.wall_time
         
         now = submitted = datetime.now()
         for msg_id in self.msg_ids:
             if msg_id in self._client.metadata:
                 stamp = self._client.metadata[msg_id]['submitted']
                 if stamp and stamp < submitted:
                     submitted = stamp
         return (now-submitted).total_seconds()
     
     @property
     @check_ready
     def serial_time(self):
         """serial computation time of a parallel calculation
         
         Computed as the sum of (completed-started) of each task
         """
         t = 0
         for md in self._metadata:
             t += (md['completed'] - md['started']).total_seconds()
         return t
     
     @property
     @check_ready
     def wall_time(self):
         """actual computation time of a parallel calculation
         
         Computed as the time between the latest `received` stamp
         and the earliest `submitted`.
         
         Only reliable if Client was spinning/waiting when the task finished, because
         the `received` timestamp is created when a result is pulled off of the zmq queue,
         which happens as a result of `client.spin()`.
         
         For similar comparison of other timestamp pairs, check out AsyncResult.timedelta.
         
         """
         return self.timedelta(self.submitted, self.received)
     
     def wait_interactive(self, interval=1., timeout=-1):
         """interactive wait, printing progress at regular intervals"""
         if timeout is None:
             timeout = -1
         N = len(self)
         tic = time.time()
         while not self.ready() and (timeout < 0 or time.time() - tic <= timeout):
             self.wait(interval)
             clear_output(wait=True)
             print("%4i/%i tasks finished after %4i s" % (self.progress, N, self.elapsed), end="")
             sys.stdout.flush()
         print()
         print("done")
     
     def _republish_displaypub(self, content, eid):
         """republish individual displaypub content dicts"""
         try:
             ip = get_ipython()
         except NameError:
             # displaypub is meaningless outside IPython
             return
         md = content['metadata'] or {}
         md['engine'] = eid
         ip.display_pub.publish(content['source'], content['data'], md)
     
     def _display_stream(self, text, prefix='', file=None):
         if not text:
             # nothing to display
             return
         if file is None:
             file = sys.stdout
         end = '' if text.endswith('\n') else '\n'
         
         multiline = text.count('\n') > int(text.endswith('\n'))
         if prefix and multiline and not text.startswith('\n'):
             prefix = prefix + '\n'
         print("%s%s" % (prefix, text), file=file, end=end)
         
     
     def _display_single_result(self):
         self._display_stream(self.stdout)
         self._display_stream(self.stderr, file=sys.stderr)
         
         try:
             get_ipython()
         except NameError:
             # displaypub is meaningless outside IPython
             return
         
         for output in self.outputs:
             self._republish_displaypub(output, self.engine_id)
         
         if self.execute_result is not None:
             display(self.get())
     
     def _wait_for_outputs(self, timeout=-1):
         """wait for the 'status=idle' message that indicates we have all outputs
         """
         if self._outputs_ready or not self._success:
             # don't wait on errors
             return
         
         # cast None to -1 for infinite timeout
         if timeout is None:
             timeout = -1
         
         tic = time.time()
         while True:
             self._client._flush_iopub(self._client._iopub_socket)
             self._outputs_ready = all(md['outputs_ready']
                                       for md in self._metadata)
             if self._outputs_ready or \
                (timeout >= 0 and time.time() > tic + timeout):
                 break
             time.sleep(0.01)
     
     @check_ready
     def display_outputs(self, groupby="type"):
         """republish the outputs of the computation
         
         Parameters
         ----------
         
         groupby : str [default: type]
             if 'type':
                 Group outputs by type (show all stdout, then all stderr, etc.):
                 
                 [stdout:1] foo
                 [stdout:2] foo
                 [stderr:1] bar
                 [stderr:2] bar
             if 'engine':
                 Display outputs for each engine before moving on to the next:
                 
                 [stdout:1] foo
                 [stderr:1] bar
                 [stdout:2] foo
                 [stderr:2] bar
                 
             if 'order':
                 Like 'type', but further collate individual displaypub
                 outputs.  This is meant for cases of each command producing
                 several plots, and you would like to see all of the first
                 plots together, then all of the second plots, and so on.
         """
         if self._single_result:
             self._display_single_result()
             return
         
         stdouts = self.stdout
         stderrs = self.stderr
         execute_results  = self.execute_result
         output_lists = self.outputs
         results = self.get()
         
         targets = self.engine_id
         
         if groupby == "engine":
             for eid,stdout,stderr,outputs,r,execute_result in zip(
                     targets, stdouts, stderrs, output_lists, results, execute_results
                 ):
                 self._display_stream(stdout, '[stdout:%i] ' % eid)
                 self._display_stream(stderr, '[stderr:%i] ' % eid, file=sys.stderr)
                 
                 try:
                     get_ipython()
                 except NameError:
                     # displaypub is meaningless outside IPython
                     return 
                 
                 if outputs or execute_result is not None:
                     _raw_text('[output:%i]' % eid)
                 
                 for output in outputs:
                     self._republish_displaypub(output, eid)
                 
                 if execute_result is not None:
                     display(r)
         
         elif groupby in ('type', 'order'):
             # republish stdout:
             for eid,stdout in zip(targets, stdouts):
                 self._display_stream(stdout, '[stdout:%i] ' % eid)
         
             # republish stderr:
             for eid,stderr in zip(targets, stderrs):
                 self._display_stream(stderr, '[stderr:%i] ' % eid, file=sys.stderr)
         
             try:
                 get_ipython()
             except NameError:
                 # displaypub is meaningless outside IPython
                 return
             
             if groupby == 'order':
                 output_dict = dict((eid, outputs) for eid,outputs in zip(targets, output_lists))
                 N = max(len(outputs) for outputs in output_lists)
                 for i in range(N):
                     for eid in targets:
                         outputs = output_dict[eid]
                         if len(outputs) >= N:
                             _raw_text('[output:%i]' % eid)
                             self._republish_displaypub(outputs[i], eid)
             else:
                 # republish displaypub output
                 for eid,outputs in zip(targets, output_lists):
                     if outputs:
                         _raw_text('[output:%i]' % eid)
                     for output in outputs:
                         self._republish_displaypub(output, eid)
         
             # finally, add execute_result:
             for eid,r,execute_result in zip(targets, results, execute_results):
                 if execute_result is not None:
                     display(r)
         
         else:
             raise ValueError("groupby must be one of 'type', 'engine', 'collate', not %r" % groupby)
         
         
 
 
 class AsyncMapResult(AsyncResult):
     """Class for representing results of non-blocking gathers.
 
     This will properly reconstruct the gather.
     
     This class is iterable at any time, and will wait on results as they come.
     
     If ordered=False, then the first results to arrive will come first, otherwise
     results will be yielded in the order they were submitted.
     
     """
 
     def __init__(self, client, msg_ids, mapObject, fname='', ordered=True):
         AsyncResult.__init__(self, client, msg_ids, fname=fname)
         self._mapObject = mapObject
         self._single_result = False
         self.ordered = ordered
 
     def _reconstruct_result(self, res):
         """Perform the gather on the actual results."""
         return self._mapObject.joinPartitions(res)
 
     # asynchronous iterator:
     def __iter__(self):
         it = self._ordered_iter if self.ordered else self._unordered_iter
         for r in it():
             yield r
 
     # asynchronous ordered iterator:
     def _ordered_iter(self):
         """iterator for results *as they arrive*, preserving submission order."""
         try:
             rlist = self.get(0)
         except error.TimeoutError:
             # wait for each result individually
             for msg_id in self.msg_ids:
                 ar = AsyncResult(self._client, msg_id, self._fname)
                 rlist = ar.get()
                 try:
                     for r in rlist:
                         yield r
                 except TypeError:
                     # flattened, not a list
                     # this could get broken by flattened data that returns iterables
                     # but most calls to map do not expose the `flatten` argument
                     yield rlist
         else:
             # already done
             for r in rlist:
                 yield r
 
     # asynchronous unordered iterator:
     def _unordered_iter(self):
         """iterator for results *as they arrive*, on FCFS basis, ignoring submission order."""
         try:
             rlist = self.get(0)
         except error.TimeoutError:
             pending = set(self.msg_ids)
             while pending:
                 try:
                     self._client.wait(pending, 1e-3)
                 except error.TimeoutError:
                     # ignore timeout error, because that only means
                     # *some* jobs are outstanding
                     pass
                 # update ready set with those no longer outstanding:
                 ready = pending.difference(self._client.outstanding)
                 # update pending to exclude those that are finished
                 pending = pending.difference(ready)
                 while ready:
                     msg_id = ready.pop()
                     ar = AsyncResult(self._client, msg_id, self._fname)
                     rlist = ar.get()
                     try:
                         for r in rlist:
                             yield r
                     except TypeError:
                         # flattened, not a list
                         # this could get broken by flattened data that returns iterables
                         # but most calls to map do not expose the `flatten` argument
                         yield rlist
         else:
             # already done
             for r in rlist:
                 yield r
 
 
 class AsyncHubResult(AsyncResult):
     """Class to wrap pending results that must be requested from the Hub.
 
     Note that waiting/polling on these objects requires polling the Hubover the network,
     so use `AsyncHubResult.wait()` sparingly.
     """
 
     def _wait_for_outputs(self, timeout=-1):
         """no-op, because HubResults are never incomplete"""
         self._outputs_ready = True
     
     def wait(self, timeout=-1):
         """wait for result to complete."""
         start = time.time()
         if self._ready:
             return
         local_ids = [m for m in self.msg_ids if m in self._client.outstanding]
         local_ready = self._client.wait(local_ids, timeout)
         if local_ready:
             remote_ids = [m for m in self.msg_ids if m not in self._client.results]
             if not remote_ids:
                 self._ready = True
             else:
                 rdict = self._client.result_status(remote_ids, status_only=False)
                 pending = rdict['pending']
                 while pending and (timeout < 0 or time.time() < start+timeout):
                     rdict = self._client.result_status(remote_ids, status_only=False)
                     pending = rdict['pending']
                     if pending:
                         time.sleep(0.1)
                 if not pending:
                     self._ready = True
         if self._ready:
             try:
                 results = list(map(self._client.results.get, self.msg_ids))
                 self._result = results
                 if self._single_result:
                     r = results[0]
                     if isinstance(r, Exception):
                         raise r
                 else:
                     results = error.collect_exceptions(results, self._fname)
                 self._result = self._reconstruct_result(results)
             except Exception as e:
                 self._exception = e
                 self._success = False
             else:
                 self._success = True
             finally:
                 self._metadata = [self._client.metadata[mid] for mid in self.msg_ids]
+                if self.owner:
+                    [self._client.metadata.pop(mid) for mid in self.msg_ids]
+                    [self._client.results.pop(mid) for mid in self.msg_ids]
+            
 
 __all__ = ['AsyncResult', 'AsyncMapResult', 'AsyncHubResult']

IPython/parallel/client/client.py

@@ -1,1863 +1,1868 b''
 """A semi-synchronous Client for IPython parallel"""
 
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 from __future__ import print_function
 
 import os
 import json
 import sys
 from threading import Thread, Event
 import time
 import warnings
 from datetime import datetime
 from getpass import getpass
 from pprint import pprint
 
 pjoin = os.path.join
 
 import zmq
 
 from IPython.config.configurable import MultipleInstanceError
 from IPython.core.application import BaseIPythonApplication
 from IPython.core.profiledir import ProfileDir, ProfileDirError
 
 from IPython.utils.capture import RichOutput
 from IPython.utils.coloransi import TermColors
 from IPython.utils.jsonutil import rekey, extract_dates, parse_date
 from IPython.utils.localinterfaces import localhost, is_local_ip
 from IPython.utils.path import get_ipython_dir
 from IPython.utils.py3compat import cast_bytes, string_types, xrange, iteritems
 from IPython.utils.traitlets import (HasTraits, Integer, Instance, Unicode,
                                     Dict, List, Bool, Set, Any)
 from IPython.external.decorator import decorator
 from IPython.external.ssh import tunnel
 
 from IPython.parallel import Reference
 from IPython.parallel import error
 from IPython.parallel import util
 
 from IPython.kernel.zmq.session import Session, Message
 from IPython.kernel.zmq import serialize
 
 from .asyncresult import AsyncResult, AsyncHubResult
 from .view import DirectView, LoadBalancedView
 
 #--------------------------------------------------------------------------
 # Decorators for Client methods
 #--------------------------------------------------------------------------
 
 @decorator
 def spin_first(f, self, *args, **kwargs):
     """Call spin() to sync state prior to calling the method."""
     self.spin()
     return f(self, *args, **kwargs)
 
 
 #--------------------------------------------------------------------------
 # Classes
 #--------------------------------------------------------------------------
 
 
 class ExecuteReply(RichOutput):
     """wrapper for finished Execute results"""
     def __init__(self, msg_id, content, metadata):
         self.msg_id = msg_id
         self._content = content
         self.execution_count = content['execution_count']
         self.metadata = metadata
     
     # RichOutput overrides
     
     @property
     def source(self):
         execute_result = self.metadata['execute_result']
         if execute_result:
             return execute_result.get('source', '')
     
     @property
     def data(self):
         execute_result = self.metadata['execute_result']
         if execute_result:
             return execute_result.get('data', {})
     
     @property
     def _metadata(self):
         execute_result = self.metadata['execute_result']
         if execute_result:
             return execute_result.get('metadata', {})
     
     def display(self):
         from IPython.display import publish_display_data
         publish_display_data(self.data, self.metadata)
     
     def _repr_mime_(self, mime):
         if mime not in self.data:
             return
         data = self.data[mime]
         if mime in self._metadata:
             return data, self._metadata[mime]
         else:
             return data
     
     def __getitem__(self, key):
         return self.metadata[key]
     
     def __getattr__(self, key):
         if key not in self.metadata:
             raise AttributeError(key)
         return self.metadata[key]
     
     def __repr__(self):
         execute_result = self.metadata['execute_result'] or {'data':{}}
         text_out = execute_result['data'].get('text/plain', '')
         if len(text_out) > 32:
             text_out = text_out[:29] + '...'
         
         return "<ExecuteReply[%i]: %s>" % (self.execution_count, text_out)
     
     def _repr_pretty_(self, p, cycle):
         execute_result = self.metadata['execute_result'] or {'data':{}}
         text_out = execute_result['data'].get('text/plain', '')
         
         if not text_out:
             return
         
         try:
             ip = get_ipython()
         except NameError:
             colors = "NoColor"
         else:
             colors = ip.colors
         
         if colors == "NoColor":
             out = normal = ""
         else:
             out = TermColors.Red
             normal = TermColors.Normal
         
         if '\n' in text_out and not text_out.startswith('\n'):
             # add newline for multiline reprs
             text_out = '\n' + text_out
         
         p.text(
             out + u'Out[%i:%i]: ' % (
                 self.metadata['engine_id'], self.execution_count
             ) + normal + text_out
         )
 
 
 class Metadata(dict):
     """Subclass of dict for initializing metadata values.
 
     Attribute access works on keys.
 
     These objects have a strict set of keys - errors will raise if you try
     to add new keys.
     """
     def __init__(self, *args, **kwargs):
         dict.__init__(self)
         md = {'msg_id' : None,
               'submitted' : None,
               'started' : None,
               'completed' : None,
               'received' : None,
               'engine_uuid' : None,
               'engine_id' : None,
               'follow' : None,
               'after' : None,
               'status' : None,
 
               'execute_input' : None,
               'execute_result' : None,
               'error' : None,
               'stdout' : '',
               'stderr' : '',
               'outputs' : [],
               'data': {},
               'outputs_ready' : False,
             }
         self.update(md)
         self.update(dict(*args, **kwargs))
 
     def __getattr__(self, key):
         """getattr aliased to getitem"""
         if key in self:
             return self[key]
         else:
             raise AttributeError(key)
 
     def __setattr__(self, key, value):
         """setattr aliased to setitem, with strict"""
         if key in self:
             self[key] = value
         else:
             raise AttributeError(key)
 
     def __setitem__(self, key, value):
         """strict static key enforcement"""
         if key in self:
             dict.__setitem__(self, key, value)
         else:
             raise KeyError(key)
 
 
 class Client(HasTraits):
     """A semi-synchronous client to the IPython ZMQ cluster
 
     Parameters
     ----------
 
     url_file : str/unicode; path to ipcontroller-client.json
         This JSON file should contain all the information needed to connect to a cluster,
         and is likely the only argument needed.
         Connection information for the Hub's registration.  If a json connector
         file is given, then likely no further configuration is necessary.
         [Default: use profile]
     profile : bytes
         The name of the Cluster profile to be used to find connector information.
         If run from an IPython application, the default profile will be the same
         as the running application, otherwise it will be 'default'.
     cluster_id : str
         String id to added to runtime files, to prevent name collisions when using
         multiple clusters with a single profile simultaneously.
         When set, will look for files named like: 'ipcontroller-<cluster_id>-client.json'
         Since this is text inserted into filenames, typical recommendations apply:
         Simple character strings are ideal, and spaces are not recommended (but
         should generally work)
     context : zmq.Context
         Pass an existing zmq.Context instance, otherwise the client will create its own.
     debug : bool
         flag for lots of message printing for debug purposes
     timeout : int/float
         time (in seconds) to wait for connection replies from the Hub
         [Default: 10]
 
     #-------------- session related args ----------------
 
     config : Config object
         If specified, this will be relayed to the Session for configuration
     username : str
         set username for the session object
 
     #-------------- ssh related args ----------------
     # These are args for configuring the ssh tunnel to be used
     # credentials are used to forward connections over ssh to the Controller
     # Note that the ip given in `addr` needs to be relative to sshserver
     # The most basic case is to leave addr as pointing to localhost (127.0.0.1),
     # and set sshserver as the same machine the Controller is on. However,
     # the only requirement is that sshserver is able to see the Controller
     # (i.e. is within the same trusted network).
 
     sshserver : str
         A string of the form passed to ssh, i.e. 'server.tld' or '[e-mail]:port'
         If keyfile or password is specified, and this is not, it will default to
         the ip given in addr.
     sshkey : str; path to ssh private key file
         This specifies a key to be used in ssh login, default None.
         Regular default ssh keys will be used without specifying this argument.
     password : str
         Your ssh password to sshserver. Note that if this is left None,
         you will be prompted for it if passwordless key based login is unavailable.
     paramiko : bool
         flag for whether to use paramiko instead of shell ssh for tunneling.
         [default: True on win32, False else]
 
 
     Attributes
     ----------
 
     ids : list of int engine IDs
         requesting the ids attribute always synchronizes
         the registration state. To request ids without synchronization,
         use semi-private _ids attributes.
 
     history : list of msg_ids
         a list of msg_ids, keeping track of all the execution
         messages you have submitted in order.
 
     outstanding : set of msg_ids
         a set of msg_ids that have been submitted, but whose
         results have not yet been received.
 
     results : dict
         a dict of all our results, keyed by msg_id
 
     block : bool
         determines default behavior when block not specified
         in execution methods
 
     Methods
     -------
 
     spin
         flushes incoming results and registration state changes
         control methods spin, and requesting `ids` also ensures up to date
 
     wait
         wait on one or more msg_ids
 
     execution methods
         apply
         legacy: execute, run
 
     data movement
         push, pull, scatter, gather
 
     query methods
         queue_status, get_result, purge, result_status
 
     control methods
         abort, shutdown
 
     """
 
 
     block = Bool(False)
     outstanding = Set()
     results = Instance('collections.defaultdict', (dict,))
     metadata = Instance('collections.defaultdict', (Metadata,))
     history = List()
     debug = Bool(False)
     _spin_thread = Any()
     _stop_spinning = Any()
 
     profile=Unicode()
     def _profile_default(self):
         if BaseIPythonApplication.initialized():
             # an IPython app *might* be running, try to get its profile
             try:
                 return BaseIPythonApplication.instance().profile
             except (AttributeError, MultipleInstanceError):
                 # could be a *different* subclass of config.Application,
                 # which would raise one of these two errors.
                 return u'default'
         else:
             return u'default'
 
 
     _outstanding_dict = Instance('collections.defaultdict', (set,))
     _ids = List()
     _connected=Bool(False)
     _ssh=Bool(False)
     _context = Instance('zmq.Context')
     _config = Dict()
     _engines=Instance(util.ReverseDict, (), {})
     # _hub_socket=Instance('zmq.Socket')
     _query_socket=Instance('zmq.Socket')
     _control_socket=Instance('zmq.Socket')
     _iopub_socket=Instance('zmq.Socket')
     _notification_socket=Instance('zmq.Socket')
     _mux_socket=Instance('zmq.Socket')
     _task_socket=Instance('zmq.Socket')
     _task_scheme=Unicode()
     _closed = False
     _ignored_control_replies=Integer(0)
     _ignored_hub_replies=Integer(0)
 
     def __new__(self, *args, **kw):
         # don't raise on positional args
         return HasTraits.__new__(self, **kw)
 
     def __init__(self, url_file=None, profile=None, profile_dir=None, ipython_dir=None,
             context=None, debug=False,
             sshserver=None, sshkey=None, password=None, paramiko=None,
             timeout=10, cluster_id=None, **extra_args
             ):
         if profile:
             super(Client, self).__init__(debug=debug, profile=profile)
         else:
             super(Client, self).__init__(debug=debug)
         if context is None:
             context = zmq.Context.instance()
         self._context = context
         self._stop_spinning = Event()
         
         if 'url_or_file' in extra_args:
             url_file = extra_args['url_or_file']
             warnings.warn("url_or_file arg no longer supported, use url_file", DeprecationWarning)
         
         if url_file and util.is_url(url_file):
             raise ValueError("single urls cannot be specified, url-files must be used.")
 
         self._setup_profile_dir(self.profile, profile_dir, ipython_dir)
         
         if self._cd is not None:
             if url_file is None:
                 if not cluster_id:
                     client_json = 'ipcontroller-client.json'
                 else:
                     client_json = 'ipcontroller-%s-client.json' % cluster_id
                 url_file = pjoin(self._cd.security_dir, client_json)
         if url_file is None:
             raise ValueError(
                 "I can't find enough information to connect to a hub!"
                 " Please specify at least one of url_file or profile."
             )
         
         with open(url_file) as f:
             cfg = json.load(f)
         
         self._task_scheme = cfg['task_scheme']
 
         # sync defaults from args, json:
         if sshserver:
             cfg['ssh'] = sshserver
 
         location = cfg.setdefault('location', None)
         
         proto,addr = cfg['interface'].split('://')
         addr = util.disambiguate_ip_address(addr, location)
         cfg['interface'] = "%s://%s" % (proto, addr)
         
         # turn interface,port into full urls:
         for key in ('control', 'task', 'mux', 'iopub', 'notification', 'registration'):
             cfg[key] = cfg['interface'] + ':%i' % cfg[key]
         
         url = cfg['registration']
         
         if location is not None and addr == localhost():
             # location specified, and connection is expected to be local
             if not is_local_ip(location) and not sshserver:
                 # load ssh from JSON *only* if the controller is not on
                 # this machine
                 sshserver=cfg['ssh']
             if not is_local_ip(location) and not sshserver:
                 # warn if no ssh specified, but SSH is probably needed
                 # This is only a warning, because the most likely cause
                 # is a local Controller on a laptop whose IP is dynamic
                 warnings.warn("""
             Controller appears to be listening on localhost, but not on this machine.
             If this is true, you should specify Client(...,sshserver='you@%s')
             or instruct your controller to listen on an external IP."""%location,
                 RuntimeWarning)
         elif not sshserver:
             # otherwise sync with cfg
             sshserver = cfg['ssh']
 
         self._config = cfg
 
         self._ssh = bool(sshserver or sshkey or password)
         if self._ssh and sshserver is None:
             # default to ssh via localhost
             sshserver = addr
         if self._ssh and password is None:
             if tunnel.try_passwordless_ssh(sshserver, sshkey, paramiko):
                 password=False
             else:
                 password = getpass("SSH Password for %s: "%sshserver)
         ssh_kwargs = dict(keyfile=sshkey, password=password, paramiko=paramiko)
 
         # configure and construct the session
         try:
             extra_args['packer'] = cfg['pack']
             extra_args['unpacker'] = cfg['unpack']
             extra_args['key'] = cast_bytes(cfg['key'])
             extra_args['signature_scheme'] = cfg['signature_scheme']
         except KeyError as exc:
             msg = '\n'.join([
                 "Connection file is invalid (missing '{}'), possibly from an old version of IPython.",
                 "If you are reusing connection files, remove them and start ipcontroller again."
             ])
             raise ValueError(msg.format(exc.message))
         
         self.session = Session(**extra_args)
 
         self._query_socket = self._context.socket(zmq.DEALER)
 
         if self._ssh:
             tunnel.tunnel_connection(self._query_socket, cfg['registration'], sshserver, **ssh_kwargs)
         else:
             self._query_socket.connect(cfg['registration'])
 
         self.session.debug = self.debug
 
         self._notification_handlers = {'registration_notification' : self._register_engine,
                                     'unregistration_notification' : self._unregister_engine,
                                     'shutdown_notification' : lambda msg: self.close(),
                                     }
         self._queue_handlers = {'execute_reply' : self._handle_execute_reply,
                                 'apply_reply' : self._handle_apply_reply}
         
         try:
             self._connect(sshserver, ssh_kwargs, timeout)
         except:
             self.close(linger=0)
             raise
         
         # last step: setup magics, if we are in IPython:
         
         try:
             ip = get_ipython()
         except NameError:
             return
         else:
             if 'px' not in ip.magics_manager.magics:
                 # in IPython but we are the first Client.
                 # activate a default view for parallel magics.
                 self.activate()
 
     def __del__(self):
         """cleanup sockets, but _not_ context."""
         self.close()
 
     def _setup_profile_dir(self, profile, profile_dir, ipython_dir):
         if ipython_dir is None:
             ipython_dir = get_ipython_dir()
         if profile_dir is not None:
             try:
                 self._cd = ProfileDir.find_profile_dir(profile_dir)
                 return
             except ProfileDirError:
                 pass
         elif profile is not None:
             try:
                 self._cd = ProfileDir.find_profile_dir_by_name(
                     ipython_dir, profile)
                 return
             except ProfileDirError:
                 pass
         self._cd = None
 
     def _update_engines(self, engines):
         """Update our engines dict and _ids from a dict of the form: {id:uuid}."""
         for k,v in iteritems(engines):
             eid = int(k)
             if eid not in self._engines:
                 self._ids.append(eid)
             self._engines[eid] = v
         self._ids = sorted(self._ids)
         if sorted(self._engines.keys()) != list(range(len(self._engines))) and \
                         self._task_scheme == 'pure' and self._task_socket:
             self._stop_scheduling_tasks()
 
     def _stop_scheduling_tasks(self):
         """Stop scheduling tasks because an engine has been unregistered
         from a pure ZMQ scheduler.
         """
         self._task_socket.close()
         self._task_socket = None
         msg = "An engine has been unregistered, and we are using pure " +\
               "ZMQ task scheduling.  Task farming will be disabled."
         if self.outstanding:
             msg += " If you were running tasks when this happened, " +\
                    "some `outstanding` msg_ids may never resolve."
         warnings.warn(msg, RuntimeWarning)
 
     def _build_targets(self, targets):
         """Turn valid target IDs or 'all' into two lists:
         (int_ids, uuids).
         """
         if not self._ids:
             # flush notification socket if no engines yet, just in case
             if not self.ids:
                 raise error.NoEnginesRegistered("Can't build targets without any engines")
 
         if targets is None:
             targets = self._ids
         elif isinstance(targets, string_types):
             if targets.lower() == 'all':
                 targets = self._ids
             else:
                 raise TypeError("%r not valid str target, must be 'all'"%(targets))
         elif isinstance(targets, int):
             if targets < 0:
                 targets = self.ids[targets]
             if targets not in self._ids:
                 raise IndexError("No such engine: %i"%targets)
             targets = [targets]
 
         if isinstance(targets, slice):
             indices = list(range(len(self._ids))[targets])
             ids = self.ids
             targets = [ ids[i] for i in indices ]
 
         if not isinstance(targets, (tuple, list, xrange)):
             raise TypeError("targets by int/slice/collection of ints only, not %s"%(type(targets)))
 
         return [cast_bytes(self._engines[t]) for t in targets], list(targets)
 
     def _connect(self, sshserver, ssh_kwargs, timeout):
         """setup all our socket connections to the cluster. This is called from
         __init__."""
 
         # Maybe allow reconnecting?
         if self._connected:
             return
         self._connected=True
 
         def connect_socket(s, url):
             if self._ssh:
                 return tunnel.tunnel_connection(s, url, sshserver, **ssh_kwargs)
             else:
                 return s.connect(url)
 
         self.session.send(self._query_socket, 'connection_request')
         # use Poller because zmq.select has wrong units in pyzmq 2.1.7
         poller = zmq.Poller()
         poller.register(self._query_socket, zmq.POLLIN)
         # poll expects milliseconds, timeout is seconds
         evts = poller.poll(timeout*1000)
         if not evts:
             raise error.TimeoutError("Hub connection request timed out")
         idents,msg = self.session.recv(self._query_socket,mode=0)
         if self.debug:
             pprint(msg)
         content = msg['content']
         # self._config['registration'] = dict(content)
         cfg = self._config
         if content['status'] == 'ok':
             self._mux_socket = self._context.socket(zmq.DEALER)
             connect_socket(self._mux_socket, cfg['mux'])
 
             self._task_socket = self._context.socket(zmq.DEALER)
             connect_socket(self._task_socket, cfg['task'])
 
             self._notification_socket = self._context.socket(zmq.SUB)
             self._notification_socket.setsockopt(zmq.SUBSCRIBE, b'')
             connect_socket(self._notification_socket, cfg['notification'])
 
             self._control_socket = self._context.socket(zmq.DEALER)
             connect_socket(self._control_socket, cfg['control'])
 
             self._iopub_socket = self._context.socket(zmq.SUB)
             self._iopub_socket.setsockopt(zmq.SUBSCRIBE, b'')
             connect_socket(self._iopub_socket, cfg['iopub'])
 
             self._update_engines(dict(content['engines']))
         else:
             self._connected = False
             raise Exception("Failed to connect!")
 
     #--------------------------------------------------------------------------
     # handlers and callbacks for incoming messages
     #--------------------------------------------------------------------------
 
     def _unwrap_exception(self, content):
         """unwrap exception, and remap engine_id to int."""
         e = error.unwrap_exception(content)
         # print e.traceback
         if e.engine_info:
             e_uuid = e.engine_info['engine_uuid']
             eid = self._engines[e_uuid]
             e.engine_info['engine_id'] = eid
         return e
 
     def _extract_metadata(self, msg):
         header = msg['header']
         parent = msg['parent_header']
         msg_meta = msg['metadata']
         content = msg['content']
         md = {'msg_id' : parent['msg_id'],
               'received' : datetime.now(),
               'engine_uuid' : msg_meta.get('engine', None),
               'follow' : msg_meta.get('follow', []),
               'after' : msg_meta.get('after', []),
               'status' : content['status'],
             }
 
         if md['engine_uuid'] is not None:
             md['engine_id'] = self._engines.get(md['engine_uuid'], None)
 
         if 'date' in parent:
             md['submitted'] = parent['date']
         if 'started' in msg_meta:
             md['started'] = parse_date(msg_meta['started'])
         if 'date' in header:
             md['completed'] = header['date']
         return md
 
     def _register_engine(self, msg):
         """Register a new engine, and update our connection info."""
         content = msg['content']
         eid = content['id']
         d = {eid : content['uuid']}
         self._update_engines(d)
 
     def _unregister_engine(self, msg):
         """Unregister an engine that has died."""
         content = msg['content']
         eid = int(content['id'])
         if eid in self._ids:
             self._ids.remove(eid)
             uuid = self._engines.pop(eid)
 
             self._handle_stranded_msgs(eid, uuid)
 
         if self._task_socket and self._task_scheme == 'pure':
             self._stop_scheduling_tasks()
 
     def _handle_stranded_msgs(self, eid, uuid):
         """Handle messages known to be on an engine when the engine unregisters.
 
         It is possible that this will fire prematurely - that is, an engine will
         go down after completing a result, and the client will be notified
         of the unregistration and later receive the successful result.
         """
 
         outstanding = self._outstanding_dict[uuid]
 
         for msg_id in list(outstanding):
             if msg_id in self.results:
                 # we already
                 continue
             try:
                 raise error.EngineError("Engine %r died while running task %r"%(eid, msg_id))
             except:
                 content = error.wrap_exception()
             # build a fake message:
             msg = self.session.msg('apply_reply', content=content)
             msg['parent_header']['msg_id'] = msg_id
             msg['metadata']['engine'] = uuid
             self._handle_apply_reply(msg)
 
     def _handle_execute_reply(self, msg):
         """Save the reply to an execute_request into our results.
 
         execute messages are never actually used. apply is used instead.
         """
 
         parent = msg['parent_header']
         msg_id = parent['msg_id']
         if msg_id not in self.outstanding:
             if msg_id in self.history:
                 print("got stale result: %s"%msg_id)
             else:
                 print("got unknown result: %s"%msg_id)
         else:
             self.outstanding.remove(msg_id)
 
         content = msg['content']
         header = msg['header']
 
         # construct metadata:
         md = self.metadata[msg_id]
         md.update(self._extract_metadata(msg))
         # is this redundant?
         self.metadata[msg_id] = md
         
         e_outstanding = self._outstanding_dict[md['engine_uuid']]
         if msg_id in e_outstanding:
             e_outstanding.remove(msg_id)
 
         # construct result:
         if content['status'] == 'ok':
             self.results[msg_id] = ExecuteReply(msg_id, content, md)
         elif content['status'] == 'aborted':
             self.results[msg_id] = error.TaskAborted(msg_id)
         elif content['status'] == 'resubmitted':
             # TODO: handle resubmission
             pass
         else:
             self.results[msg_id] = self._unwrap_exception(content)
 
     def _handle_apply_reply(self, msg):
         """Save the reply to an apply_request into our results."""
         parent = msg['parent_header']
         msg_id = parent['msg_id']
         if msg_id not in self.outstanding:
             if msg_id in self.history:
                 print("got stale result: %s"%msg_id)
                 print(self.results[msg_id])
                 print(msg)
             else:
                 print("got unknown result: %s"%msg_id)
         else:
             self.outstanding.remove(msg_id)
         content = msg['content']
         header = msg['header']
 
         # construct metadata:
         md = self.metadata[msg_id]
         md.update(self._extract_metadata(msg))
         # is this redundant?
         self.metadata[msg_id] = md
 
         e_outstanding = self._outstanding_dict[md['engine_uuid']]
         if msg_id in e_outstanding:
             e_outstanding.remove(msg_id)
 
         # construct result:
         if content['status'] == 'ok':
             self.results[msg_id] = serialize.unserialize_object(msg['buffers'])[0]
         elif content['status'] == 'aborted':
             self.results[msg_id] = error.TaskAborted(msg_id)
         elif content['status'] == 'resubmitted':
             # TODO: handle resubmission
             pass
         else:
             self.results[msg_id] = self._unwrap_exception(content)
 
     def _flush_notifications(self):
         """Flush notifications of engine registrations waiting
         in ZMQ queue."""
         idents,msg = self.session.recv(self._notification_socket, mode=zmq.NOBLOCK)
         while msg is not None:
             if self.debug:
                 pprint(msg)
             msg_type = msg['header']['msg_type']
             handler = self._notification_handlers.get(msg_type, None)
             if handler is None:
                 raise Exception("Unhandled message type: %s" % msg_type)
             else:
                 handler(msg)
             idents,msg = self.session.recv(self._notification_socket, mode=zmq.NOBLOCK)
 
     def _flush_results(self, sock):
         """Flush task or queue results waiting in ZMQ queue."""
         idents,msg = self.session.recv(sock, mode=zmq.NOBLOCK)
         while msg is not None:
             if self.debug:
                 pprint(msg)
             msg_type = msg['header']['msg_type']
             handler = self._queue_handlers.get(msg_type, None)
             if handler is None:
                 raise Exception("Unhandled message type: %s" % msg_type)
             else:
                 handler(msg)
             idents,msg = self.session.recv(sock, mode=zmq.NOBLOCK)
 
     def _flush_control(self, sock):
         """Flush replies from the control channel waiting
         in the ZMQ queue.
 
         Currently: ignore them."""
         if self._ignored_control_replies <= 0:
             return
         idents,msg = self.session.recv(sock, mode=zmq.NOBLOCK)
         while msg is not None:
             self._ignored_control_replies -= 1
             if self.debug:
                 pprint(msg)
             idents,msg = self.session.recv(sock, mode=zmq.NOBLOCK)
 
     def _flush_ignored_control(self):
         """flush ignored control replies"""
         while self._ignored_control_replies > 0:
             self.session.recv(self._control_socket)
             self._ignored_control_replies -= 1
 
     def _flush_ignored_hub_replies(self):
         ident,msg = self.session.recv(self._query_socket, mode=zmq.NOBLOCK)
         while msg is not None:
             ident,msg = self.session.recv(self._query_socket, mode=zmq.NOBLOCK)
 
     def _flush_iopub(self, sock):
         """Flush replies from the iopub channel waiting
         in the ZMQ queue.
         """
         idents,msg = self.session.recv(sock, mode=zmq.NOBLOCK)
         while msg is not None:
             if self.debug:
                 pprint(msg)
             parent = msg['parent_header']
             # ignore IOPub messages with no parent.
             # Caused by print statements or warnings from before the first execution.
             if not parent:
                 idents,msg = self.session.recv(sock, mode=zmq.NOBLOCK)
                 continue
             msg_id = parent['msg_id']
             content = msg['content']
             header = msg['header']
             msg_type = msg['header']['msg_type']
 
             # init metadata:
             md = self.metadata[msg_id]
 
             if msg_type == 'stream':
                 name = content['name']
                 s = md[name] or ''
                 md[name] = s + content['data']
             elif msg_type == 'error':
                 md.update({'error' : self._unwrap_exception(content)})
             elif msg_type == 'execute_input':
                 md.update({'execute_input' : content['code']})
             elif msg_type == 'display_data':
                 md['outputs'].append(content)
             elif msg_type == 'execute_result':
                 md['execute_result'] = content
             elif msg_type == 'data_message':
                 data, remainder = serialize.unserialize_object(msg['buffers'])
                 md['data'].update(data)
             elif msg_type == 'status':
                 # idle message comes after all outputs
                 if content['execution_state'] == 'idle':
                     md['outputs_ready'] = True
             else:
                 # unhandled msg_type (status, etc.)
                 pass
 
             # reduntant?
             self.metadata[msg_id] = md
 
             idents,msg = self.session.recv(sock, mode=zmq.NOBLOCK)
 
     #--------------------------------------------------------------------------
     # len, getitem
     #--------------------------------------------------------------------------
 
     def __len__(self):
         """len(client) returns # of engines."""
         return len(self.ids)
 
     def __getitem__(self, key):
         """index access returns DirectView multiplexer objects
 
         Must be int, slice, or list/tuple/xrange of ints"""
         if not isinstance(key, (int, slice, tuple, list, xrange)):
             raise TypeError("key by int/slice/iterable of ints only, not %s"%(type(key)))
         else:
             return self.direct_view(key)
     
     def __iter__(self):
         """Since we define getitem, Client is iterable
         
         but unless we also define __iter__, it won't work correctly unless engine IDs
         start at zero and are continuous.
         """
         for eid in self.ids:
             yield self.direct_view(eid)
     
     #--------------------------------------------------------------------------
     # Begin public methods
     #--------------------------------------------------------------------------
 
     @property
     def ids(self):
         """Always up-to-date ids property."""
         self._flush_notifications()
         # always copy:
         return list(self._ids)
 
     def activate(self, targets='all', suffix=''):
         """Create a DirectView and register it with IPython magics
         
         Defines the magics `%px, %autopx, %pxresult, %%px`
         
         Parameters
         ----------
         
         targets: int, list of ints, or 'all'
             The engines on which the view's magics will run
         suffix: str [default: '']
             The suffix, if any, for the magics.  This allows you to have
             multiple views associated with parallel magics at the same time.
             
             e.g. ``rc.activate(targets=0, suffix='0')`` will give you
             the magics ``%px0``, ``%pxresult0``, etc. for running magics just
             on engine 0.
         """
         view = self.direct_view(targets)
         view.block = True
         view.activate(suffix)
         return view
 
     def close(self, linger=None):
         """Close my zmq Sockets
         
         If `linger`, set the zmq LINGER socket option,
         which allows discarding of messages.
         """
         if self._closed:
             return
         self.stop_spin_thread()
         snames = [ trait for trait in self.trait_names() if trait.endswith("socket") ]
         for name in snames:
             socket = getattr(self, name)
             if socket is not None and not socket.closed:
                 if linger is not None:
                     socket.close(linger=linger)
                 else:
                     socket.close()
         self._closed = True
 
     def _spin_every(self, interval=1):
         """target func for use in spin_thread"""
         while True:
             if self._stop_spinning.is_set():
                 return
             time.sleep(interval)
             self.spin()
 
     def spin_thread(self, interval=1):
         """call Client.spin() in a background thread on some regular interval
         
         This helps ensure that messages don't pile up too much in the zmq queue
         while you are working on other things, or just leaving an idle terminal.
         
         It also helps limit potential padding of the `received` timestamp
         on AsyncResult objects, used for timings.
         
         Parameters
         ----------
         
         interval : float, optional
             The interval on which to spin the client in the background thread
             (simply passed to time.sleep).
         
         Notes
         -----
         
         For precision timing, you may want to use this method to put a bound
         on the jitter (in seconds) in `received` timestamps used
         in AsyncResult.wall_time.
         
         """
         if self._spin_thread is not None:
             self.stop_spin_thread()
         self._stop_spinning.clear()
         self._spin_thread = Thread(target=self._spin_every, args=(interval,))
         self._spin_thread.daemon = True
         self._spin_thread.start()
     
     def stop_spin_thread(self):
         """stop background spin_thread, if any"""
         if self._spin_thread is not None:
             self._stop_spinning.set()
             self._spin_thread.join()
             self._spin_thread = None
 
     def spin(self):
         """Flush any registration notifications and execution results
         waiting in the ZMQ queue.
         """
         if self._notification_socket:
             self._flush_notifications()
         if self._iopub_socket:
             self._flush_iopub(self._iopub_socket)
         if self._mux_socket:
             self._flush_results(self._mux_socket)
         if self._task_socket:
             self._flush_results(self._task_socket)
         if self._control_socket:
             self._flush_control(self._control_socket)
         if self._query_socket:
             self._flush_ignored_hub_replies()
 
     def wait(self, jobs=None, timeout=-1):
         """waits on one or more `jobs`, for up to `timeout` seconds.
 
         Parameters
         ----------
 
         jobs : int, str, or list of ints and/or strs, or one or more AsyncResult objects
                 ints are indices to self.history
                 strs are msg_ids
                 default: wait on all outstanding messages
         timeout : float
                 a time in seconds, after which to give up.
                 default is -1, which means no timeout
 
         Returns
         -------
 
         True : when all msg_ids are done
         False : timeout reached, some msg_ids still outstanding
         """
         tic = time.time()
         if jobs is None:
             theids = self.outstanding
         else:
             if isinstance(jobs, string_types + (int, AsyncResult)):
                 jobs = [jobs]
             theids = set()
             for job in jobs:
                 if isinstance(job, int):
                     # index access
                     job = self.history[job]
                 elif isinstance(job, AsyncResult):
                     theids.update(job.msg_ids)
                     continue
                 theids.add(job)
         if not theids.intersection(self.outstanding):
             return True
         self.spin()
         while theids.intersection(self.outstanding):
             if timeout >= 0 and ( time.time()-tic ) > timeout:
                 break
             time.sleep(1e-3)
             self.spin()
         return len(theids.intersection(self.outstanding)) == 0
 
     #--------------------------------------------------------------------------
     # Control methods
     #--------------------------------------------------------------------------
 
     @spin_first
     def clear(self, targets=None, block=None):
         """Clear the namespace in target(s)."""
         block = self.block if block is None else block
         targets = self._build_targets(targets)[0]
         for t in targets:
             self.session.send(self._control_socket, 'clear_request', content={}, ident=t)
         error = False
         if block:
             self._flush_ignored_control()
             for i in range(len(targets)):
                 idents,msg = self.session.recv(self._control_socket,0)
                 if self.debug:
                     pprint(msg)
                 if msg['content']['status'] != 'ok':
                     error = self._unwrap_exception(msg['content'])
         else:
             self._ignored_control_replies += len(targets)
         if error:
             raise error
 
 
     @spin_first
     def abort(self, jobs=None, targets=None, block=None):
         """Abort specific jobs from the execution queues of target(s).
 
         This is a mechanism to prevent jobs that have already been submitted
         from executing.
 
         Parameters
         ----------
 
         jobs : msg_id, list of msg_ids, or AsyncResult
             The jobs to be aborted
             
             If unspecified/None: abort all outstanding jobs.
 
         """
         block = self.block if block is None else block
         jobs = jobs if jobs is not None else list(self.outstanding)
         targets = self._build_targets(targets)[0]
         
         msg_ids = []
         if isinstance(jobs, string_types + (AsyncResult,)):
             jobs = [jobs]
         bad_ids = [obj for obj in jobs if not isinstance(obj, string_types + (AsyncResult,))]
         if bad_ids:
             raise TypeError("Invalid msg_id type %r, expected str or AsyncResult"%bad_ids[0])
         for j in jobs:
             if isinstance(j, AsyncResult):
                 msg_ids.extend(j.msg_ids)
             else:
                 msg_ids.append(j)
         content = dict(msg_ids=msg_ids)
         for t in targets:
             self.session.send(self._control_socket, 'abort_request',
                     content=content, ident=t)
         error = False
         if block:
             self._flush_ignored_control()
             for i in range(len(targets)):
                 idents,msg = self.session.recv(self._control_socket,0)
                 if self.debug:
                     pprint(msg)
                 if msg['content']['status'] != 'ok':
                     error = self._unwrap_exception(msg['content'])
         else:
             self._ignored_control_replies += len(targets)
         if error:
             raise error
 
     @spin_first
     def shutdown(self, targets='all', restart=False, hub=False, block=None):
         """Terminates one or more engine processes, optionally including the hub.
         
         Parameters
         ----------
         
         targets: list of ints or 'all' [default: all]
             Which engines to shutdown.
         hub: bool [default: False]
             Whether to include the Hub.  hub=True implies targets='all'.
         block: bool [default: self.block]
             Whether to wait for clean shutdown replies or not.
         restart: bool [default: False]
             NOT IMPLEMENTED
             whether to restart engines after shutting them down.
         """
         from IPython.parallel.error import NoEnginesRegistered
         if restart:
             raise NotImplementedError("Engine restart is not yet implemented")
         
         block = self.block if block is None else block
         if hub:
             targets = 'all'
         try:
             targets = self._build_targets(targets)[0]
         except NoEnginesRegistered:
             targets = []
         for t in targets:
             self.session.send(self._control_socket, 'shutdown_request',
                         content={'restart':restart},ident=t)
         error = False
         if block or hub:
             self._flush_ignored_control()
             for i in range(len(targets)):
                 idents,msg = self.session.recv(self._control_socket, 0)
                 if self.debug:
                     pprint(msg)
                 if msg['content']['status'] != 'ok':
                     error = self._unwrap_exception(msg['content'])
         else:
             self._ignored_control_replies += len(targets)
 
         if hub:
             time.sleep(0.25)
             self.session.send(self._query_socket, 'shutdown_request')
             idents,msg = self.session.recv(self._query_socket, 0)
             if self.debug:
                 pprint(msg)
             if msg['content']['status'] != 'ok':
                 error = self._unwrap_exception(msg['content'])
 
         if error:
             raise error
 
     #--------------------------------------------------------------------------
     # Execution related methods
     #--------------------------------------------------------------------------
 
     def _maybe_raise(self, result):
         """wrapper for maybe raising an exception if apply failed."""
         if isinstance(result, error.RemoteError):
             raise result
 
         return result
 
     def send_apply_request(self, socket, f, args=None, kwargs=None, metadata=None, track=False,
                             ident=None):
         """construct and send an apply message via a socket.
 
         This is the principal method with which all engine execution is performed by views.
         """
 
         if self._closed:
             raise RuntimeError("Client cannot be used after its sockets have been closed")
         
         # defaults:
         args = args if args is not None else []
         kwargs = kwargs if kwargs is not None else {}
         metadata = metadata if metadata is not None else {}
 
         # validate arguments
         if not callable(f) and not isinstance(f, Reference):
             raise TypeError("f must be callable, not %s"%type(f))
         if not isinstance(args, (tuple, list)):
             raise TypeError("args must be tuple or list, not %s"%type(args))
         if not isinstance(kwargs, dict):
             raise TypeError("kwargs must be dict, not %s"%type(kwargs))
         if not isinstance(metadata, dict):
             raise TypeError("metadata must be dict, not %s"%type(metadata))
 
         bufs = serialize.pack_apply_message(f, args, kwargs,
             buffer_threshold=self.session.buffer_threshold,
             item_threshold=self.session.item_threshold,
         )
 
         msg = self.session.send(socket, "apply_request", buffers=bufs, ident=ident,
                             metadata=metadata, track=track)
 
         msg_id = msg['header']['msg_id']
         self.outstanding.add(msg_id)
         if ident:
             # possibly routed to a specific engine
             if isinstance(ident, list):
                 ident = ident[-1]
             if ident in self._engines.values():
                 # save for later, in case of engine death
                 self._outstanding_dict[ident].add(msg_id)
         self.history.append(msg_id)
         self.metadata[msg_id]['submitted'] = datetime.now()
 
         return msg
 
     def send_execute_request(self, socket, code, silent=True, metadata=None, ident=None):
         """construct and send an execute request via a socket.
 
         """
 
         if self._closed:
             raise RuntimeError("Client cannot be used after its sockets have been closed")
         
         # defaults:
         metadata = metadata if metadata is not None else {}
 
         # validate arguments
         if not isinstance(code, string_types):
             raise TypeError("code must be text, not %s" % type(code))
         if not isinstance(metadata, dict):
             raise TypeError("metadata must be dict, not %s" % type(metadata))
         
         content = dict(code=code, silent=bool(silent), user_expressions={})
 
 
         msg = self.session.send(socket, "execute_request", content=content, ident=ident,
                             metadata=metadata)
 
         msg_id = msg['header']['msg_id']
         self.outstanding.add(msg_id)
         if ident:
             # possibly routed to a specific engine
             if isinstance(ident, list):
                 ident = ident[-1]
             if ident in self._engines.values():
                 # save for later, in case of engine death
                 self._outstanding_dict[ident].add(msg_id)
         self.history.append(msg_id)
         self.metadata[msg_id]['submitted'] = datetime.now()
 
         return msg
 
     #--------------------------------------------------------------------------
     # construct a View object
     #--------------------------------------------------------------------------
 
     def load_balanced_view(self, targets=None):
         """construct a DirectView object.
 
         If no arguments are specified, create a LoadBalancedView
         using all engines.
 
         Parameters
         ----------
 
         targets: list,slice,int,etc. [default: use all engines]
             The subset of engines across which to load-balance
         """
         if targets == 'all':
             targets = None
         if targets is not None:
             targets = self._build_targets(targets)[1]
         return LoadBalancedView(client=self, socket=self._task_socket, targets=targets)
 
     def direct_view(self, targets='all'):
         """construct a DirectView object.
 
         If no targets are specified, create a DirectView using all engines.
         
         rc.direct_view('all') is distinguished from rc[:] in that 'all' will
         evaluate the target engines at each execution, whereas rc[:] will connect to
         all *current* engines, and that list will not change.
         
         That is, 'all' will always use all engines, whereas rc[:] will not use
         engines added after the DirectView is constructed.
 
         Parameters
         ----------
 
         targets: list,slice,int,etc. [default: use all engines]
             The engines to use for the View
         """
         single = isinstance(targets, int)
         # allow 'all' to be lazily evaluated at each execution
         if targets != 'all':
             targets = self._build_targets(targets)[1]
         if single:
             targets = targets[0]
         return DirectView(client=self, socket=self._mux_socket, targets=targets)
 
     #--------------------------------------------------------------------------
     # Query methods
     #--------------------------------------------------------------------------
 
     @spin_first
-    def get_result(self, indices_or_msg_ids=None, block=None):
+    def get_result(self, indices_or_msg_ids=None, block=None, owner=True):
         """Retrieve a result by msg_id or history index, wrapped in an AsyncResult object.
 
         If the client already has the results, no request to the Hub will be made.
 
         This is a convenient way to construct AsyncResult objects, which are wrappers
         that include metadata about execution, and allow for awaiting results that
         were not submitted by this Client.
 
         It can also be a convenient way to retrieve the metadata associated with
         blocking execution, since it always retrieves
 
         Examples
         --------
         ::
 
             In [10]: r = client.apply()
 
         Parameters
         ----------
 
         indices_or_msg_ids : integer history index, str msg_id, or list of either
             The indices or msg_ids of indices to be retrieved
 
         block : bool
             Whether to wait for the result to be done
+        owner : bool [default: True]
+            Whether this AsyncResult should own the result.
+            If so, calling `ar.get()` will remove data from the
+            client's result and metadata cache.
+            There should only be one owner of any given msg_id.
 
         Returns
         -------
 
         AsyncResult
             A single AsyncResult object will always be returned.
 
         AsyncHubResult
             A subclass of AsyncResult that retrieves results from the Hub
 
         """
         block = self.block if block is None else block
         if indices_or_msg_ids is None:
             indices_or_msg_ids = -1
         
         single_result = False
         if not isinstance(indices_or_msg_ids, (list,tuple)):
             indices_or_msg_ids = [indices_or_msg_ids]
             single_result = True
 
         theids = []
         for id in indices_or_msg_ids:
             if isinstance(id, int):
                 id = self.history[id]
             if not isinstance(id, string_types):
                 raise TypeError("indices must be str or int, not %r"%id)
             theids.append(id)
 
         local_ids = [msg_id for msg_id in theids if (msg_id in self.outstanding or msg_id in self.results)]
         remote_ids = [msg_id for msg_id in theids if msg_id not in local_ids]
         
         # given single msg_id initially, get_result shot get the result itself,
         # not a length-one list
         if single_result:
             theids = theids[0]
 
         if remote_ids:
-            ar = AsyncHubResult(self, msg_ids=theids)
+            ar = AsyncHubResult(self, msg_ids=theids, owner=owner)
         else:
-            ar = AsyncResult(self, msg_ids=theids)
+            ar = AsyncResult(self, msg_ids=theids, owner=owner)
 
         if block:
             ar.wait()
 
         return ar
 
     @spin_first
     def resubmit(self, indices_or_msg_ids=None, metadata=None, block=None):
         """Resubmit one or more tasks.
 
         in-flight tasks may not be resubmitted.
 
         Parameters
         ----------
 
         indices_or_msg_ids : integer history index, str msg_id, or list of either
             The indices or msg_ids of indices to be retrieved
 
         block : bool
             Whether to wait for the result to be done
 
         Returns
         -------
 
         AsyncHubResult
             A subclass of AsyncResult that retrieves results from the Hub
 
         """
         block = self.block if block is None else block
         if indices_or_msg_ids is None:
             indices_or_msg_ids = -1
 
         if not isinstance(indices_or_msg_ids, (list,tuple)):
             indices_or_msg_ids = [indices_or_msg_ids]
 
         theids = []
         for id in indices_or_msg_ids:
             if isinstance(id, int):
                 id = self.history[id]
             if not isinstance(id, string_types):
                 raise TypeError("indices must be str or int, not %r"%id)
             theids.append(id)
 
         content = dict(msg_ids = theids)
 
         self.session.send(self._query_socket, 'resubmit_request', content)
 
         zmq.select([self._query_socket], [], [])
         idents,msg = self.session.recv(self._query_socket, zmq.NOBLOCK)
         if self.debug:
             pprint(msg)
         content = msg['content']
         if content['status'] != 'ok':
             raise self._unwrap_exception(content)
         mapping = content['resubmitted']
         new_ids = [ mapping[msg_id] for msg_id in theids ]
 
         ar = AsyncHubResult(self, msg_ids=new_ids)
 
         if block:
             ar.wait()
 
         return ar
 
     @spin_first
     def result_status(self, msg_ids, status_only=True):
         """Check on the status of the result(s) of the apply request with `msg_ids`.
 
         If status_only is False, then the actual results will be retrieved, else
         only the status of the results will be checked.
 
         Parameters
         ----------
 
         msg_ids : list of msg_ids
             if int:
                 Passed as index to self.history for convenience.
         status_only : bool (default: True)
             if False:
                 Retrieve the actual results of completed tasks.
 
         Returns
         -------
 
         results : dict
             There will always be the keys 'pending' and 'completed', which will
             be lists of msg_ids that are incomplete or complete. If `status_only`
             is False, then completed results will be keyed by their `msg_id`.
         """
         if not isinstance(msg_ids, (list,tuple)):
             msg_ids = [msg_ids]
 
         theids = []
         for msg_id in msg_ids:
             if isinstance(msg_id, int):
                 msg_id = self.history[msg_id]
             if not isinstance(msg_id, string_types):
                 raise TypeError("msg_ids must be str, not %r"%msg_id)
             theids.append(msg_id)
 
         completed = []
         local_results = {}
 
         # comment this block out to temporarily disable local shortcut:
         for msg_id in theids:
             if msg_id in self.results:
                 completed.append(msg_id)
                 local_results[msg_id] = self.results[msg_id]
                 theids.remove(msg_id)
 
         if theids: # some not locally cached
             content = dict(msg_ids=theids, status_only=status_only)
             msg = self.session.send(self._query_socket, "result_request", content=content)
             zmq.select([self._query_socket], [], [])
             idents,msg = self.session.recv(self._query_socket, zmq.NOBLOCK)
             if self.debug:
                 pprint(msg)
             content = msg['content']
             if content['status'] != 'ok':
                 raise self._unwrap_exception(content)
             buffers = msg['buffers']
         else:
             content = dict(completed=[],pending=[])
 
         content['completed'].extend(completed)
 
         if status_only:
             return content
 
         failures = []
         # load cached results into result:
         content.update(local_results)
 
         # update cache with results:
         for msg_id in sorted(theids):
             if msg_id in content['completed']:
                 rec = content[msg_id]
                 parent = extract_dates(rec['header'])
                 header = extract_dates(rec['result_header'])
                 rcontent = rec['result_content']
                 iodict = rec['io']
                 if isinstance(rcontent, str):
                     rcontent = self.session.unpack(rcontent)
 
                 md = self.metadata[msg_id]
                 md_msg = dict(
                     content=rcontent,
                     parent_header=parent,
                     header=header,
                     metadata=rec['result_metadata'],
                 )
                 md.update(self._extract_metadata(md_msg))
                 if rec.get('received'):
                     md['received'] = parse_date(rec['received'])
                 md.update(iodict)
                 
                 if rcontent['status'] == 'ok':
                     if header['msg_type'] == 'apply_reply':
                         res,buffers = serialize.unserialize_object(buffers)
                     elif header['msg_type'] == 'execute_reply':
                         res = ExecuteReply(msg_id, rcontent, md)
                     else:
                         raise KeyError("unhandled msg type: %r" % header['msg_type'])
                 else:
                     res = self._unwrap_exception(rcontent)
                     failures.append(res)
 
                 self.results[msg_id] = res
                 content[msg_id] = res
 
         if len(theids) == 1 and failures:
             raise failures[0]
 
         error.collect_exceptions(failures, "result_status")
         return content
 
     @spin_first
     def queue_status(self, targets='all', verbose=False):
         """Fetch the status of engine queues.
 
         Parameters
         ----------
 
         targets : int/str/list of ints/strs
                 the engines whose states are to be queried.
                 default : all
         verbose : bool
                 Whether to return lengths only, or lists of ids for each element
         """
         if targets == 'all':
             # allow 'all' to be evaluated on the engine
             engine_ids = None
         else:
             engine_ids = self._build_targets(targets)[1]
         content = dict(targets=engine_ids, verbose=verbose)
         self.session.send(self._query_socket, "queue_request", content=content)
         idents,msg = self.session.recv(self._query_socket, 0)
         if self.debug:
             pprint(msg)
         content = msg['content']
         status = content.pop('status')
         if status != 'ok':
             raise self._unwrap_exception(content)
         content = rekey(content)
         if isinstance(targets, int):
             return content[targets]
         else:
             return content
 
     def _build_msgids_from_target(self, targets=None):
         """Build a list of msg_ids from the list of engine targets"""
         if not targets: # needed as _build_targets otherwise uses all engines
             return []
         target_ids = self._build_targets(targets)[0] 
         return [md_id for md_id in self.metadata if self.metadata[md_id]["engine_uuid"] in target_ids]
     
     def _build_msgids_from_jobs(self, jobs=None):
         """Build a list of msg_ids from "jobs" """
         if not jobs:
             return []
         msg_ids = []
         if isinstance(jobs, string_types + (AsyncResult,)):
             jobs = [jobs]
         bad_ids = [obj for obj in jobs if not isinstance(obj, string_types + (AsyncResult,))]
         if bad_ids:
             raise TypeError("Invalid msg_id type %r, expected str or AsyncResult"%bad_ids[0])
         for j in jobs:
             if isinstance(j, AsyncResult):
                 msg_ids.extend(j.msg_ids)
             else:
                 msg_ids.append(j)
         return msg_ids        
         
     def purge_local_results(self, jobs=[], targets=[]):
         """Clears the client caches of results and their metadata.
 
         Individual results can be purged by msg_id, or the entire
         history of specific targets can be purged.
 
         Use `purge_local_results('all')` to scrub everything from the Clients's
         results and metadata caches.
 
         After this call all `AsyncResults` are invalid and should be discarded.
 
         If you must "reget" the results, you can still do so by using
         `client.get_result(msg_id)` or `client.get_result(asyncresult)`. This will
         redownload the results from the hub if they are still available
         (i.e `client.purge_hub_results(...)` has not been called.        
 
         Parameters
         ----------
 
         jobs : str or list of str or AsyncResult objects
                 the msg_ids whose results should be purged.
         targets : int/list of ints
                 The engines, by integer ID, whose entire result histories are to be purged.
 
         Raises
         ------
 
         RuntimeError : if any of the tasks to be purged are still outstanding.
 
         """
         if not targets and not jobs:
             raise ValueError("Must specify at least one of `targets` and `jobs`")
         
         if jobs == 'all':
             if self.outstanding:
                 raise RuntimeError("Can't purge outstanding tasks: %s" % self.outstanding)
             self.results.clear()
             self.metadata.clear()
         else:
             msg_ids = set()
             msg_ids.update(self._build_msgids_from_target(targets))
             msg_ids.update(self._build_msgids_from_jobs(jobs))
             still_outstanding = self.outstanding.intersection(msg_ids)
             if still_outstanding:
                 raise RuntimeError("Can't purge outstanding tasks: %s" % still_outstanding)
             for mid in msg_ids:
-                self.results.pop(mid)
-                self.metadata.pop(mid)
+                self.results.pop(mid, None)
+                self.metadata.pop(mid, None)
 
 
     @spin_first
     def purge_hub_results(self, jobs=[], targets=[]):
         """Tell the Hub to forget results.
 
         Individual results can be purged by msg_id, or the entire
         history of specific targets can be purged.
 
         Use `purge_results('all')` to scrub everything from the Hub's db.
 
         Parameters
         ----------
 
         jobs : str or list of str or AsyncResult objects
                 the msg_ids whose results should be forgotten.
         targets : int/str/list of ints/strs
                 The targets, by int_id, whose entire history is to be purged.
 
                 default : None
         """
         if not targets and not jobs:
             raise ValueError("Must specify at least one of `targets` and `jobs`")
         if targets:
             targets = self._build_targets(targets)[1]
 
         # construct msg_ids from jobs
         if jobs == 'all':
             msg_ids = jobs
         else:
             msg_ids = self._build_msgids_from_jobs(jobs)
 
         content = dict(engine_ids=targets, msg_ids=msg_ids)
         self.session.send(self._query_socket, "purge_request", content=content)
         idents, msg = self.session.recv(self._query_socket, 0)
         if self.debug:
             pprint(msg)
         content = msg['content']
         if content['status'] != 'ok':
             raise self._unwrap_exception(content)
 
     def purge_results(self,  jobs=[], targets=[]):
         """Clears the cached results from both the hub and the local client
                 
         Individual results can be purged by msg_id, or the entire
         history of specific targets can be purged.
 
         Use `purge_results('all')` to scrub every cached result from both the Hub's and 
         the Client's db.
         
         Equivalent to calling both `purge_hub_results()` and `purge_client_results()` with 
         the same arguments.
 
         Parameters
         ----------
 
         jobs : str or list of str or AsyncResult objects
                 the msg_ids whose results should be forgotten.
         targets : int/str/list of ints/strs
                 The targets, by int_id, whose entire history is to be purged.
 
                 default : None
         """
         self.purge_local_results(jobs=jobs, targets=targets)
         self.purge_hub_results(jobs=jobs, targets=targets)
 
     def purge_everything(self):
         """Clears all content from previous Tasks from both the hub and the local client
         
         In addition to calling `purge_results("all")` it also deletes the history and 
         other bookkeeping lists.        
         """
         self.purge_results("all")
         self.history = []
         self.session.digest_history.clear()
 
     @spin_first
     def hub_history(self):
         """Get the Hub's history
 
         Just like the Client, the Hub has a history, which is a list of msg_ids.
         This will contain the history of all clients, and, depending on configuration,
         may contain history across multiple cluster sessions.
 
         Any msg_id returned here is a valid argument to `get_result`.
 
         Returns
         -------
 
         msg_ids : list of strs
                 list of all msg_ids, ordered by task submission time.
         """
 
         self.session.send(self._query_socket, "history_request", content={})
         idents, msg = self.session.recv(self._query_socket, 0)
 
         if self.debug:
             pprint(msg)
         content = msg['content']
         if content['status'] != 'ok':
             raise self._unwrap_exception(content)
         else:
             return content['history']
 
     @spin_first
     def db_query(self, query, keys=None):
         """Query the Hub's TaskRecord database
 
         This will return a list of task record dicts that match `query`
 
         Parameters
         ----------
 
         query : mongodb query dict
             The search dict. See mongodb query docs for details.
         keys : list of strs [optional]
             The subset of keys to be returned.  The default is to fetch everything but buffers.
             'msg_id' will *always* be included.
         """
         if isinstance(keys, string_types):
             keys = [keys]
         content = dict(query=query, keys=keys)
         self.session.send(self._query_socket, "db_request", content=content)
         idents, msg = self.session.recv(self._query_socket, 0)
         if self.debug:
             pprint(msg)
         content = msg['content']
         if content['status'] != 'ok':
             raise self._unwrap_exception(content)
 
         records = content['records']
 
         buffer_lens = content['buffer_lens']
         result_buffer_lens = content['result_buffer_lens']
         buffers = msg['buffers']
         has_bufs = buffer_lens is not None
         has_rbufs = result_buffer_lens is not None
         for i,rec in enumerate(records):
             # unpack datetime objects
             for hkey in ('header', 'result_header'):
                 if hkey in rec:
                     rec[hkey] = extract_dates(rec[hkey])
             for dtkey in ('submitted', 'started', 'completed', 'received'):
                 if dtkey in rec:
                     rec[dtkey] = parse_date(rec[dtkey])
             # relink buffers
             if has_bufs:
                 blen = buffer_lens[i]
                 rec['buffers'], buffers = buffers[:blen],buffers[blen:]
             if has_rbufs:
                 blen = result_buffer_lens[i]
                 rec['result_buffers'], buffers = buffers[:blen],buffers[blen:]
 
         return records
 
 __all__ = [ 'Client' ]

IPython/parallel/client/view.py

@@ -1,1131 +1,1125 b''
-"""Views of remote engines.
+"""Views of remote engines."""
 
-Authors:
+# Copyright (c) IPython Development Team.
+# Distributed under the terms of the Modified BSD License.
 
-* Min RK
-"""
 from __future__ import print_function
-#-----------------------------------------------------------------------------
-#  Copyright (C) 2010-2011  The IPython Development Team
-#
-#  Distributed under the terms of the BSD License.  The full license is in
-#  the file COPYING, distributed as part of this software.
-#-----------------------------------------------------------------------------
-
-#-----------------------------------------------------------------------------
-# Imports
-#-----------------------------------------------------------------------------
 
 import imp
 import sys
 import warnings
 from contextlib import contextmanager
 from types import ModuleType
 
 import zmq
 
 from IPython.testing.skipdoctest import skip_doctest
 from IPython.utils import pickleutil
 from IPython.utils.traitlets import (
     HasTraits, Any, Bool, List, Dict, Set, Instance, CFloat, Integer
 )
 from IPython.external.decorator import decorator
 
 from IPython.parallel import util
 from IPython.parallel.controller.dependency import Dependency, dependent
 from IPython.utils.py3compat import string_types, iteritems, PY3
 
 from . import map as Map
 from .asyncresult import AsyncResult, AsyncMapResult
 from .remotefunction import ParallelFunction, parallel, remote, getname
 
 #-----------------------------------------------------------------------------
 # Decorators
 #-----------------------------------------------------------------------------
 
 @decorator
 def save_ids(f, self, *args, **kwargs):
     """Keep our history and outstanding attributes up to date after a method call."""
     n_previous = len(self.client.history)
     try:
         ret = f(self, *args, **kwargs)
     finally:
         nmsgs = len(self.client.history) - n_previous
         msg_ids = self.client.history[-nmsgs:]
         self.history.extend(msg_ids)
         self.outstanding.update(msg_ids)
     return ret
 
 @decorator
 def sync_results(f, self, *args, **kwargs):
     """sync relevant results from self.client to our results attribute."""
     if self._in_sync_results:
         return f(self, *args, **kwargs)
     self._in_sync_results = True
     try:
         ret = f(self, *args, **kwargs)
     finally:
         self._in_sync_results = False
         self._sync_results()
     return ret
 
 @decorator
 def spin_after(f, self, *args, **kwargs):
     """call spin after the method."""
     ret = f(self, *args, **kwargs)
     self.spin()
     return ret
 
 #-----------------------------------------------------------------------------
 # Classes
 #-----------------------------------------------------------------------------
 
 @skip_doctest
 class View(HasTraits):
     """Base View class for more convenint apply(f,*args,**kwargs) syntax via attributes.
 
     Don't use this class, use subclasses.
 
     Methods
     -------
 
     spin
         flushes incoming results and registration state changes
         control methods spin, and requesting `ids` also ensures up to date
 
     wait
         wait on one or more msg_ids
 
     execution methods
         apply
         legacy: execute, run
 
     data movement
         push, pull, scatter, gather
 
     query methods
         get_result, queue_status, purge_results, result_status
 
     control methods
         abort, shutdown
 
     """
     # flags
     block=Bool(False)
     track=Bool(True)
     targets = Any()
 
     history=List()
     outstanding = Set()
     results = Dict()
     client = Instance('IPython.parallel.Client')
 
     _socket = Instance('zmq.Socket')
     _flag_names = List(['targets', 'block', 'track'])
     _in_sync_results = Bool(False)
     _targets = Any()
     _idents = Any()
 
     def __init__(self, client=None, socket=None, **flags):
         super(View, self).__init__(client=client, _socket=socket)
         self.results = client.results
         self.block = client.block
 
         self.set_flags(**flags)
 
         assert not self.__class__ is View, "Don't use base View objects, use subclasses"
 
     def __repr__(self):
         strtargets = str(self.targets)
         if len(strtargets) > 16:
             strtargets = strtargets[:12]+'...]'
         return "<%s %s>"%(self.__class__.__name__, strtargets)
     
     def __len__(self):
         if isinstance(self.targets, list):
             return len(self.targets)
         elif isinstance(self.targets, int):
             return 1
         else:
             return len(self.client)
     
     def set_flags(self, **kwargs):
         """set my attribute flags by keyword.
 
         Views determine behavior with a few attributes (`block`, `track`, etc.).
         These attributes can be set all at once by name with this method.
 
         Parameters
         ----------
 
         block : bool
             whether to wait for results
         track : bool
             whether to create a MessageTracker to allow the user to
             safely edit after arrays and buffers during non-copying
             sends.
         """
         for name, value in iteritems(kwargs):
             if name not in self._flag_names:
                 raise KeyError("Invalid name: %r"%name)
             else:
                 setattr(self, name, value)
 
     @contextmanager
     def temp_flags(self, **kwargs):
         """temporarily set flags, for use in `with` statements.
 
         See set_flags for permanent setting of flags
 
         Examples
         --------
 
         >>> view.track=False
         ...
         >>> with view.temp_flags(track=True):
         ...    ar = view.apply(dostuff, my_big_array)
         ...    ar.tracker.wait() # wait for send to finish
         >>> view.track
         False
 
         """
         # preflight: save flags, and set temporaries
         saved_flags = {}
         for f in self._flag_names:
             saved_flags[f] = getattr(self, f)
         self.set_flags(**kwargs)
         # yield to the with-statement block
         try:
             yield
         finally:
             # postflight: restore saved flags
             self.set_flags(**saved_flags)
 
 
     #----------------------------------------------------------------
     # apply
     #----------------------------------------------------------------
 
     def _sync_results(self):
         """to be called by @sync_results decorator
         
         after submitting any tasks.
         """
         delta = self.outstanding.difference(self.client.outstanding)
         completed = self.outstanding.intersection(delta)
         self.outstanding = self.outstanding.difference(completed)
         
     @sync_results
     @save_ids
     def _really_apply(self, f, args, kwargs, block=None, **options):
         """wrapper for client.send_apply_request"""
         raise NotImplementedError("Implement in subclasses")
 
     def apply(self, f, *args, **kwargs):
         """calls ``f(*args, **kwargs)`` on remote engines, returning the result.
 
         This method sets all apply flags via this View's attributes.
 
         Returns :class:`~IPython.parallel.client.asyncresult.AsyncResult`
         instance if ``self.block`` is False, otherwise the return value of
         ``f(*args, **kwargs)``.
         """
         return self._really_apply(f, args, kwargs)
 
     def apply_async(self, f, *args, **kwargs):
         """calls ``f(*args, **kwargs)`` on remote engines in a nonblocking manner.
 
         Returns :class:`~IPython.parallel.client.asyncresult.AsyncResult` instance.
         """
         return self._really_apply(f, args, kwargs, block=False)
 
     @spin_after
     def apply_sync(self, f, *args, **kwargs):
         """calls ``f(*args, **kwargs)`` on remote engines in a blocking manner,
          returning the result.
         """
         return self._really_apply(f, args, kwargs, block=True)
 
     #----------------------------------------------------------------
     # wrappers for client and control methods
     #----------------------------------------------------------------
     @sync_results
     def spin(self):
         """spin the client, and sync"""
         self.client.spin()
 
     @sync_results
     def wait(self, jobs=None, timeout=-1):
         """waits on one or more `jobs`, for up to `timeout` seconds.
 
         Parameters
         ----------
 
         jobs : int, str, or list of ints and/or strs, or one or more AsyncResult objects
                 ints are indices to self.history
                 strs are msg_ids
                 default: wait on all outstanding messages
         timeout : float
                 a time in seconds, after which to give up.
                 default is -1, which means no timeout
 
         Returns
         -------
 
         True : when all msg_ids are done
         False : timeout reached, some msg_ids still outstanding
         """
         if jobs is None:
             jobs = self.history
         return self.client.wait(jobs, timeout)
 
     def abort(self, jobs=None, targets=None, block=None):
         """Abort jobs on my engines.
 
         Parameters
         ----------
 
         jobs : None, str, list of strs, optional
             if None: abort all jobs.
             else: abort specific msg_id(s).
         """
         block = block if block is not None else self.block
         targets = targets if targets is not None else self.targets
         jobs = jobs if jobs is not None else list(self.outstanding)
         
         return self.client.abort(jobs=jobs, targets=targets, block=block)
 
     def queue_status(self, targets=None, verbose=False):
         """Fetch the Queue status of my engines"""
         targets = targets if targets is not None else self.targets
         return self.client.queue_status(targets=targets, verbose=verbose)
 
     def purge_results(self, jobs=[], targets=[]):
         """Instruct the controller to forget specific results."""
         if targets is None or targets == 'all':
             targets = self.targets
         return self.client.purge_results(jobs=jobs, targets=targets)
 
     def shutdown(self, targets=None, restart=False, hub=False, block=None):
         """Terminates one or more engine processes, optionally including the hub.
         """
         block = self.block if block is None else block
         if targets is None or targets == 'all':
             targets = self.targets
         return self.client.shutdown(targets=targets, restart=restart, hub=hub, block=block)
 
     @spin_after
-    def get_result(self, indices_or_msg_ids=None):
+    def get_result(self, indices_or_msg_ids=None, block=None, owner=True):
         """return one or more results, specified by history index or msg_id.
 
         See :meth:`IPython.parallel.client.client.Client.get_result` for details.
         """
 
         if indices_or_msg_ids is None:
             indices_or_msg_ids = -1
         if isinstance(indices_or_msg_ids, int):
             indices_or_msg_ids = self.history[indices_or_msg_ids]
         elif isinstance(indices_or_msg_ids, (list,tuple,set)):
             indices_or_msg_ids = list(indices_or_msg_ids)
             for i,index in enumerate(indices_or_msg_ids):
                 if isinstance(index, int):
                     indices_or_msg_ids[i] = self.history[index]
-        return self.client.get_result(indices_or_msg_ids)
+        return self.client.get_result(indices_or_msg_ids, block=block, owner=owner)
 
     #-------------------------------------------------------------------
     # Map
     #-------------------------------------------------------------------
 
     @sync_results
     def map(self, f, *sequences, **kwargs):
         """override in subclasses"""
         raise NotImplementedError
 
     def map_async(self, f, *sequences, **kwargs):
         """Parallel version of builtin :func:`python:map`, using this view's engines.
 
         This is equivalent to ``map(...block=False)``.
 
         See `self.map` for details.
         """
         if 'block' in kwargs:
             raise TypeError("map_async doesn't take a `block` keyword argument.")
         kwargs['block'] = False
         return self.map(f,*sequences,**kwargs)
 
     def map_sync(self, f, *sequences, **kwargs):
         """Parallel version of builtin :func:`python:map`, using this view's engines.
 
         This is equivalent to ``map(...block=True)``.
 
         See `self.map` for details.
         """
         if 'block' in kwargs:
             raise TypeError("map_sync doesn't take a `block` keyword argument.")
         kwargs['block'] = True
         return self.map(f,*sequences,**kwargs)
 
     def imap(self, f, *sequences, **kwargs):
         """Parallel version of :func:`itertools.imap`.
 
         See `self.map` for details.
 
         """
 
         return iter(self.map_async(f,*sequences, **kwargs))
 
     #-------------------------------------------------------------------
     # Decorators
     #-------------------------------------------------------------------
 
     def remote(self, block=None, **flags):
         """Decorator for making a RemoteFunction"""
         block = self.block if block is None else block
         return remote(self, block=block, **flags)
 
     def parallel(self, dist='b', block=None, **flags):
         """Decorator for making a ParallelFunction"""
         block = self.block if block is None else block
         return parallel(self, dist=dist, block=block, **flags)
 
 @skip_doctest
 class DirectView(View):
     """Direct Multiplexer View of one or more engines.
 
     These are created via indexed access to a client:
 
     >>> dv_1 = client[1]
     >>> dv_all = client[:]
     >>> dv_even = client[::2]
     >>> dv_some = client[1:3]
 
     This object provides dictionary access to engine namespaces:
 
     # push a=5:
     >>> dv['a'] = 5
     # pull 'foo':
     >>> dv['foo']
 
     """
 
     def __init__(self, client=None, socket=None, targets=None):
         super(DirectView, self).__init__(client=client, socket=socket, targets=targets)
 
     @property
     def importer(self):
         """sync_imports(local=True) as a property.
 
         See sync_imports for details.
 
         """
         return self.sync_imports(True)
 
     @contextmanager
     def sync_imports(self, local=True, quiet=False):
         """Context Manager for performing simultaneous local and remote imports.
 
         'import x as y' will *not* work.  The 'as y' part will simply be ignored.
         
         If `local=True`, then the package will also be imported locally.
         
         If `quiet=True`, no output will be produced when attempting remote 
         imports. 
         
         Note that remote-only (`local=False`) imports have not been implemented.
         
         >>> with view.sync_imports():
         ...    from numpy import recarray
         importing recarray from numpy on engine(s)
 
         """
         from IPython.utils.py3compat import builtin_mod
         local_import = builtin_mod.__import__
         modules = set()
         results = []
         @util.interactive
         def remote_import(name, fromlist, level):
             """the function to be passed to apply, that actually performs the import
             on the engine, and loads up the user namespace.
             """
             import sys
             user_ns = globals()
             mod = __import__(name, fromlist=fromlist, level=level)
             if fromlist:
                 for key in fromlist:
                     user_ns[key] = getattr(mod, key)
             else:
                 user_ns[name] = sys.modules[name]
 
         def view_import(name, globals={}, locals={}, fromlist=[], level=0):
             """the drop-in replacement for __import__, that optionally imports
             locally as well.
             """
             # don't override nested imports
             save_import = builtin_mod.__import__
             builtin_mod.__import__ = local_import
 
             if imp.lock_held():
                 # this is a side-effect import, don't do it remotely, or even
                 # ignore the local effects
                 return local_import(name, globals, locals, fromlist, level)
 
             imp.acquire_lock()
             if local:
                 mod = local_import(name, globals, locals, fromlist, level)
             else:
                 raise NotImplementedError("remote-only imports not yet implemented")
             imp.release_lock()
 
             key = name+':'+','.join(fromlist or [])
             if level <= 0 and key not in modules:
                 modules.add(key)
                 if not quiet:
                     if fromlist:
                         print("importing %s from %s on engine(s)"%(','.join(fromlist), name))
                     else:
                         print("importing %s on engine(s)"%name)
                 results.append(self.apply_async(remote_import, name, fromlist, level))
             # restore override
             builtin_mod.__import__ = save_import
 
             return mod
 
         # override __import__
         builtin_mod.__import__ = view_import
         try:
             # enter the block
             yield
         except ImportError:
             if local:
                 raise
             else:
                 # ignore import errors if not doing local imports
                 pass
         finally:
             # always restore __import__
             builtin_mod.__import__ = local_import
 
         for r in results:
             # raise possible remote ImportErrors here
             r.get()
     
     def use_dill(self):
         """Expand serialization support with dill
         
         adds support for closures, etc.
         
         This calls IPython.utils.pickleutil.use_dill() here and on each engine.
         """
         pickleutil.use_dill()
         return self.apply(pickleutil.use_dill)
 
     def use_cloudpickle(self):
         """Expand serialization support with cloudpickle.
         """
         pickleutil.use_cloudpickle()
         return self.apply(pickleutil.use_cloudpickle)
 
 
     @sync_results
     @save_ids
     def _really_apply(self, f, args=None, kwargs=None, targets=None, block=None, track=None):
         """calls f(*args, **kwargs) on remote engines, returning the result.
 
         This method sets all of `apply`'s flags via this View's attributes.
 
         Parameters
         ----------
 
         f : callable
 
         args : list [default: empty]
 
         kwargs : dict [default: empty]
 
         targets : target list [default: self.targets]
             where to run
         block : bool [default: self.block]
             whether to block
         track : bool [default: self.track]
             whether to ask zmq to track the message, for safe non-copying sends
 
         Returns
         -------
 
         if self.block is False:
             returns AsyncResult
         else:
             returns actual result of f(*args, **kwargs) on the engine(s)
             This will be a list of self.targets is also a list (even length 1), or
             the single result if self.targets is an integer engine id
         """
         args = [] if args is None else args
         kwargs = {} if kwargs is None else kwargs
         block = self.block if block is None else block
         track = self.track if track is None else track
         targets = self.targets if targets is None else targets
         
         _idents, _targets = self.client._build_targets(targets)
         msg_ids = []
         trackers = []
         for ident in _idents:
             msg = self.client.send_apply_request(self._socket, f, args, kwargs, track=track,
                                     ident=ident)
             if track:
                 trackers.append(msg['tracker'])
             msg_ids.append(msg['header']['msg_id'])
         if isinstance(targets, int):
             msg_ids = msg_ids[0]
         tracker = None if track is False else zmq.MessageTracker(*trackers)
-        ar = AsyncResult(self.client, msg_ids, fname=getname(f), targets=_targets, tracker=tracker)
+        ar = AsyncResult(self.client, msg_ids, fname=getname(f), targets=_targets,
+            tracker=tracker, owner=True,
+        )
         if block:
             try:
                 return ar.get()
             except KeyboardInterrupt:
                 pass
         return ar
 
 
     @sync_results
     def map(self, f, *sequences, **kwargs):
         """``view.map(f, *sequences, block=self.block)`` => list|AsyncMapResult
 
         Parallel version of builtin `map`, using this View's `targets`.
 
         There will be one task per target, so work will be chunked
         if the sequences are longer than `targets`.
 
         Results can be iterated as they are ready, but will become available in chunks.
 
         Parameters
         ----------
 
         f : callable
             function to be mapped
         *sequences: one or more sequences of matching length
             the sequences to be distributed and passed to `f`
         block : bool
             whether to wait for the result or not [default self.block]
 
         Returns
         -------
 
 
         If block=False
           An :class:`~IPython.parallel.client.asyncresult.AsyncMapResult` instance.
           An object like AsyncResult, but which reassembles the sequence of results
           into a single list. AsyncMapResults can be iterated through before all
           results are complete.
         else
             A list, the result of ``map(f,*sequences)``
         """
 
         block = kwargs.pop('block', self.block)
         for k in kwargs.keys():
             if k not in ['block', 'track']:
                 raise TypeError("invalid keyword arg, %r"%k)
 
         assert len(sequences) > 0, "must have some sequences to map onto!"
         pf = ParallelFunction(self, f, block=block, **kwargs)
         return pf.map(*sequences)
 
     @sync_results
     @save_ids
     def execute(self, code, silent=True, targets=None, block=None):
         """Executes `code` on `targets` in blocking or nonblocking manner.
 
         ``execute`` is always `bound` (affects engine namespace)
 
         Parameters
         ----------
 
         code : str
                 the code string to be executed
         block : bool
                 whether or not to wait until done to return
                 default: self.block
         """
         block = self.block if block is None else block
         targets = self.targets if targets is None else targets
 
         _idents, _targets = self.client._build_targets(targets)
         msg_ids = []
         trackers = []
         for ident in _idents:
             msg = self.client.send_execute_request(self._socket, code, silent=silent, ident=ident)
             msg_ids.append(msg['header']['msg_id'])
         if isinstance(targets, int):
             msg_ids = msg_ids[0]
-        ar = AsyncResult(self.client, msg_ids, fname='execute', targets=_targets)
+        ar = AsyncResult(self.client, msg_ids, fname='execute', targets=_targets, owner=True)
         if block:
             try:
                 ar.get()
             except KeyboardInterrupt:
                 pass
         return ar
 
     def run(self, filename, targets=None, block=None):
         """Execute contents of `filename` on my engine(s).
 
         This simply reads the contents of the file and calls `execute`.
 
         Parameters
         ----------
 
         filename : str
                 The path to the file
         targets : int/str/list of ints/strs
                 the engines on which to execute
                 default : all
         block : bool
                 whether or not to wait until done
                 default: self.block
 
         """
         with open(filename, 'r') as f:
             # add newline in case of trailing indented whitespace
             # which will cause SyntaxError
             code = f.read()+'\n'
         return self.execute(code, block=block, targets=targets)
 
     def update(self, ns):
         """update remote namespace with dict `ns`
 
         See `push` for details.
         """
         return self.push(ns, block=self.block, track=self.track)
 
     def push(self, ns, targets=None, block=None, track=None):
         """update remote namespace with dict `ns`
 
         Parameters
         ----------
 
         ns : dict
             dict of keys with which to update engine namespace(s)
         block : bool [default : self.block]
             whether to wait to be notified of engine receipt
 
         """
 
         block = block if block is not None else self.block
         track = track if track is not None else self.track
         targets = targets if targets is not None else self.targets
         # applier = self.apply_sync if block else self.apply_async
         if not isinstance(ns, dict):
             raise TypeError("Must be a dict, not %s"%type(ns))
         return self._really_apply(util._push, kwargs=ns, block=block, track=track, targets=targets)
 
     def get(self, key_s):
         """get object(s) by `key_s` from remote namespace
 
         see `pull` for details.
         """
         # block = block if block is not None else self.block
         return self.pull(key_s, block=True)
 
     def pull(self, names, targets=None, block=None):
         """get object(s) by `name` from remote namespace
 
         will return one object if it is a key.
         can also take a list of keys, in which case it will return a list of objects.
         """
         block = block if block is not None else self.block
         targets = targets if targets is not None else self.targets
         applier = self.apply_sync if block else self.apply_async
         if isinstance(names, string_types):
             pass
         elif isinstance(names, (list,tuple,set)):
             for key in names:
                 if not isinstance(key, string_types):
                     raise TypeError("keys must be str, not type %r"%type(key))
         else:
             raise TypeError("names must be strs, not %r"%names)
         return self._really_apply(util._pull, (names,), block=block, targets=targets)
 
     def scatter(self, key, seq, dist='b', flatten=False, targets=None, block=None, track=None):
         """
         Partition a Python sequence and send the partitions to a set of engines.
         """
         block = block if block is not None else self.block
         track = track if track is not None else self.track
         targets = targets if targets is not None else self.targets
         
         # construct integer ID list:
         targets = self.client._build_targets(targets)[1]
 
         mapObject = Map.dists[dist]()
         nparts = len(targets)
         msg_ids = []
         trackers = []
         for index, engineid in enumerate(targets):
             partition = mapObject.getPartition(seq, index, nparts)
             if flatten and len(partition) == 1:
                 ns = {key: partition[0]}
             else:
                 ns = {key: partition}
             r = self.push(ns, block=False, track=track, targets=engineid)
             msg_ids.extend(r.msg_ids)
             if track:
                 trackers.append(r._tracker)
 
         if track:
             tracker = zmq.MessageTracker(*trackers)
         else:
             tracker = None
 
-        r = AsyncResult(self.client, msg_ids, fname='scatter', targets=targets, tracker=tracker)
+        r = AsyncResult(self.client, msg_ids, fname='scatter', targets=targets,
+            tracker=tracker, owner=True,
+        )
         if block:
             r.wait()
         else:
             return r
 
     @sync_results
     @save_ids
     def gather(self, key, dist='b', targets=None, block=None):
         """
         Gather a partitioned sequence on a set of engines as a single local seq.
         """
         block = block if block is not None else self.block
         targets = targets if targets is not None else self.targets
         mapObject = Map.dists[dist]()
         msg_ids = []
 
         # construct integer ID list:
         targets = self.client._build_targets(targets)[1]
         
         for index, engineid in enumerate(targets):
             msg_ids.extend(self.pull(key, block=False, targets=engineid).msg_ids)
 
         r = AsyncMapResult(self.client, msg_ids, mapObject, fname='gather')
 
         if block:
             try:
                 return r.get()
             except KeyboardInterrupt:
                 pass
         return r
 
     def __getitem__(self, key):
         return self.get(key)
 
     def __setitem__(self,key, value):
         self.update({key:value})
 
     def clear(self, targets=None, block=None):
         """Clear the remote namespaces on my engines."""
         block = block if block is not None else self.block
         targets = targets if targets is not None else self.targets
         return self.client.clear(targets=targets, block=block)
 
     #----------------------------------------
     # activate for %px, %autopx, etc. magics
     #----------------------------------------
 
     def activate(self, suffix=''):
         """Activate IPython magics associated with this View
         
         Defines the magics `%px, %autopx, %pxresult, %%px, %pxconfig`
         
         Parameters
         ----------
         
         suffix: str [default: '']
             The suffix, if any, for the magics.  This allows you to have
             multiple views associated with parallel magics at the same time.
             
             e.g. ``rc[::2].activate(suffix='_even')`` will give you
             the magics ``%px_even``, ``%pxresult_even``, etc. for running magics 
             on the even engines.
         """
         
         from IPython.parallel.client.magics import ParallelMagics
         
         try:
             # This is injected into __builtins__.
             ip = get_ipython()
         except NameError:
             print("The IPython parallel magics (%px, etc.) only work within IPython.")
             return
         
         M = ParallelMagics(ip, self, suffix)
         ip.magics_manager.register(M)
 
 
 @skip_doctest
 class LoadBalancedView(View):
     """An load-balancing View that only executes via the Task scheduler.
 
     Load-balanced views can be created with the client's `view` method:
 
     >>> v = client.load_balanced_view()
 
     or targets can be specified, to restrict the potential destinations:
 
     >>> v = client.client.load_balanced_view([1,3])
 
     which would restrict loadbalancing to between engines 1 and 3.
 
     """
 
     follow=Any()
     after=Any()
     timeout=CFloat()
     retries = Integer(0)
 
     _task_scheme = Any()
     _flag_names = List(['targets', 'block', 'track', 'follow', 'after', 'timeout', 'retries'])
 
     def __init__(self, client=None, socket=None, **flags):
         super(LoadBalancedView, self).__init__(client=client, socket=socket, **flags)
         self._task_scheme=client._task_scheme
 
     def _validate_dependency(self, dep):
         """validate a dependency.
 
         For use in `set_flags`.
         """
         if dep is None or isinstance(dep, string_types + (AsyncResult, Dependency)):
             return True
         elif isinstance(dep, (list,set, tuple)):
             for d in dep:
                 if not isinstance(d, string_types + (AsyncResult,)):
                     return False
         elif isinstance(dep, dict):
             if set(dep.keys()) != set(Dependency().as_dict().keys()):
                 return False
             if not isinstance(dep['msg_ids'], list):
                 return False
             for d in dep['msg_ids']:
                 if not isinstance(d, string_types):
                     return False
         else:
             return False
 
         return True
 
     def _render_dependency(self, dep):
         """helper for building jsonable dependencies from various input forms."""
         if isinstance(dep, Dependency):
             return dep.as_dict()
         elif isinstance(dep, AsyncResult):
             return dep.msg_ids
         elif dep is None:
             return []
         else:
             # pass to Dependency constructor
             return list(Dependency(dep))
 
     def set_flags(self, **kwargs):
         """set my attribute flags by keyword.
 
         A View is a wrapper for the Client's apply method, but with attributes
         that specify keyword arguments, those attributes can be set by keyword
         argument with this method.
 
         Parameters
         ----------
 
         block : bool
             whether to wait for results
         track : bool
             whether to create a MessageTracker to allow the user to
             safely edit after arrays and buffers during non-copying
             sends.
 
         after : Dependency or collection of msg_ids
             Only for load-balanced execution (targets=None)
             Specify a list of msg_ids as a time-based dependency.
             This job will only be run *after* the dependencies
             have been met.
 
         follow : Dependency or collection of msg_ids
             Only for load-balanced execution (targets=None)
             Specify a list of msg_ids as a location-based dependency.
             This job will only be run on an engine where this dependency
             is met.
 
         timeout : float/int or None
             Only for load-balanced execution (targets=None)
             Specify an amount of time (in seconds) for the scheduler to
             wait for dependencies to be met before failing with a
             DependencyTimeout.
 
         retries : int
             Number of times a task will be retried on failure.
         """
 
         super(LoadBalancedView, self).set_flags(**kwargs)
         for name in ('follow', 'after'):
             if name in kwargs:
                 value = kwargs[name]
                 if self._validate_dependency(value):
                     setattr(self, name, value)
                 else:
                     raise ValueError("Invalid dependency: %r"%value)
         if 'timeout' in kwargs:
             t = kwargs['timeout']
             if not isinstance(t, (int, float, type(None))):
                 if (not PY3) and (not isinstance(t, long)):
                     raise TypeError("Invalid type for timeout: %r"%type(t))
             if t is not None:
                 if t < 0:
                     raise ValueError("Invalid timeout: %s"%t)
             self.timeout = t
 
     @sync_results
     @save_ids
     def _really_apply(self, f, args=None, kwargs=None, block=None, track=None,
                                         after=None, follow=None, timeout=None,
                                         targets=None, retries=None):
         """calls f(*args, **kwargs) on a remote engine, returning the result.
 
         This method temporarily sets all of `apply`'s flags for a single call.
 
         Parameters
         ----------
 
         f : callable
 
         args : list [default: empty]
 
         kwargs : dict [default: empty]
 
         block : bool [default: self.block]
             whether to block
         track : bool [default: self.track]
             whether to ask zmq to track the message, for safe non-copying sends
 
         !!!!!! TODO: THE REST HERE  !!!!
 
         Returns
         -------
 
         if self.block is False:
             returns AsyncResult
         else:
             returns actual result of f(*args, **kwargs) on the engine(s)
             This will be a list of self.targets is also a list (even length 1), or
             the single result if self.targets is an integer engine id
         """
 
         # validate whether we can run
         if self._socket.closed:
             msg = "Task farming is disabled"
             if self._task_scheme == 'pure':
                 msg += " because the pure ZMQ scheduler cannot handle"
                 msg += " disappearing engines."
             raise RuntimeError(msg)
 
         if self._task_scheme == 'pure':
             # pure zmq scheme doesn't support extra features
             msg = "Pure ZMQ scheduler doesn't support the following flags:"
             "follow, after, retries, targets, timeout"
             if (follow or after or retries or targets or timeout):
                 # hard fail on Scheduler flags
                 raise RuntimeError(msg)
             if isinstance(f, dependent):
                 # soft warn on functional dependencies
                 warnings.warn(msg, RuntimeWarning)
 
         # build args
         args = [] if args is None else args
         kwargs = {} if kwargs is None else kwargs
         block = self.block if block is None else block
         track = self.track if track is None else track
         after = self.after if after is None else after
         retries = self.retries if retries is None else retries
         follow = self.follow if follow is None else follow
         timeout = self.timeout if timeout is None else timeout
         targets = self.targets if targets is None else targets
 
         if not isinstance(retries, int):
             raise TypeError('retries must be int, not %r'%type(retries))
 
         if targets is None:
             idents = []
         else:
             idents = self.client._build_targets(targets)[0]
             # ensure *not* bytes
             idents = [ ident.decode() for ident in idents ]
 
         after = self._render_dependency(after)
         follow = self._render_dependency(follow)
         metadata = dict(after=after, follow=follow, timeout=timeout, targets=idents, retries=retries)
 
         msg = self.client.send_apply_request(self._socket, f, args, kwargs, track=track,
                                 metadata=metadata)
         tracker = None if track is False else msg['tracker']
 
-        ar = AsyncResult(self.client, msg['header']['msg_id'], fname=getname(f), targets=None, tracker=tracker)
-
+        ar = AsyncResult(self.client, msg['header']['msg_id'], fname=getname(f),
+            targets=None, tracker=tracker, owner=True,
+        )
         if block:
             try:
                 return ar.get()
             except KeyboardInterrupt:
                 pass
         return ar
 
     @sync_results
     @save_ids
     def map(self, f, *sequences, **kwargs):
         """``view.map(f, *sequences, block=self.block, chunksize=1, ordered=True)`` => list|AsyncMapResult
 
         Parallel version of builtin `map`, load-balanced by this View.
 
         `block`, and `chunksize` can be specified by keyword only.
 
         Each `chunksize` elements will be a separate task, and will be
         load-balanced. This lets individual elements be available for iteration
         as soon as they arrive.
 
         Parameters
         ----------
 
         f : callable
             function to be mapped
         *sequences: one or more sequences of matching length
             the sequences to be distributed and passed to `f`
         block : bool [default self.block]
             whether to wait for the result or not
         track : bool
             whether to create a MessageTracker to allow the user to
             safely edit after arrays and buffers during non-copying
             sends.
         chunksize : int [default 1]
             how many elements should be in each task.
         ordered : bool [default True]
             Whether the results should be gathered as they arrive, or enforce
             the order of submission.
             
             Only applies when iterating through AsyncMapResult as results arrive.
             Has no effect when block=True.
 
         Returns
         -------
 
         if block=False
           An :class:`~IPython.parallel.client.asyncresult.AsyncMapResult` instance.
           An object like AsyncResult, but which reassembles the sequence of results
           into a single list. AsyncMapResults can be iterated through before all
           results are complete.
         else
             A list, the result of ``map(f,*sequences)``
         """
 
         # default
         block = kwargs.get('block', self.block)
         chunksize = kwargs.get('chunksize', 1)
         ordered = kwargs.get('ordered', True)
 
         keyset = set(kwargs.keys())
         extra_keys = keyset.difference_update(set(['block', 'chunksize']))
         if extra_keys:
             raise TypeError("Invalid kwargs: %s"%list(extra_keys))
 
         assert len(sequences) > 0, "must have some sequences to map onto!"
 
         pf = ParallelFunction(self, f, block=block, chunksize=chunksize, ordered=ordered)
         return pf.map(*sequences)
 
 __all__ = ['LoadBalancedView', 'DirectView']

IPython/parallel/tests/test_asyncresult.py

@@ -1,327 +1,342 b''
-"""Tests for asyncresult.py
+"""Tests for asyncresult.py"""
 
-Authors:
-
-* Min RK
-"""
-
-#-------------------------------------------------------------------------------
-#  Copyright (C) 2011  The IPython Development Team
-#
-#  Distributed under the terms of the BSD License.  The full license is in
-#  the file COPYING, distributed as part of this software.
-#-------------------------------------------------------------------------------
-
-#-------------------------------------------------------------------------------
-# Imports
-#-------------------------------------------------------------------------------
+# Copyright (c) IPython Development Team.
+# Distributed under the terms of the Modified BSD License.
 
 import time
 
 import nose.tools as nt
 
 from IPython.utils.io import capture_output
 
 from IPython.parallel.error import TimeoutError
 from IPython.parallel import error, Client
 from IPython.parallel.tests import add_engines
 from .clienttest import ClusterTestCase
 from IPython.utils.py3compat import iteritems
 
 def setup():
     add_engines(2, total=True)
 
 def wait(n):
     import time
     time.sleep(n)
     return n
 
 def echo(x):
     return x
 
 class AsyncResultTest(ClusterTestCase):
 
     def test_single_result_view(self):
         """various one-target views get the right value for single_result"""
         eid = self.client.ids[-1]
         ar = self.client[eid].apply_async(lambda : 42)
         self.assertEqual(ar.get(), 42)
         ar = self.client[[eid]].apply_async(lambda : 42)
         self.assertEqual(ar.get(), [42])
         ar = self.client[-1:].apply_async(lambda : 42)
         self.assertEqual(ar.get(), [42])
 
     def test_get_after_done(self):
         ar = self.client[-1].apply_async(lambda : 42)
         ar.wait()
         self.assertTrue(ar.ready())
         self.assertEqual(ar.get(), 42)
         self.assertEqual(ar.get(), 42)
 
     def test_get_before_done(self):
         ar = self.client[-1].apply_async(wait, 0.1)
         self.assertRaises(TimeoutError, ar.get, 0)
         ar.wait(0)
         self.assertFalse(ar.ready())
         self.assertEqual(ar.get(), 0.1)
 
     def test_get_after_error(self):
         ar = self.client[-1].apply_async(lambda : 1/0)
         ar.wait(10)
         self.assertRaisesRemote(ZeroDivisionError, ar.get)
         self.assertRaisesRemote(ZeroDivisionError, ar.get)
         self.assertRaisesRemote(ZeroDivisionError, ar.get_dict)
     
     def test_get_dict(self):
         n = len(self.client)
         ar = self.client[:].apply_async(lambda : 5)
         self.assertEqual(ar.get(), [5]*n)
         d = ar.get_dict()
         self.assertEqual(sorted(d.keys()), sorted(self.client.ids))
         for eid,r in iteritems(d):
             self.assertEqual(r, 5)
     
     def test_get_dict_single(self):
         view = self.client[-1]
         for v in (list(range(5)), 5, ('abc', 'def'), 'string'):
             ar = view.apply_async(echo, v)
             self.assertEqual(ar.get(), v)
             d = ar.get_dict()
             self.assertEqual(d, {view.targets : v})
     
     def test_get_dict_bad(self):
         ar = self.client[:].apply_async(lambda : 5)
         ar2 = self.client[:].apply_async(lambda : 5)
         ar = self.client.get_result(ar.msg_ids + ar2.msg_ids)
         self.assertRaises(ValueError, ar.get_dict)
     
     def test_list_amr(self):
         ar = self.client.load_balanced_view().map_async(wait, [0.1]*5)
         rlist = list(ar)
     
     def test_getattr(self):
         ar = self.client[:].apply_async(wait, 0.5)
         self.assertEqual(ar.engine_id, [None] * len(ar))
         self.assertRaises(AttributeError, lambda : ar._foo)
         self.assertRaises(AttributeError, lambda : ar.__length_hint__())
         self.assertRaises(AttributeError, lambda : ar.foo)
         self.assertFalse(hasattr(ar, '__length_hint__'))
         self.assertFalse(hasattr(ar, 'foo'))
         self.assertTrue(hasattr(ar, 'engine_id'))
         ar.get(5)
         self.assertRaises(AttributeError, lambda : ar._foo)
         self.assertRaises(AttributeError, lambda : ar.__length_hint__())
         self.assertRaises(AttributeError, lambda : ar.foo)
         self.assertTrue(isinstance(ar.engine_id, list))
         self.assertEqual(ar.engine_id, ar['engine_id'])
         self.assertFalse(hasattr(ar, '__length_hint__'))
         self.assertFalse(hasattr(ar, 'foo'))
         self.assertTrue(hasattr(ar, 'engine_id'))
 
     def test_getitem(self):
         ar = self.client[:].apply_async(wait, 0.5)
         self.assertEqual(ar['engine_id'], [None] * len(ar))
         self.assertRaises(KeyError, lambda : ar['foo'])
         ar.get(5)
         self.assertRaises(KeyError, lambda : ar['foo'])
         self.assertTrue(isinstance(ar['engine_id'], list))
         self.assertEqual(ar.engine_id, ar['engine_id'])
     
     def test_single_result(self):
         ar = self.client[-1].apply_async(wait, 0.5)
         self.assertRaises(KeyError, lambda : ar['foo'])
         self.assertEqual(ar['engine_id'], None)
         self.assertTrue(ar.get(5) == 0.5)
         self.assertTrue(isinstance(ar['engine_id'], int))
         self.assertTrue(isinstance(ar.engine_id, int))
         self.assertEqual(ar.engine_id, ar['engine_id'])
     
     def test_abort(self):
         e = self.client[-1]
         ar = e.execute('import time; time.sleep(1)', block=False)
         ar2 = e.apply_async(lambda : 2)
         ar2.abort()
         self.assertRaises(error.TaskAborted, ar2.get)
         ar.get()
     
     def test_len(self):
         v = self.client.load_balanced_view()
         ar = v.map_async(lambda x: x, list(range(10)))
         self.assertEqual(len(ar), 10)
         ar = v.apply_async(lambda x: x, list(range(10)))
         self.assertEqual(len(ar), 1)
         ar = self.client[:].apply_async(lambda x: x, list(range(10)))
         self.assertEqual(len(ar), len(self.client.ids))
     
     def test_wall_time_single(self):
         v = self.client.load_balanced_view()
         ar = v.apply_async(time.sleep, 0.25)
         self.assertRaises(TimeoutError, getattr, ar, 'wall_time')
         ar.get(2)
         self.assertTrue(ar.wall_time < 1.)
         self.assertTrue(ar.wall_time > 0.2)
 
     def test_wall_time_multi(self):
         self.minimum_engines(4)
         v = self.client[:]
         ar = v.apply_async(time.sleep, 0.25)
         self.assertRaises(TimeoutError, getattr, ar, 'wall_time')
         ar.get(2)
         self.assertTrue(ar.wall_time < 1.)
         self.assertTrue(ar.wall_time > 0.2)
 
     def test_serial_time_single(self):
         v = self.client.load_balanced_view()
         ar = v.apply_async(time.sleep, 0.25)
         self.assertRaises(TimeoutError, getattr, ar, 'serial_time')
         ar.get(2)
         self.assertTrue(ar.serial_time < 1.)
         self.assertTrue(ar.serial_time > 0.2)
 
     def test_serial_time_multi(self):
         self.minimum_engines(4)
         v = self.client[:]
         ar = v.apply_async(time.sleep, 0.25)
         self.assertRaises(TimeoutError, getattr, ar, 'serial_time')
         ar.get(2)
         self.assertTrue(ar.serial_time < 2.)
         self.assertTrue(ar.serial_time > 0.8)
 
     def test_elapsed_single(self):
         v = self.client.load_balanced_view()
         ar = v.apply_async(time.sleep, 0.25)
         while not ar.ready():
             time.sleep(0.01)
             self.assertTrue(ar.elapsed < 1)
         self.assertTrue(ar.elapsed < 1)
         ar.get(2)
 
     def test_elapsed_multi(self):
         v = self.client[:]
         ar = v.apply_async(time.sleep, 0.25)
         while not ar.ready():
             time.sleep(0.01)
             self.assertTrue(ar.elapsed < 1)
         self.assertTrue(ar.elapsed < 1)
         ar.get(2)
 
     def test_hubresult_timestamps(self):
         self.minimum_engines(4)
         v = self.client[:]
         ar = v.apply_async(time.sleep, 0.25)
         ar.get(2)
         rc2 = Client(profile='iptest')
         # must have try/finally to close second Client, otherwise
         # will have dangling sockets causing problems
         try:
             time.sleep(0.25)
             hr = rc2.get_result(ar.msg_ids)
             self.assertTrue(hr.elapsed > 0., "got bad elapsed: %s" % hr.elapsed)
             hr.get(1)
             self.assertTrue(hr.wall_time < ar.wall_time + 0.2, "got bad wall_time: %s > %s" % (hr.wall_time, ar.wall_time))
             self.assertEqual(hr.serial_time, ar.serial_time)
         finally:
             rc2.close()
 
     def test_display_empty_streams_single(self):
         """empty stdout/err are not displayed (single result)"""
         self.minimum_engines(1)
         
         v = self.client[-1]
         ar = v.execute("print (5555)")
         ar.get(5)
         with capture_output() as io:
             ar.display_outputs()
         self.assertEqual(io.stderr, '')
         self.assertEqual('5555\n', io.stdout)
 
         ar = v.execute("a=5")
         ar.get(5)
         with capture_output() as io:
             ar.display_outputs()
         self.assertEqual(io.stderr, '')
         self.assertEqual(io.stdout, '')
         
     def test_display_empty_streams_type(self):
         """empty stdout/err are not displayed (groupby type)"""
         self.minimum_engines(1)
         
         v = self.client[:]
         ar = v.execute("print (5555)")
         ar.get(5)
         with capture_output() as io:
             ar.display_outputs()
         self.assertEqual(io.stderr, '')
         self.assertEqual(io.stdout.count('5555'), len(v), io.stdout)
         self.assertFalse('\n\n' in io.stdout, io.stdout)
         self.assertEqual(io.stdout.count('[stdout:'), len(v), io.stdout)
 
         ar = v.execute("a=5")
         ar.get(5)
         with capture_output() as io:
             ar.display_outputs()
         self.assertEqual(io.stderr, '')
         self.assertEqual(io.stdout, '')
 
     def test_display_empty_streams_engine(self):
         """empty stdout/err are not displayed (groupby engine)"""
         self.minimum_engines(1)
         
         v = self.client[:]
         ar = v.execute("print (5555)")
         ar.get(5)
         with capture_output() as io:
             ar.display_outputs('engine')
         self.assertEqual(io.stderr, '')
         self.assertEqual(io.stdout.count('5555'), len(v), io.stdout)
         self.assertFalse('\n\n' in io.stdout, io.stdout)
         self.assertEqual(io.stdout.count('[stdout:'), len(v), io.stdout)
 
         ar = v.execute("a=5")
         ar.get(5)
         with capture_output() as io:
             ar.display_outputs('engine')
         self.assertEqual(io.stderr, '')
         self.assertEqual(io.stdout, '')
     
     def test_await_data(self):
         """asking for ar.data flushes outputs"""
         self.minimum_engines(1)
         
         v = self.client[-1]
         ar = v.execute('\n'.join([
             "import time",
             "from IPython.kernel.zmq.datapub import publish_data",
             "for i in range(5):",
             "    publish_data(dict(i=i))",
             "    time.sleep(0.1)",
         ]), block=False)
         found = set()
         tic = time.time()
         # timeout after 10s
         while time.time() <= tic + 10:
             if ar.data:
                 i = ar.data['i']
                 found.add(i)
                 if i == 4:
                     break
             time.sleep(0.05)
         
         ar.get(5)
         nt.assert_in(4, found)
         self.assertTrue(len(found) > 1, "should have seen data multiple times, but got: %s" % found)
     
     def test_not_single_result(self):
         save_build = self.client._build_targets
         def single_engine(*a, **kw):
             idents, targets = save_build(*a, **kw)
             return idents[:1], targets[:1]
         ids = single_engine('all')[1]
         self.client._build_targets = single_engine
         for targets in ('all', None, ids):
             dv = self.client.direct_view(targets=targets)
             ar = dv.apply_async(lambda : 5)
             self.assertEqual(ar.get(10), [5])
         self.client._build_targets = save_build
 
+    def test_owner_pop(self):
+        self.minimum_engines(1)
+        
+        view = self.client[-1]
+        ar = view.apply_async(lambda : 1)
+        ar.get()
+        msg_id = ar.msg_ids[0]
+        self.assertNotIn(msg_id, self.client.results)
+        self.assertNotIn(msg_id, self.client.metadata)
+
+    def test_non_owner(self):
+        self.minimum_engines(1)
+        
+        view = self.client[-1]
+        ar = view.apply_async(lambda : 1)
+        ar.owner = False
+        ar.get()
+        msg_id = ar.msg_ids[0]
+        self.assertIn(msg_id, self.client.results)
+        self.assertIn(msg_id, self.client.metadata)
+        ar2 = self.client.get_result(msg_id, owner=True)
+        self.assertIs(type(ar2), type(ar))
+        self.assertTrue(ar2.owner)
+        self.assertEqual(ar.get(), ar2.get())
+        ar2.get()
+        self.assertNotIn(msg_id, self.client.results)
+        self.assertNotIn(msg_id, self.client.metadata)
+
 

IPython/parallel/tests/test_client.py

@@ -1,547 +1,550 b''
 """Tests for parallel client.py"""
 
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 from __future__ import division
 
 import time
 from datetime import datetime
 
 import zmq
 
 from IPython import parallel
 from IPython.parallel.client import client as clientmod
 from IPython.parallel import error
 from IPython.parallel import AsyncResult, AsyncHubResult
 from IPython.parallel import LoadBalancedView, DirectView
 
 from .clienttest import ClusterTestCase, segfault, wait, add_engines
 
 def setup():
     add_engines(4, total=True)
 
 class TestClient(ClusterTestCase):
     
     def test_ids(self):
         n = len(self.client.ids)
         self.add_engines(2)
         self.assertEqual(len(self.client.ids), n+2)
     
     def test_iter(self):
         self.minimum_engines(4)
         engine_ids = [ view.targets for view in self.client ]
         self.assertEqual(engine_ids, self.client.ids)
     
     def test_view_indexing(self):
         """test index access for views"""
         self.minimum_engines(4)
         targets = self.client._build_targets('all')[-1]
         v = self.client[:]
         self.assertEqual(v.targets, targets)
         t = self.client.ids[2]
         v = self.client[t]
         self.assertTrue(isinstance(v, DirectView))
         self.assertEqual(v.targets, t)
         t = self.client.ids[2:4]
         v = self.client[t]
         self.assertTrue(isinstance(v, DirectView))
         self.assertEqual(v.targets, t)
         v = self.client[::2]
         self.assertTrue(isinstance(v, DirectView))
         self.assertEqual(v.targets, targets[::2])
         v = self.client[1::3]
         self.assertTrue(isinstance(v, DirectView))
         self.assertEqual(v.targets, targets[1::3])
         v = self.client[:-3]
         self.assertTrue(isinstance(v, DirectView))
         self.assertEqual(v.targets, targets[:-3])
         v = self.client[-1]
         self.assertTrue(isinstance(v, DirectView))
         self.assertEqual(v.targets, targets[-1])
         self.assertRaises(TypeError, lambda : self.client[None])
     
     def test_lbview_targets(self):
         """test load_balanced_view targets"""
         v = self.client.load_balanced_view()
         self.assertEqual(v.targets, None)
         v = self.client.load_balanced_view(-1)
         self.assertEqual(v.targets, [self.client.ids[-1]])
         v = self.client.load_balanced_view('all')
         self.assertEqual(v.targets, None)
     
     def test_dview_targets(self):
         """test direct_view targets"""
         v = self.client.direct_view()
         self.assertEqual(v.targets, 'all')
         v = self.client.direct_view('all')
         self.assertEqual(v.targets, 'all')
         v = self.client.direct_view(-1)
         self.assertEqual(v.targets, self.client.ids[-1])
     
     def test_lazy_all_targets(self):
         """test lazy evaluation of rc.direct_view('all')"""
         v = self.client.direct_view()
         self.assertEqual(v.targets, 'all')
         
         def double(x):
             return x*2
         seq = list(range(100))
         ref = [ double(x) for x in seq ]
         
         # add some engines, which should be used
         self.add_engines(1)
         n1 = len(self.client.ids)
         
         # simple apply
         r = v.apply_sync(lambda : 1)
         self.assertEqual(r, [1] * n1)
         
         # map goes through remotefunction
         r = v.map_sync(double, seq)
         self.assertEqual(r, ref)
 
         # add a couple more engines, and try again
         self.add_engines(2)
         n2 = len(self.client.ids)
         self.assertNotEqual(n2, n1)
         
         # apply
         r = v.apply_sync(lambda : 1)
         self.assertEqual(r, [1] * n2)
         
         # map
         r = v.map_sync(double, seq)
         self.assertEqual(r, ref)
     
     def test_targets(self):
         """test various valid targets arguments"""
         build = self.client._build_targets
         ids = self.client.ids
         idents,targets = build(None)
         self.assertEqual(ids, targets)
     
     def test_clear(self):
         """test clear behavior"""
         self.minimum_engines(2)
         v = self.client[:]
         v.block=True
         v.push(dict(a=5))
         v.pull('a')
         id0 = self.client.ids[-1]
         self.client.clear(targets=id0, block=True)
         a = self.client[:-1].get('a')
         self.assertRaisesRemote(NameError, self.client[id0].get, 'a')
         self.client.clear(block=True)
         for i in self.client.ids:
             self.assertRaisesRemote(NameError, self.client[i].get, 'a')
     
     def test_get_result(self):
         """test getting results from the Hub."""
         c = clientmod.Client(profile='iptest')
         t = c.ids[-1]
         ar = c[t].apply_async(wait, 1)
         # give the monitor time to notice the message
         time.sleep(.25)
-        ahr = self.client.get_result(ar.msg_ids[0])
-        self.assertTrue(isinstance(ahr, AsyncHubResult))
+        ahr = self.client.get_result(ar.msg_ids[0], owner=False)
+        self.assertIsInstance(ahr, AsyncHubResult)
         self.assertEqual(ahr.get(), ar.get())
         ar2 = self.client.get_result(ar.msg_ids[0])
-        self.assertFalse(isinstance(ar2, AsyncHubResult))
+        self.assertNotIsInstance(ar2, AsyncHubResult)
+        self.assertEqual(ahr.get(), ar2.get())
         c.close()
     
     def test_get_execute_result(self):
         """test getting execute results from the Hub."""
         c = clientmod.Client(profile='iptest')
         t = c.ids[-1]
         cell = '\n'.join([
             'import time',
             'time.sleep(0.25)',
             '5'
         ])
         ar = c[t].execute("import time; time.sleep(1)", silent=False)
         # give the monitor time to notice the message
         time.sleep(.25)
-        ahr = self.client.get_result(ar.msg_ids[0])
-        self.assertTrue(isinstance(ahr, AsyncHubResult))
+        ahr = self.client.get_result(ar.msg_ids[0], owner=False)
+        self.assertIsInstance(ahr, AsyncHubResult)
         self.assertEqual(ahr.get().execute_result, ar.get().execute_result)
         ar2 = self.client.get_result(ar.msg_ids[0])
-        self.assertFalse(isinstance(ar2, AsyncHubResult))
+        self.assertNotIsInstance(ar2, AsyncHubResult)
+        self.assertEqual(ahr.get(), ar2.get())
         c.close()
     
     def test_ids_list(self):
         """test client.ids"""
         ids = self.client.ids
         self.assertEqual(ids, self.client._ids)
         self.assertFalse(ids is self.client._ids)
         ids.remove(ids[-1])
         self.assertNotEqual(ids, self.client._ids)
     
     def test_queue_status(self):
         ids = self.client.ids
         id0 = ids[0]
         qs = self.client.queue_status(targets=id0)
         self.assertTrue(isinstance(qs, dict))
         self.assertEqual(sorted(qs.keys()), ['completed', 'queue', 'tasks'])
         allqs = self.client.queue_status()
         self.assertTrue(isinstance(allqs, dict))
         intkeys = list(allqs.keys())
         intkeys.remove('unassigned')
         print("intkeys", intkeys)
         intkeys = sorted(intkeys)
         ids = self.client.ids
         print("client.ids", ids)
         ids = sorted(self.client.ids)
         self.assertEqual(intkeys, ids)
         unassigned = allqs.pop('unassigned')
         for eid,qs in allqs.items():
             self.assertTrue(isinstance(qs, dict))
             self.assertEqual(sorted(qs.keys()), ['completed', 'queue', 'tasks'])
 
     def test_shutdown(self):
         ids = self.client.ids
         id0 = ids[0]
         self.client.shutdown(id0, block=True)
         while id0 in self.client.ids:
             time.sleep(0.1)
             self.client.spin()
         
         self.assertRaises(IndexError, lambda : self.client[id0])
         
     def test_result_status(self):
         pass
         # to be written
     
     def test_db_query_dt(self):
         """test db query by date"""
         hist = self.client.hub_history()
         middle = self.client.db_query({'msg_id' : hist[len(hist)//2]})[0]
         tic = middle['submitted']
         before = self.client.db_query({'submitted' : {'$lt' : tic}})
         after = self.client.db_query({'submitted' : {'$gte' : tic}})
         self.assertEqual(len(before)+len(after),len(hist))
         for b in before:
             self.assertTrue(b['submitted'] < tic)
         for a in after:
             self.assertTrue(a['submitted'] >= tic)
         same = self.client.db_query({'submitted' : tic})
         for s in same:
             self.assertTrue(s['submitted'] == tic)
     
     def test_db_query_keys(self):
         """test extracting subset of record keys"""
         found = self.client.db_query({'msg_id': {'$ne' : ''}},keys=['submitted', 'completed'])
         for rec in found:
             self.assertEqual(set(rec.keys()), set(['msg_id', 'submitted', 'completed']))
     
     def test_db_query_default_keys(self):
         """default db_query excludes buffers"""
         found = self.client.db_query({'msg_id': {'$ne' : ''}})
         for rec in found:
             keys = set(rec.keys())
             self.assertFalse('buffers' in keys, "'buffers' should not be in: %s" % keys)
             self.assertFalse('result_buffers' in keys, "'result_buffers' should not be in: %s" % keys)
     
     def test_db_query_msg_id(self):
         """ensure msg_id is always in db queries"""
         found = self.client.db_query({'msg_id': {'$ne' : ''}},keys=['submitted', 'completed'])
         for rec in found:
             self.assertTrue('msg_id' in rec.keys())
         found = self.client.db_query({'msg_id': {'$ne' : ''}},keys=['submitted'])
         for rec in found:
             self.assertTrue('msg_id' in rec.keys())
         found = self.client.db_query({'msg_id': {'$ne' : ''}},keys=['msg_id'])
         for rec in found:
             self.assertTrue('msg_id' in rec.keys())
     
     def test_db_query_get_result(self):
         """pop in db_query shouldn't pop from result itself"""
         self.client[:].apply_sync(lambda : 1)
         found = self.client.db_query({'msg_id': {'$ne' : ''}})
         rc2 = clientmod.Client(profile='iptest')
         # If this bug is not fixed, this call will hang:
         ar = rc2.get_result(self.client.history[-1])
         ar.wait(2)
         self.assertTrue(ar.ready())
         ar.get()
         rc2.close()
     
     def test_db_query_in(self):
         """test db query with '$in','$nin' operators"""
         hist = self.client.hub_history()
         even = hist[::2]
         odd = hist[1::2]
         recs = self.client.db_query({ 'msg_id' : {'$in' : even}})
         found = [ r['msg_id'] for r in recs ]
         self.assertEqual(set(even), set(found))
         recs = self.client.db_query({ 'msg_id' : {'$nin' : even}})
         found = [ r['msg_id'] for r in recs ]
         self.assertEqual(set(odd), set(found))
     
     def test_hub_history(self):
         hist = self.client.hub_history()
         recs = self.client.db_query({ 'msg_id' : {"$ne":''}})
         recdict = {}
         for rec in recs:
             recdict[rec['msg_id']] = rec
         
         latest = datetime(1984,1,1)
         for msg_id in hist:
             rec = recdict[msg_id]
             newt = rec['submitted']
             self.assertTrue(newt >= latest)
             latest = newt
         ar = self.client[-1].apply_async(lambda : 1)
         ar.get()
         time.sleep(0.25)
         self.assertEqual(self.client.hub_history()[-1:],ar.msg_ids)
     
     def _wait_for_idle(self):
         """wait for the cluster to become idle, according to the everyone."""
         rc = self.client
         
         # step 0. wait for local results
         # this should be sufficient 99% of the time.
         rc.wait(timeout=5)
         
         # step 1. wait for all requests to be noticed
         # timeout 5s, polling every 100ms
         msg_ids = set(rc.history)
         hub_hist = rc.hub_history()
         for i in range(50):
             if msg_ids.difference(hub_hist):
                 time.sleep(0.1)
                 hub_hist = rc.hub_history()
             else:
                 break
         
         self.assertEqual(len(msg_ids.difference(hub_hist)), 0)
         
         # step 2. wait for all requests to be done
         # timeout 5s, polling every 100ms
         qs = rc.queue_status()
         for i in range(50):
             if qs['unassigned'] or any(qs[eid]['tasks'] + qs[eid]['queue'] for eid in qs if eid != 'unassigned'):
                 time.sleep(0.1)
                 qs = rc.queue_status()
             else:
                 break
         
         # ensure Hub up to date:
         self.assertEqual(qs['unassigned'], 0)
         for eid in [ eid for eid in qs if eid != 'unassigned' ]:
             self.assertEqual(qs[eid]['tasks'], 0)
             self.assertEqual(qs[eid]['queue'], 0)
     
     
     def test_resubmit(self):
         def f():
             import random
             return random.random()
         v = self.client.load_balanced_view()
         ar = v.apply_async(f)
         r1 = ar.get(1)
         # give the Hub a chance to notice:
         self._wait_for_idle()
         ahr = self.client.resubmit(ar.msg_ids)
         r2 = ahr.get(1)
         self.assertFalse(r1 == r2)
 
     def test_resubmit_chain(self):
         """resubmit resubmitted tasks"""
         v = self.client.load_balanced_view()
         ar = v.apply_async(lambda x: x, 'x'*1024)
         ar.get()
         self._wait_for_idle()
         ars = [ar]
         
         for i in range(10):
             ar = ars[-1]
             ar2 = self.client.resubmit(ar.msg_ids)
         
         [ ar.get() for ar in ars ]
 
     def test_resubmit_header(self):
         """resubmit shouldn't clobber the whole header"""
         def f():
             import random
             return random.random()
         v = self.client.load_balanced_view()
         v.retries = 1
         ar = v.apply_async(f)
         r1 = ar.get(1)
         # give the Hub a chance to notice:
         self._wait_for_idle()
         ahr = self.client.resubmit(ar.msg_ids)
         ahr.get(1)
         time.sleep(0.5)
         records = self.client.db_query({'msg_id': {'$in': ar.msg_ids + ahr.msg_ids}}, keys='header')
         h1,h2 = [ r['header'] for r in records ]
         for key in set(h1.keys()).union(set(h2.keys())):
             if key in ('msg_id', 'date'):
                 self.assertNotEqual(h1[key], h2[key])
             else:
                 self.assertEqual(h1[key], h2[key])
 
     def test_resubmit_aborted(self):
         def f():
             import random
             return random.random()
         v = self.client.load_balanced_view()
         # restrict to one engine, so we can put a sleep
         # ahead of the task, so it will get aborted
         eid = self.client.ids[-1]
         v.targets = [eid]
         sleep = v.apply_async(time.sleep, 0.5)
         ar = v.apply_async(f)
         ar.abort()
         self.assertRaises(error.TaskAborted, ar.get)
         # Give the Hub a chance to get up to date:
         self._wait_for_idle()
         ahr = self.client.resubmit(ar.msg_ids)
         r2 = ahr.get(1)
 
     def test_resubmit_inflight(self):
         """resubmit of inflight task"""
         v = self.client.load_balanced_view()
         ar = v.apply_async(time.sleep,1)
         # give the message a chance to arrive
         time.sleep(0.2)
         ahr = self.client.resubmit(ar.msg_ids)
         ar.get(2)
         ahr.get(2)
 
     def test_resubmit_badkey(self):
         """ensure KeyError on resubmit of nonexistant task"""
         self.assertRaisesRemote(KeyError, self.client.resubmit, ['invalid'])
 
     def test_purge_hub_results(self):
         # ensure there are some tasks
         for i in range(5):
             self.client[:].apply_sync(lambda : 1)
         # Wait for the Hub to realise the result is done:
         # This prevents a race condition, where we
         # might purge a result the Hub still thinks is pending.
         self._wait_for_idle()
         rc2 = clientmod.Client(profile='iptest')
         hist = self.client.hub_history()
         ahr = rc2.get_result([hist[-1]])
         ahr.wait(10)
         self.client.purge_hub_results(hist[-1])
         newhist = self.client.hub_history()
         self.assertEqual(len(newhist)+1,len(hist))
         rc2.spin()
         rc2.close()
 
     def test_purge_local_results(self):
         # ensure there are some tasks
         res = []
         for i in range(5):
             res.append(self.client[:].apply_async(lambda : 1))
         self._wait_for_idle()
         self.client.wait(10) # wait for the results to come back
         before = len(self.client.results)
         self.assertEqual(len(self.client.metadata),before)
         self.client.purge_local_results(res[-1])
         self.assertEqual(len(self.client.results),before-len(res[-1]), msg="Not removed from results")
         self.assertEqual(len(self.client.metadata),before-len(res[-1]), msg="Not removed from metadata")
     
     def test_purge_local_results_outstanding(self):
         v = self.client[-1]
         ar = v.apply_async(lambda : 1)
         msg_id = ar.msg_ids[0]
+        ar.owner = False
         ar.get()
         self._wait_for_idle()
         ar2 = v.apply_async(time.sleep, 1)
         self.assertIn(msg_id, self.client.results)
         self.assertIn(msg_id, self.client.metadata)
         self.client.purge_local_results(ar)
         self.assertNotIn(msg_id, self.client.results)
         self.assertNotIn(msg_id, self.client.metadata)
         with self.assertRaises(RuntimeError):
             self.client.purge_local_results(ar2)
         ar2.get()
         self.client.purge_local_results(ar2)
     
     def test_purge_all_local_results_outstanding(self):
         v = self.client[-1]
         ar = v.apply_async(time.sleep, 1)
         with self.assertRaises(RuntimeError):
             self.client.purge_local_results('all')
         ar.get()
         self.client.purge_local_results('all')
     
     def test_purge_all_hub_results(self):
         self.client.purge_hub_results('all')
         hist = self.client.hub_history()
         self.assertEqual(len(hist), 0)
 
     def test_purge_all_local_results(self):
         self.client.purge_local_results('all')
         self.assertEqual(len(self.client.results), 0, msg="Results not empty")
         self.assertEqual(len(self.client.metadata), 0, msg="metadata not empty")
 
     def test_purge_all_results(self):
         # ensure there are some tasks
         for i in range(5):
             self.client[:].apply_sync(lambda : 1)
         self.client.wait(10)
         self._wait_for_idle()
         self.client.purge_results('all')
         self.assertEqual(len(self.client.results), 0, msg="Results not empty")
         self.assertEqual(len(self.client.metadata), 0, msg="metadata not empty")
         hist = self.client.hub_history()
         self.assertEqual(len(hist), 0, msg="hub history not empty")
         
     def test_purge_everything(self):
         # ensure there are some tasks
         for i in range(5):
             self.client[:].apply_sync(lambda : 1)
         self.client.wait(10)
         self._wait_for_idle()
         self.client.purge_everything()
         # The client results
         self.assertEqual(len(self.client.results), 0, msg="Results not empty")
         self.assertEqual(len(self.client.metadata), 0, msg="metadata not empty")
         # The client "bookkeeping"
         self.assertEqual(len(self.client.session.digest_history), 0, msg="session digest not empty")
         self.assertEqual(len(self.client.history), 0, msg="client history not empty")
         # the hub results
         hist = self.client.hub_history()
         self.assertEqual(len(hist), 0, msg="hub history not empty")
         
     
     def test_spin_thread(self):
         self.client.spin_thread(0.01)
         ar = self.client[-1].apply_async(lambda : 1)
         md = self.client.metadata[ar.msg_ids[0]]
         # 3s timeout, 100ms poll
         for i in range(30):
             time.sleep(0.1)
             if md['received'] is not None:
                 break
         self.assertIsInstance(md['received'], datetime)
     
     def test_stop_spin_thread(self):
         self.client.spin_thread(0.01)
         self.client.stop_spin_thread()
         ar = self.client[-1].apply_async(lambda : 1)
         md = self.client.metadata[ar.msg_ids[0]]
         # 500ms timeout, 100ms poll
         for i in range(5):
             time.sleep(0.1)
             self.assertIsNone(md['received'], None)
     
     def test_activate(self):
         ip = get_ipython()
         magics = ip.magics_manager.magics
         self.assertTrue('px' in magics['line'])
         self.assertTrue('px' in magics['cell'])
         v0 = self.client.activate(-1, '0')
         self.assertTrue('px0' in magics['line'])
         self.assertTrue('px0' in magics['cell'])
         self.assertEqual(v0.targets, self.client.ids[-1])
         v0 = self.client.activate('all', 'all')
         self.assertTrue('pxall' in magics['line'])
         self.assertTrue('pxall' in magics['cell'])
         self.assertEqual(v0.targets, 'all')

IPython/parallel/tests/test_view.py

@@ -1,842 +1,843 b''
 # -*- coding: utf-8 -*-
 """test View objects"""
 
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 import base64
 import sys
 import platform
 import time
 from collections import namedtuple
 from tempfile import NamedTemporaryFile
 
 import zmq
 from nose.plugins.attrib import attr
 
 from IPython.testing import decorators as dec
 from IPython.utils.io import capture_output
 from IPython.utils.py3compat import unicode_type
 
 from IPython import parallel  as pmod
 from IPython.parallel import error
 from IPython.parallel import AsyncResult, AsyncHubResult, AsyncMapResult
 from IPython.parallel.util import interactive
 
 from IPython.parallel.tests import add_engines
 
 from .clienttest import ClusterTestCase, crash, wait, skip_without
 
 def setup():
     add_engines(3, total=True)
 
 point = namedtuple("point", "x y")
 
 class TestView(ClusterTestCase):
     
     def setUp(self):
         # On Win XP, wait for resource cleanup, else parallel test group fails
         if platform.system() == "Windows" and platform.win32_ver()[0] == "XP":
             # 1 sec fails. 1.5 sec seems ok. Using 2 sec for margin of safety
             time.sleep(2)
         super(TestView, self).setUp()
 
     @attr('crash')
     def test_z_crash_mux(self):
         """test graceful handling of engine death (direct)"""
         # self.add_engines(1)
         eid = self.client.ids[-1]
         ar = self.client[eid].apply_async(crash)
         self.assertRaisesRemote(error.EngineError, ar.get, 10)
         eid = ar.engine_id
         tic = time.time()
         while eid in self.client.ids and time.time()-tic < 5:
             time.sleep(.01)
             self.client.spin()
         self.assertFalse(eid in self.client.ids, "Engine should have died")
     
     def test_push_pull(self):
         """test pushing and pulling"""
         data = dict(a=10, b=1.05, c=list(range(10)), d={'e':(1,2),'f':'hi'})
         t = self.client.ids[-1]
         v = self.client[t]
         push = v.push
         pull = v.pull
         v.block=True
         nengines = len(self.client)
         push({'data':data})
         d = pull('data')
         self.assertEqual(d, data)
         self.client[:].push({'data':data})
         d = self.client[:].pull('data', block=True)
         self.assertEqual(d, nengines*[data])
         ar = push({'data':data}, block=False)
         self.assertTrue(isinstance(ar, AsyncResult))
         r = ar.get()
         ar = self.client[:].pull('data', block=False)
         self.assertTrue(isinstance(ar, AsyncResult))
         r = ar.get()
         self.assertEqual(r, nengines*[data])
         self.client[:].push(dict(a=10,b=20))
         r = self.client[:].pull(('a','b'), block=True)
         self.assertEqual(r, nengines*[[10,20]])
     
     def test_push_pull_function(self):
         "test pushing and pulling functions"
         def testf(x):
             return 2.0*x
         
         t = self.client.ids[-1]
         v = self.client[t]
         v.block=True
         push = v.push
         pull = v.pull
         execute = v.execute
         push({'testf':testf})
         r = pull('testf')
         self.assertEqual(r(1.0), testf(1.0))
         execute('r = testf(10)')
         r = pull('r')
         self.assertEqual(r, testf(10))
         ar = self.client[:].push({'testf':testf}, block=False)
         ar.get()
         ar = self.client[:].pull('testf', block=False)
         rlist = ar.get()
         for r in rlist:
             self.assertEqual(r(1.0), testf(1.0))
         execute("def g(x): return x*x")
         r = pull(('testf','g'))
         self.assertEqual((r[0](10),r[1](10)), (testf(10), 100))
     
     def test_push_function_globals(self):
         """test that pushed functions have access to globals"""
         @interactive
         def geta():
             return a
         # self.add_engines(1)
         v = self.client[-1]
         v.block=True
         v['f'] = geta
         self.assertRaisesRemote(NameError, v.execute, 'b=f()')
         v.execute('a=5')
         v.execute('b=f()')
         self.assertEqual(v['b'], 5)
     
     def test_push_function_defaults(self):
         """test that pushed functions preserve default args"""
         def echo(a=10):
             return a
         v = self.client[-1]
         v.block=True
         v['f'] = echo
         v.execute('b=f()')
         self.assertEqual(v['b'], 10)
 
     def test_get_result(self):
         """test getting results from the Hub."""
         c = pmod.Client(profile='iptest')
         # self.add_engines(1)
         t = c.ids[-1]
         v = c[t]
         v2 = self.client[t]
         ar = v.apply_async(wait, 1)
         # give the monitor time to notice the message
         time.sleep(.25)
-        ahr = v2.get_result(ar.msg_ids[0])
-        self.assertTrue(isinstance(ahr, AsyncHubResult))
+        ahr = v2.get_result(ar.msg_ids[0], owner=False)
+        self.assertIsInstance(ahr, AsyncHubResult)
         self.assertEqual(ahr.get(), ar.get())
         ar2 = v2.get_result(ar.msg_ids[0])
-        self.assertFalse(isinstance(ar2, AsyncHubResult))
+        self.assertNotIsInstance(ar2, AsyncHubResult)
+        self.assertEqual(ahr.get(), ar2.get())
         c.spin()
         c.close()
     
     def test_run_newline(self):
         """test that run appends newline to files"""
         with NamedTemporaryFile('w', delete=False) as f:
             f.write("""def g():
                 return 5
                 """)
         v = self.client[-1]
         v.run(f.name, block=True)
         self.assertEqual(v.apply_sync(lambda f: f(), pmod.Reference('g')), 5)
 
     def test_apply_tracked(self):
         """test tracking for apply"""
         # self.add_engines(1)
         t = self.client.ids[-1]
         v = self.client[t]
         v.block=False
         def echo(n=1024*1024, **kwargs):
             with v.temp_flags(**kwargs):
                 return v.apply(lambda x: x, 'x'*n)
         ar = echo(1, track=False)
         self.assertTrue(isinstance(ar._tracker, zmq.MessageTracker))
         self.assertTrue(ar.sent)
         ar = echo(track=True)
         self.assertTrue(isinstance(ar._tracker, zmq.MessageTracker))
         self.assertEqual(ar.sent, ar._tracker.done)
         ar._tracker.wait()
         self.assertTrue(ar.sent)
         
     def test_push_tracked(self):
         t = self.client.ids[-1]
         ns = dict(x='x'*1024*1024)
         v = self.client[t]
         ar = v.push(ns, block=False, track=False)
         self.assertTrue(isinstance(ar._tracker, zmq.MessageTracker))
         self.assertTrue(ar.sent)
         
         ar = v.push(ns, block=False, track=True)
         self.assertTrue(isinstance(ar._tracker, zmq.MessageTracker))
         ar._tracker.wait()
         self.assertEqual(ar.sent, ar._tracker.done)
         self.assertTrue(ar.sent)
         ar.get()
         
     def test_scatter_tracked(self):
         t = self.client.ids
         x='x'*1024*1024
         ar = self.client[t].scatter('x', x, block=False, track=False)
         self.assertTrue(isinstance(ar._tracker, zmq.MessageTracker))
         self.assertTrue(ar.sent)
         
         ar = self.client[t].scatter('x', x, block=False, track=True)
         self.assertTrue(isinstance(ar._tracker, zmq.MessageTracker))
         self.assertEqual(ar.sent, ar._tracker.done)
         ar._tracker.wait()
         self.assertTrue(ar.sent)
         ar.get()
     
     def test_remote_reference(self):
         v = self.client[-1]
         v['a'] = 123
         ra = pmod.Reference('a')
         b = v.apply_sync(lambda x: x, ra)
         self.assertEqual(b, 123)
 
 
     def test_scatter_gather(self):
         view = self.client[:]
         seq1 = list(range(16))
         view.scatter('a', seq1)
         seq2 = view.gather('a', block=True)
         self.assertEqual(seq2, seq1)
         self.assertRaisesRemote(NameError, view.gather, 'asdf', block=True)
     
     @skip_without('numpy')
     def test_scatter_gather_numpy(self):
         import numpy
         from numpy.testing.utils import assert_array_equal
         view = self.client[:]
         a = numpy.arange(64)
         view.scatter('a', a, block=True)
         b = view.gather('a', block=True)
         assert_array_equal(b, a)
     
     def test_scatter_gather_lazy(self):
         """scatter/gather with targets='all'"""
         view = self.client.direct_view(targets='all')
         x = list(range(64))
         view.scatter('x', x)
         gathered = view.gather('x', block=True)
         self.assertEqual(gathered, x)
         
 
     @dec.known_failure_py3
     @skip_without('numpy')
     def test_push_numpy_nocopy(self):
         import numpy
         view = self.client[:]
         a = numpy.arange(64)
         view['A'] = a
         @interactive
         def check_writeable(x):
             return x.flags.writeable
         
         for flag in view.apply_sync(check_writeable, pmod.Reference('A')):
             self.assertFalse(flag, "array is writeable, push shouldn't have pickled it")
         
         view.push(dict(B=a))
         for flag in view.apply_sync(check_writeable, pmod.Reference('B')):
             self.assertFalse(flag, "array is writeable, push shouldn't have pickled it")
     
     @skip_without('numpy')
     def test_apply_numpy(self):
         """view.apply(f, ndarray)"""
         import numpy
         from numpy.testing.utils import assert_array_equal
         
         A = numpy.random.random((100,100))
         view = self.client[-1]
         for dt in [ 'int32', 'uint8', 'float32', 'float64' ]:
             B = A.astype(dt)
             C = view.apply_sync(lambda x:x, B)
             assert_array_equal(B,C)
     
     @skip_without('numpy')
     def test_apply_numpy_object_dtype(self):
         """view.apply(f, ndarray) with dtype=object"""
         import numpy
         from numpy.testing.utils import assert_array_equal
         view = self.client[-1]
         
         A = numpy.array([dict(a=5)])
         B = view.apply_sync(lambda x:x, A)
         assert_array_equal(A,B)
         
         A = numpy.array([(0, dict(b=10))], dtype=[('i', int), ('o', object)])
         B = view.apply_sync(lambda x:x, A)
         assert_array_equal(A,B)
     
     @skip_without('numpy')
     def test_push_pull_recarray(self):
         """push/pull recarrays"""
         import numpy
         from numpy.testing.utils import assert_array_equal
         
         view = self.client[-1]
         
         R = numpy.array([
             (1, 'hi', 0.),
             (2**30, 'there', 2.5),
             (-99999, 'world', -12345.6789),
         ], [('n', int), ('s', '|S10'), ('f', float)])
         
         view['RR'] = R
         R2 = view['RR']
         
         r_dtype, r_shape = view.apply_sync(interactive(lambda : (RR.dtype, RR.shape)))
         self.assertEqual(r_dtype, R.dtype)
         self.assertEqual(r_shape, R.shape)
         self.assertEqual(R2.dtype, R.dtype)
         self.assertEqual(R2.shape, R.shape)
         assert_array_equal(R2, R)
 
     @skip_without('pandas')
     def test_push_pull_timeseries(self):
         """push/pull pandas.TimeSeries"""
         import pandas
         
         ts = pandas.TimeSeries(list(range(10)))
         
         view = self.client[-1]
         
         view.push(dict(ts=ts), block=True)
         rts = view['ts']
         
         self.assertEqual(type(rts), type(ts))
         self.assertTrue((ts == rts).all())
     
     def test_map(self):
         view = self.client[:]
         def f(x):
             return x**2
         data = list(range(16))
         r = view.map_sync(f, data)
         self.assertEqual(r, list(map(f, data)))
     
     def test_map_empty_sequence(self):
         view = self.client[:]
         r = view.map_sync(lambda x: x, [])
         self.assertEqual(r, [])
     
     def test_map_iterable(self):
         """test map on iterables (direct)"""
         view = self.client[:]
         # 101 is prime, so it won't be evenly distributed
         arr = range(101)
         # ensure it will be an iterator, even in Python 3
         it = iter(arr)
         r = view.map_sync(lambda x: x, it)
         self.assertEqual(r, list(arr))
 
     @skip_without('numpy')
     def test_map_numpy(self):
         """test map on numpy arrays (direct)"""
         import numpy
         from numpy.testing.utils import assert_array_equal
 
         view = self.client[:]
         # 101 is prime, so it won't be evenly distributed
         arr = numpy.arange(101)
         r = view.map_sync(lambda x: x, arr)
         assert_array_equal(r, arr)
     
     def test_scatter_gather_nonblocking(self):
         data = list(range(16))
         view = self.client[:]
         view.scatter('a', data, block=False)
         ar = view.gather('a', block=False)
         self.assertEqual(ar.get(), data)
     
     @skip_without('numpy')
     def test_scatter_gather_numpy_nonblocking(self):
         import numpy
         from numpy.testing.utils import assert_array_equal
         a = numpy.arange(64)
         view = self.client[:]
         ar = view.scatter('a', a, block=False)
         self.assertTrue(isinstance(ar, AsyncResult))
         amr = view.gather('a', block=False)
         self.assertTrue(isinstance(amr, AsyncMapResult))
         assert_array_equal(amr.get(), a)
 
     def test_execute(self):
         view = self.client[:]
         # self.client.debug=True
         execute = view.execute
         ar = execute('c=30', block=False)
         self.assertTrue(isinstance(ar, AsyncResult))
         ar = execute('d=[0,1,2]', block=False)
         self.client.wait(ar, 1)
         self.assertEqual(len(ar.get()), len(self.client))
         for c in view['c']:
             self.assertEqual(c, 30)
     
     def test_abort(self):
         view = self.client[-1]
         ar = view.execute('import time; time.sleep(1)', block=False)
         ar2 = view.apply_async(lambda : 2)
         ar3 = view.apply_async(lambda : 3)
         view.abort(ar2)
         view.abort(ar3.msg_ids)
         self.assertRaises(error.TaskAborted, ar2.get)
         self.assertRaises(error.TaskAborted, ar3.get)
     
     def test_abort_all(self):
         """view.abort() aborts all outstanding tasks"""
         view = self.client[-1]
         ars = [ view.apply_async(time.sleep, 0.25) for i in range(10) ]
         view.abort()
         view.wait(timeout=5)
         for ar in ars[5:]:
             self.assertRaises(error.TaskAborted, ar.get)
     
     def test_temp_flags(self):
         view = self.client[-1]
         view.block=True
         with view.temp_flags(block=False):
             self.assertFalse(view.block)
         self.assertTrue(view.block)
     
     @dec.known_failure_py3
     def test_importer(self):
         view = self.client[-1]
         view.clear(block=True)
         with view.importer:
             import re
         
         @interactive
         def findall(pat, s):
             # this globals() step isn't necessary in real code
             # only to prevent a closure in the test
             re = globals()['re']
             return re.findall(pat, s)
         
         self.assertEqual(view.apply_sync(findall, '\w+', 'hello world'), 'hello world'.split())
     
     def test_unicode_execute(self):
         """test executing unicode strings"""
         v = self.client[-1]
         v.block=True
         if sys.version_info[0] >= 3:
             code="a='é'"
         else:
             code=u"a=u'é'"
         v.execute(code)
         self.assertEqual(v['a'], u'é')
         
     def test_unicode_apply_result(self):
         """test unicode apply results"""
         v = self.client[-1]
         r = v.apply_sync(lambda : u'é')
         self.assertEqual(r, u'é')
     
     def test_unicode_apply_arg(self):
         """test passing unicode arguments to apply"""
         v = self.client[-1]
         
         @interactive
         def check_unicode(a, check):
             assert not isinstance(a, bytes), "%r is bytes, not unicode"%a
             assert isinstance(check, bytes), "%r is not bytes"%check
             assert a.encode('utf8') == check, "%s != %s"%(a,check)
         
         for s in [ u'é', u'ßø®∫',u'asdf' ]:
             try:
                 v.apply_sync(check_unicode, s, s.encode('utf8'))
             except error.RemoteError as e:
                 if e.ename == 'AssertionError':
                     self.fail(e.evalue)
                 else:
                     raise e
     
     def test_map_reference(self):
         """view.map(<Reference>, *seqs) should work"""
         v = self.client[:]
         v.scatter('n', self.client.ids, flatten=True)
         v.execute("f = lambda x,y: x*y")
         rf = pmod.Reference('f')
         nlist = list(range(10))
         mlist = nlist[::-1]
         expected = [ m*n for m,n in zip(mlist, nlist) ]
         result = v.map_sync(rf, mlist, nlist)
         self.assertEqual(result, expected)
 
     def test_apply_reference(self):
         """view.apply(<Reference>, *args) should work"""
         v = self.client[:]
         v.scatter('n', self.client.ids, flatten=True)
         v.execute("f = lambda x: n*x")
         rf = pmod.Reference('f')
         result = v.apply_sync(rf, 5)
         expected = [ 5*id for id in self.client.ids ]
         self.assertEqual(result, expected)
     
     def test_eval_reference(self):
         v = self.client[self.client.ids[0]]
         v['g'] = list(range(5))
         rg = pmod.Reference('g[0]')
         echo = lambda x:x
         self.assertEqual(v.apply_sync(echo, rg), 0)
     
     def test_reference_nameerror(self):
         v = self.client[self.client.ids[0]]
         r = pmod.Reference('elvis_has_left')
         echo = lambda x:x
         self.assertRaisesRemote(NameError, v.apply_sync, echo, r)
 
     def test_single_engine_map(self):
         e0 = self.client[self.client.ids[0]]
         r = list(range(5))
         check = [ -1*i for i in r ]
         result = e0.map_sync(lambda x: -1*x, r)
         self.assertEqual(result, check)
     
     def test_len(self):
         """len(view) makes sense"""
         e0 = self.client[self.client.ids[0]]
         self.assertEqual(len(e0), 1)
         v = self.client[:]
         self.assertEqual(len(v), len(self.client.ids))
         v = self.client.direct_view('all')
         self.assertEqual(len(v), len(self.client.ids))
         v = self.client[:2]
         self.assertEqual(len(v), 2)
         v = self.client[:1]
         self.assertEqual(len(v), 1)
         v = self.client.load_balanced_view()
         self.assertEqual(len(v), len(self.client.ids))
 
     
     # begin execute tests
     
     def test_execute_reply(self):
         e0 = self.client[self.client.ids[0]]
         e0.block = True
         ar = e0.execute("5", silent=False)
         er = ar.get()
         self.assertEqual(str(er), "<ExecuteReply[%i]: 5>" % er.execution_count)
         self.assertEqual(er.execute_result['data']['text/plain'], '5')
 
     def test_execute_reply_rich(self):
         e0 = self.client[self.client.ids[0]]
         e0.block = True
         e0.execute("from IPython.display import Image, HTML")
         ar = e0.execute("Image(data=b'garbage', format='png', width=10)", silent=False)
         er = ar.get()
         b64data = base64.encodestring(b'garbage').decode('ascii')
         self.assertEqual(er._repr_png_(), (b64data, dict(width=10)))
         ar = e0.execute("HTML('<b>bold</b>')", silent=False)
         er = ar.get()
         self.assertEqual(er._repr_html_(), "<b>bold</b>")
 
     def test_execute_reply_stdout(self):
         e0 = self.client[self.client.ids[0]]
         e0.block = True
         ar = e0.execute("print (5)", silent=False)
         er = ar.get()
         self.assertEqual(er.stdout.strip(), '5')
         
     def test_execute_result(self):
         """execute triggers execute_result with silent=False"""
         view = self.client[:]
         ar = view.execute("5", silent=False, block=True)
         
         expected = [{'text/plain' : '5'}] * len(view)
         mimes = [ out['data'] for out in ar.execute_result ]
         self.assertEqual(mimes, expected)
     
     def test_execute_silent(self):
         """execute does not trigger execute_result with silent=True"""
         view = self.client[:]
         ar = view.execute("5", block=True)
         expected = [None] * len(view)
         self.assertEqual(ar.execute_result, expected)
     
     def test_execute_magic(self):
         """execute accepts IPython commands"""
         view = self.client[:]
         view.execute("a = 5")
         ar = view.execute("%whos", block=True)
         # this will raise, if that failed
         ar.get(5)
         for stdout in ar.stdout:
             lines = stdout.splitlines()
             self.assertEqual(lines[0].split(), ['Variable', 'Type', 'Data/Info'])
             found = False
             for line in lines[2:]:
                 split = line.split()
                 if split == ['a', 'int', '5']:
                     found = True
                     break
             self.assertTrue(found, "whos output wrong: %s" % stdout)
     
     def test_execute_displaypub(self):
         """execute tracks display_pub output"""
         view = self.client[:]
         view.execute("from IPython.core.display import *")
         ar = view.execute("[ display(i) for i in range(5) ]", block=True)
         
         expected = [ {u'text/plain' : unicode_type(j)} for j in range(5) ]
         for outputs in ar.outputs:
             mimes = [ out['data'] for out in outputs ]
             self.assertEqual(mimes, expected)
     
     def test_apply_displaypub(self):
         """apply tracks display_pub output"""
         view = self.client[:]
         view.execute("from IPython.core.display import *")
         
         @interactive
         def publish():
             [ display(i) for i in range(5) ]
         
         ar = view.apply_async(publish)
         ar.get(5)
         expected = [ {u'text/plain' : unicode_type(j)} for j in range(5) ]
         for outputs in ar.outputs:
             mimes = [ out['data'] for out in outputs ]
             self.assertEqual(mimes, expected)
     
     def test_execute_raises(self):
         """exceptions in execute requests raise appropriately"""
         view = self.client[-1]
         ar = view.execute("1/0")
         self.assertRaisesRemote(ZeroDivisionError, ar.get, 2)
     
     def test_remoteerror_render_exception(self):
         """RemoteErrors get nice tracebacks"""
         view = self.client[-1]
         ar = view.execute("1/0")
         ip = get_ipython()
         ip.user_ns['ar'] = ar
         with capture_output() as io:
             ip.run_cell("ar.get(2)")
         
         self.assertTrue('ZeroDivisionError' in io.stdout, io.stdout)
     
     def test_compositeerror_render_exception(self):
         """CompositeErrors get nice tracebacks"""
         view = self.client[:]
         ar = view.execute("1/0")
         ip = get_ipython()
         ip.user_ns['ar'] = ar
 
         with capture_output() as io:
             ip.run_cell("ar.get(2)")
         
         count = min(error.CompositeError.tb_limit, len(view))
         
         self.assertEqual(io.stdout.count('ZeroDivisionError'), count * 2, io.stdout)
         self.assertEqual(io.stdout.count('by zero'), count, io.stdout)
         self.assertEqual(io.stdout.count(':execute'), count, io.stdout)
     
     def test_compositeerror_truncate(self):
         """Truncate CompositeErrors with many exceptions"""
         view = self.client[:]
         msg_ids = []
         for i in range(10):
             ar = view.execute("1/0")
             msg_ids.extend(ar.msg_ids)
         
         ar = self.client.get_result(msg_ids)
         try:
             ar.get()
         except error.CompositeError as _e:
             e = _e
         else:
             self.fail("Should have raised CompositeError")
         
         lines = e.render_traceback()
         with capture_output() as io:
             e.print_traceback()
         
         self.assertTrue("more exceptions" in lines[-1])
         count = e.tb_limit
         
         self.assertEqual(io.stdout.count('ZeroDivisionError'), 2 * count, io.stdout)
         self.assertEqual(io.stdout.count('by zero'), count, io.stdout)
         self.assertEqual(io.stdout.count(':execute'), count, io.stdout)
     
     @dec.skipif_not_matplotlib
     def test_magic_pylab(self):
         """%pylab works on engines"""
         view = self.client[-1]
         ar = view.execute("%pylab inline")
         # at least check if this raised:
         reply = ar.get(5)
         # include imports, in case user config
         ar = view.execute("plot(rand(100))", silent=False)
         reply = ar.get(5)
         self.assertEqual(len(reply.outputs), 1)
         output = reply.outputs[0]
         self.assertTrue("data" in output)
         data = output['data']
         self.assertTrue("image/png" in data)
     
     def test_func_default_func(self):
         """interactively defined function as apply func default"""
         def foo():
             return 'foo'
         
         def bar(f=foo):
             return f()
         
         view = self.client[-1]
         ar = view.apply_async(bar)
         r = ar.get(10)
         self.assertEqual(r, 'foo')
     def test_data_pub_single(self):
         view = self.client[-1]
         ar = view.execute('\n'.join([
             'from IPython.kernel.zmq.datapub import publish_data',
             'for i in range(5):',
             '  publish_data(dict(i=i))'
         ]), block=False)
         self.assertTrue(isinstance(ar.data, dict))
         ar.get(5)
         self.assertEqual(ar.data, dict(i=4))
 
     def test_data_pub(self):
         view = self.client[:]
         ar = view.execute('\n'.join([
             'from IPython.kernel.zmq.datapub import publish_data',
             'for i in range(5):',
             '  publish_data(dict(i=i))'
         ]), block=False)
         self.assertTrue(all(isinstance(d, dict) for d in ar.data))
         ar.get(5)
         self.assertEqual(ar.data, [dict(i=4)] * len(ar))
     
     def test_can_list_arg(self):
         """args in lists are canned"""
         view = self.client[-1]
         view['a'] = 128
         rA = pmod.Reference('a')
         ar = view.apply_async(lambda x: x, [rA])
         r = ar.get(5)
         self.assertEqual(r, [128])
 
     def test_can_dict_arg(self):
         """args in dicts are canned"""
         view = self.client[-1]
         view['a'] = 128
         rA = pmod.Reference('a')
         ar = view.apply_async(lambda x: x, dict(foo=rA))
         r = ar.get(5)
         self.assertEqual(r, dict(foo=128))
 
     def test_can_list_kwarg(self):
         """kwargs in lists are canned"""
         view = self.client[-1]
         view['a'] = 128
         rA = pmod.Reference('a')
         ar = view.apply_async(lambda x=5: x, x=[rA])
         r = ar.get(5)
         self.assertEqual(r, [128])
 
     def test_can_dict_kwarg(self):
         """kwargs in dicts are canned"""
         view = self.client[-1]
         view['a'] = 128
         rA = pmod.Reference('a')
         ar = view.apply_async(lambda x=5: x, dict(foo=rA))
         r = ar.get(5)
         self.assertEqual(r, dict(foo=128))
     
     def test_map_ref(self):
         """view.map works with references"""
         view = self.client[:]
         ranks = sorted(self.client.ids)
         view.scatter('rank', ranks, flatten=True)
         rrank = pmod.Reference('rank')
         
         amr = view.map_async(lambda x: x*2, [rrank] * len(view))
         drank = amr.get(5)
         self.assertEqual(drank, [ r*2 for r in ranks ])
         
     def test_nested_getitem_setitem(self):
         """get and set with view['a.b']"""
         view = self.client[-1]
         view.execute('\n'.join([
             'class A(object): pass',
             'a = A()',
             'a.b = 128',
             ]), block=True)
         ra = pmod.Reference('a')
 
         r = view.apply_sync(lambda x: x.b, ra)
         self.assertEqual(r, 128)
         self.assertEqual(view['a.b'], 128)
 
         view['a.b'] = 0
 
         r = view.apply_sync(lambda x: x.b, ra)
         self.assertEqual(r, 0)
         self.assertEqual(view['a.b'], 0)
     
     def test_return_namedtuple(self):
         def namedtuplify(x, y):
             from IPython.parallel.tests.test_view import point
             return point(x, y)
         
         view = self.client[-1]
         p = view.apply_sync(namedtuplify, 1, 2)
         self.assertEqual(p.x, 1)
         self.assertEqual(p.y, 2)
 
     def test_apply_namedtuple(self):
         def echoxy(p):
             return p.y, p.x
         
         view = self.client[-1]
         tup = view.apply_sync(echoxy, point(1, 2))
         self.assertEqual(tup, (2,1))
     
     def test_sync_imports(self):
         view = self.client[-1]
         with capture_output() as io:
             with view.sync_imports():
                 import IPython
         self.assertIn("IPython", io.stdout)
         
         @interactive
         def find_ipython():
             return 'IPython' in globals()
         
         assert view.apply_sync(find_ipython)
 
     def test_sync_imports_quiet(self):
         view = self.client[-1]
         with capture_output() as io:
             with view.sync_imports(quiet=True):
                 import IPython
         self.assertEqual(io.stdout, '')
         
         @interactive
         def find_ipython():
             return 'IPython' in globals()
         
         assert view.apply_sync(find_ipython)