fix/test pushed function globals

IPython/utils/pickleutil.py

@@ -1,127 +1,127 b''
 # encoding: utf-8
 
 """Pickle related utilities. Perhaps this should be called 'can'."""
 
 __docformat__ = "restructuredtext en"
 
 #-------------------------------------------------------------------------------
 #  Copyright (C) 2008  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-------------------------------------------------------------------------------
 
 #-------------------------------------------------------------------------------
 # Imports
 #-------------------------------------------------------------------------------
 
 from types import FunctionType
 import copy
 
 from IPython.zmq.parallel.dependency import dependent
 
 import codeutil
 
 #-------------------------------------------------------------------------------
 # Classes
 #-------------------------------------------------------------------------------
 
 
 class CannedObject(object):
     def __init__(self, obj, keys=[]):
         self.keys = keys
         self.obj = copy.copy(obj)
         for key in keys:
             setattr(self.obj, key, can(getattr(obj, key)))
             
     
     def getObject(self, g=None):
         if g is None:
             g = globals()
         for key in self.keys:
             setattr(self.obj, key, uncan(getattr(self.obj, key), g))
         return self.obj
 
         
 
 class CannedFunction(CannedObject):
     
     def __init__(self, f):
         self._checkType(f)    
         self.code = f.func_code
         self.__name__ = f.__name__
     
     def _checkType(self, obj):
         assert isinstance(obj, FunctionType), "Not a function type"
     
     def getFunction(self, g=None):
         if g is None:
             g = globals()
         newFunc = FunctionType(self.code, g)
         return newFunc
 
 #-------------------------------------------------------------------------------
 # Functions
 #-------------------------------------------------------------------------------
 
 
 def can(obj):
     if isinstance(obj, FunctionType):
         return CannedFunction(obj)
     elif isinstance(obj, dependent):
         keys = ('f','df')
         return CannedObject(obj, keys=keys)
     elif isinstance(obj,dict):
         return canDict(obj)
     elif isinstance(obj, (list,tuple)):
         return canSequence(obj)
     else:
         return obj
 
 def canDict(obj):
     if isinstance(obj, dict):
         newobj = {}
         for k, v in obj.iteritems():
             newobj[k] = can(v)
         return newobj
     else:
         return obj
 
 def canSequence(obj):
     if isinstance(obj, (list, tuple)):
         t = type(obj)
         return t([can(i) for i in obj])
     else:
         return obj
 
 def uncan(obj, g=None):
     if isinstance(obj, CannedFunction):
         return obj.getFunction(g)
     elif isinstance(obj, CannedObject):
         return obj.getObject(g)
     elif isinstance(obj,dict):
-        return uncanDict(obj)
+        return uncanDict(obj, g)
     elif isinstance(obj, (list,tuple)):
-        return uncanSequence(obj)
+        return uncanSequence(obj, g)
     else:
         return obj
 
 def uncanDict(obj, g=None):
     if isinstance(obj, dict):
         newobj = {}
         for k, v in obj.iteritems():
             newobj[k] = uncan(v,g)
         return newobj
     else:
         return obj
 
 def uncanSequence(obj, g=None):
     if isinstance(obj, (list, tuple)):
         t = type(obj)
         return t([uncan(i,g) for i in obj])
     else:
         return obj
 
 
 def rebindFunctionGlobals(f, glbls):
     return FunctionType(f.func_code, glbls)

IPython/zmq/parallel/error.py

@@ -1,295 +1,292 b''
 # encoding: utf-8
 
 """Classes and functions for kernel related errors and exceptions."""
 from __future__ import print_function
 
 __docformat__ = "restructuredtext en"
 
 # Tell nose to skip this module
 __test__ = {}
 
 #-------------------------------------------------------------------------------
 #  Copyright (C) 2008  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-------------------------------------------------------------------------------
 
 #-------------------------------------------------------------------------------
 # Error classes
 #-------------------------------------------------------------------------------
 class IPythonError(Exception):
     """Base exception that all of our exceptions inherit from.
 
     This can be raised by code that doesn't have any more specific
     information."""
 
     pass
 
 # Exceptions associated with the controller objects
 class ControllerError(IPythonError): pass
 
 class ControllerCreationError(ControllerError): pass
 
 
 # Exceptions associated with the Engines
 class EngineError(IPythonError): pass
 
 class EngineCreationError(EngineError): pass
 
 class KernelError(IPythonError):
     pass
 
 class NotDefined(KernelError):
     def __init__(self, name):
         self.name = name
         self.args = (name,)
 
     def __repr__(self):
         return '<NotDefined: %s>' % self.name
     
     __str__ = __repr__
 
 
 class QueueCleared(KernelError):
     pass
 
 
 class IdInUse(KernelError):
     pass
 
 
 class ProtocolError(KernelError):
     pass
 
 
 class ConnectionError(KernelError):
     pass
 
 
 class InvalidEngineID(KernelError):
     pass
 
 
 class NoEnginesRegistered(KernelError):
     pass
 
 
 class InvalidClientID(KernelError):
     pass
 
 
 class InvalidDeferredID(KernelError):
     pass
 
 
 class SerializationError(KernelError):
     pass
 
 
 class MessageSizeError(KernelError):
     pass
 
 
 class PBMessageSizeError(MessageSizeError):
     pass
 
 
 class ResultNotCompleted(KernelError):
     pass
 
 
 class ResultAlreadyRetrieved(KernelError):
     pass
 
 class ClientError(KernelError):
     pass
 
 
 class TaskAborted(KernelError):
     pass
 
 
 class TaskTimeout(KernelError):
     pass
 
 
 class NotAPendingResult(KernelError):
     pass
 
 
 class UnpickleableException(KernelError):
     pass
 
 
 class AbortedPendingDeferredError(KernelError):
     pass
 
 
 class InvalidProperty(KernelError):
     pass
 
 
 class MissingBlockArgument(KernelError):
     pass
 
 
 class StopLocalExecution(KernelError):
     pass
 
 
 class SecurityError(KernelError):
     pass
 
 
 class FileTimeoutError(KernelError):
     pass
 
 class TimeoutError(KernelError):
     pass
 
 class UnmetDependency(KernelError):
     pass
 
 class ImpossibleDependency(UnmetDependency):
     pass
 
 class DependencyTimeout(ImpossibleDependency):
     pass
 
 class InvalidDependency(ImpossibleDependency):
     pass
 
 class RemoteError(KernelError):
     """Error raised elsewhere"""
     ename=None
     evalue=None
     traceback=None
     engine_info=None
     
     def __init__(self, ename, evalue, traceback, engine_info=None):
         self.ename=ename
         self.evalue=evalue
         self.traceback=traceback
         self.engine_info=engine_info or {}
         self.args=(ename, evalue)
     
     def __repr__(self):
         engineid = self.engine_info.get('engineid', ' ')
         return "<Remote[%s]:%s(%s)>"%(engineid, self.ename, self.evalue)
         
     def __str__(self):
         sig = "%s(%s)"%(self.ename, self.evalue)
         if self.traceback:
             return sig + '\n' + self.traceback
         else:
             return sig
     
 
 class TaskRejectError(KernelError):
     """Exception to raise when a task should be rejected by an engine.
     
     This exception can be used to allow a task running on an engine to test
     if the engine (or the user's namespace on the engine) has the needed
     task dependencies.  If not, the task should raise this exception.  For
     the task to be retried on another engine, the task should be created
     with the `retries` argument > 1.
     
     The advantage of this approach over our older properties system is that
     tasks have full access to the user's namespace on the engines and the
     properties don't have to be managed or tested by the controller.
     """
 
 
-class CompositeError(KernelError):
+class CompositeError(RemoteError):
     """Error for representing possibly multiple errors on engines"""
     def __init__(self, message, elist):
         Exception.__init__(self, *(message, elist))
         # Don't use pack_exception because it will conflict with the .message
         # attribute that is being deprecated in 2.6 and beyond.
         self.msg = message
         self.elist = elist
         self.args = [ e[0] for e in elist ]
 
     def _get_engine_str(self, ei):
         if not ei:
             return '[Engine Exception]'
         else:
-            return '[%i:%s]: ' % (ei['engineid'], ei['method'])
+            return '[%s:%s]: ' % (ei['engineid'], ei['method'])
 
     def _get_traceback(self, ev):
         try:
             tb = ev._ipython_traceback_text
         except AttributeError:
             return 'No traceback available'
         else:
             return tb
 
     def __str__(self):
         s = str(self.msg)
         for en, ev, etb, ei in self.elist:
             engine_str = self._get_engine_str(ei)
             s = s + '\n' + engine_str + en + ': ' + str(ev)
         return s
     
     def __repr__(self):
         return "CompositeError(%i)"%len(self.elist)
 
     def print_tracebacks(self, excid=None):
         if excid is None:
             for (en,ev,etb,ei) in self.elist:
                 print (self._get_engine_str(ei))
                 print (etb or 'No traceback available')
                 print ()
         else:
             try:
                 en,ev,etb,ei = self.elist[excid]
             except:
                 raise IndexError("an exception with index %i does not exist"%excid)
             else:
                 print (self._get_engine_str(ei))
                 print (etb or 'No traceback available')
     
     def raise_exception(self, excid=0):
         try:
             en,ev,etb,ei = self.elist[excid]
         except:
             raise IndexError("an exception with index %i does not exist"%excid)
         else:
-            try:
-                raise RemoteError(en, ev, etb, ei)
-            except:
-                et,ev,tb = sys.exc_info()
+            raise RemoteError(en, ev, etb, ei)
 
 
 def collect_exceptions(rdict_or_list, method='unspecified'):
     """check a result dict for errors, and raise CompositeError if any exist.
     Passthrough otherwise."""
     elist = []
     if isinstance(rdict_or_list, dict):
         rlist = rdict_or_list.values()
     else:
         rlist = rdict_or_list
     for r in rlist:
         if isinstance(r, RemoteError):
             en, ev, etb, ei = r.ename, r.evalue, r.traceback, r.engine_info
             # Sometimes we could have CompositeError in our list.  Just take
             # the errors out of them and put them in our new list.  This 
             # has the effect of flattening lists of CompositeErrors into one
             # CompositeError
             if en=='CompositeError':
                 for e in ev.elist:
                     elist.append(e)
             else:
                 elist.append((en, ev, etb, ei))
     if len(elist)==0:
         return rdict_or_list
     else:
         msg = "one or more exceptions from call to method: %s" % (method)
         # This silliness is needed so the debugger has access to the exception
         # instance (e in this case)
         try:
             raise CompositeError(msg, elist)
-        except CompositeError, e:
+        except CompositeError as e:
             raise e
 

IPython/zmq/parallel/tests/clienttest.py

@@ -1,85 +1,96 b''
 import time
 from signal import SIGINT
 from multiprocessing import Process
 
 from nose import SkipTest
 
 from zmq.tests import BaseZMQTestCase
 
 from IPython.external.decorator import decorator
 
+from IPython.zmq.parallel import error
+from IPython.zmq.parallel.client import Client
 from IPython.zmq.parallel.ipcluster import launch_process
 from IPython.zmq.parallel.entry_point import select_random_ports
-from IPython.zmq.parallel.client import Client
 from IPython.zmq.parallel.tests import processes,add_engine
 
 # simple tasks for use in apply tests
 
 def segfault():
     """"""
     import ctypes
     ctypes.memset(-1,0,1)
 
 def wait(n):
     """sleep for a time"""
     import time
     time.sleep(n)
     return n
 
 def raiser(eclass):
     """raise an exception"""
     raise eclass()
 
 # test decorator for skipping tests when libraries are unavailable
 def skip_without(*names):
     """skip a test if some names are not importable"""
     @decorator
     def skip_without_names(f, *args, **kwargs):
         """decorator to skip tests in the absence of numpy."""
         for name in names:
             try:
                 __import__(name)
             except ImportError:
                 raise SkipTest
         return f(*args, **kwargs)
     return skip_without_names
 
 
 class ClusterTestCase(BaseZMQTestCase):
     
     def add_engines(self, n=1, block=True):
         """add multiple engines to our cluster"""
         for i in range(n):
             self.engines.append(add_engine())
         if block:
             self.wait_on_engines()
     
     def wait_on_engines(self, timeout=5):
         """wait for our engines to connect."""
         n = len(self.engines)+self.base_engine_count
         tic = time.time()
         while time.time()-tic < timeout and len(self.client.ids) < n:
             time.sleep(0.1)
         
         assert not self.client.ids < n, "waiting for engines timed out"
     
     def connect_client(self):
         """connect a client with my Context, and track its sockets for cleanup"""
         c = Client(profile='iptest',context=self.context)
         for name in filter(lambda n:n.endswith('socket'), dir(c)):
             self.sockets.append(getattr(c, name))
         return c
     
+    def assertRaisesRemote(self, etype, f, *args, **kwargs):
+        try:
+            f(*args, **kwargs)
+        except error.CompositeError as e:
+            e.raise_exception()
+        except error.RemoteError as e:
+            self.assertEquals(etype.__name__, e.ename, "Should have raised %r, but raised %r"%(e.ename, etype.__name__))
+        else:
+            self.fail("should have raised a RemoteError")
+            
     def setUp(self):
         BaseZMQTestCase.setUp(self)
         self.client = self.connect_client()
         self.base_engine_count=len(self.client.ids)
         self.engines=[]
     
     def tearDown(self):
         [ e.terminate() for e in filter(lambda e: e.poll() is None, self.engines) ]
         # while len(self.client.ids) > self.base_engine_count:
         #     time.sleep(.1)
         del self.engines
         BaseZMQTestCase.tearDown(self)
         
  No newline at end of file

IPython/zmq/parallel/tests/test_client.py

@@ -1,117 +1,134 b''
 import time
 
 import nose.tools as nt
 
 from IPython.zmq.parallel.asyncresult import AsyncResult
 from IPython.zmq.parallel.view import LoadBalancedView, DirectView
 
 from clienttest import ClusterTestCase, segfault
 
 class TestClient(ClusterTestCase):
     
     def test_ids(self):
         self.assertEquals(len(self.client.ids), 1)
         self.add_engines(3)
         self.assertEquals(len(self.client.ids), 4)
     
     def test_segfault(self):
         self.add_engines(1)
         eid = self.client.ids[-1]
         self.client[eid].apply(segfault)
         while eid in self.client.ids:
             time.sleep(.01)
             self.client.spin()
     
     def test_view_indexing(self):
         self.add_engines(4)
         targets = self.client._build_targets('all')[-1]
         v = self.client[:]
         self.assertEquals(v.targets, targets)
         t = self.client.ids[2]
         v = self.client[t]
         self.assert_(isinstance(v, DirectView))
         self.assertEquals(v.targets, t)
         t = self.client.ids[2:4]
         v = self.client[t]
         self.assert_(isinstance(v, DirectView))
         self.assertEquals(v.targets, t)
         v = self.client[::2]
         self.assert_(isinstance(v, DirectView))
         self.assertEquals(v.targets, targets[::2])
         v = self.client[1::3]
         self.assert_(isinstance(v, DirectView))
         self.assertEquals(v.targets, targets[1::3])
         v = self.client[:-3]
         self.assert_(isinstance(v, DirectView))
         self.assertEquals(v.targets, targets[:-3])
+        v = self.client[-1]
+        self.assert_(isinstance(v, DirectView))
+        self.assertEquals(v.targets, targets[-1])
         nt.assert_raises(TypeError, lambda : self.client[None])
     
     def test_view_cache(self):
-        """test blocking and non-blocking behavior"""
+        """test that multiple view requests return the same object"""
         v = self.client[:2]
         v2 =self.client[:2]
         self.assertTrue(v is v2)
         v = self.client.view()
         v2 = self.client.view(balanced=True)
         self.assertTrue(v is v2)
     
     def test_targets(self):
         """test various valid targets arguments"""
         pass
     
     def test_clear(self):
         """test clear behavior"""
         # self.add_engines(4)
         # self.client.push()
     
     def test_push_pull(self):
+        """test pushing and pulling"""
         data = dict(a=10, b=1.05, c=range(10), d={'e':(1,2),'f':'hi'})
         self.add_engines(4)
         push = self.client.push
         pull = self.client.pull
         self.client.block=True
         nengines = len(self.client)
         push({'data':data}, targets=0)
         d = pull('data', targets=0)
         self.assertEquals(d, data)
         push({'data':data})
         d = pull('data')
         self.assertEquals(d, nengines*[data])
         ar = push({'data':data}, block=False)
         self.assertTrue(isinstance(ar, AsyncResult))
         r = ar.get()
         ar = pull('data', block=False)
         self.assertTrue(isinstance(ar, AsyncResult))
         r = ar.get()
         self.assertEquals(r, nengines*[data])
         push(dict(a=10,b=20))
         r = pull(('a','b'))
         self.assertEquals(r, nengines*[[10,20]])
     
     def test_push_pull_function(self):
+        "test pushing and pulling functions"
         def testf(x):
             return 2.0*x
         
         self.add_engines(4)
         self.client.block=True
         push = self.client.push
         pull = self.client.pull
         execute = self.client.execute
         push({'testf':testf}, targets=0)
         r = pull('testf', targets=0)
         self.assertEqual(r(1.0), testf(1.0))
         execute('r = testf(10)', targets=0)
         r = pull('r', targets=0)
         self.assertEquals(r, testf(10))
         ar = push({'testf':testf}, block=False)
         ar.get()
         ar = pull('testf', block=False)
         rlist = ar.get()
         for r in rlist:
             self.assertEqual(r(1.0), testf(1.0))
         execute("def g(x): return x*x", targets=0)
         r = pull(('testf','g'),targets=0)
         self.assertEquals((r[0](10),r[1](10)), (testf(10), 100))
-        
+    
+    def test_push_function_globals(self):
+        """test that pushed functions have access to globals"""
+        def geta():
+            return a
+        self.add_engines(1)
+        v = self.client[-1]
+        v.block=True
+        v['f'] = geta
+        self.assertRaisesRemote(NameError, v.execute, 'b=f()')
+        v.execute('a=5')
+        v.execute('b=f()')
+        self.assertEquals(v['b'], 5)
     
         
  No newline at end of file