merge from trunk

IPython/kernel/mapper.py

@@ -0,0 +1,233 b''
+# encoding: utf-8
+
+"""A parallelized version of Python's builtin map."""
+
+__docformat__ = "restructuredtext en"
+
+#----------------------------------------------------------------------------
+#  Copyright (C) 2008  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#----------------------------------------------------------------------------
+
+#----------------------------------------------------------------------------
+# Imports
+#----------------------------------------------------------------------------
+
+from types import FunctionType
+from zope.interface import Interface, implements
+from IPython.kernel.task import MapTask
+from IPython.kernel.twistedutil import DeferredList, gatherBoth
+from IPython.kernel.util import printer
+from IPython.kernel.error import collect_exceptions
+
+#----------------------------------------------------------------------------
+# Code
+#----------------------------------------------------------------------------
+
+class IMapper(Interface):
+    """The basic interface for a Mapper.
+    
+    This defines a generic interface for mapping.  The idea of this is 
+    similar to that of Python's builtin `map` function, which applies a function
+    elementwise to a sequence.
+    """
+    
+    def map(func, *seqs):
+        """Do map in parallel.
+        
+        Equivalent to map(func, *seqs) or:
+        
+        [func(seqs[0][0], seqs[1][0],...), func(seqs[0][1], seqs[1][1],...),...]
+        
+        :Parameters:
+            func : FunctionType
+                The function to apply to the sequence
+            sequences : tuple of iterables
+                A sequence of iterables that are used for sucessive function
+                arguments.  This work just like map
+        """
+
+class IMultiEngineMapperFactory(Interface):
+    """
+    An interface for something that creates `IMapper` instances.
+    """
+    
+    def mapper(dist='b', targets='all', block=True):
+        """
+        Create an `IMapper` implementer with a given set of arguments.
+        
+        The `IMapper` created using a multiengine controller is 
+        not load balanced.
+        """
+
+class ITaskMapperFactory(Interface):
+    """
+    An interface for something that creates `IMapper` instances.
+    """
+    
+    def mapper(clear_before=False, clear_after=False, retries=0, 
+                recovery_task=None, depend=None, block=True):
+        """
+        Create an `IMapper` implementer with a given set of arguments.
+        
+        The `IMapper` created using a task controller is load balanced.
+        
+        See the documentation for `IPython.kernel.task.BaseTask` for 
+        documentation on the arguments to this method.
+        """
+
+
+class MultiEngineMapper(object):
+    """
+    A Mapper for `IMultiEngine` implementers.
+    """
+    
+    implements(IMapper)
+    
+    def __init__(self, multiengine, dist='b', targets='all', block=True):
+        """
+        Create a Mapper for a multiengine.
+        
+        The value of all arguments are used for all calls to `map`.  This
+        class allows these arguemnts to be set for a series of map calls.
+        
+        :Parameters:
+            multiengine : `IMultiEngine` implementer
+                The multiengine to use for running the map commands
+            dist : str
+                The type of decomposition to use.  Only block ('b') is
+                supported currently
+            targets : (str, int, tuple of ints)
+                The engines to use in the map
+            block : boolean
+                Whether to block when the map is applied
+        """
+        self.multiengine = multiengine
+        self.dist = dist
+        self.targets = targets
+        self.block = block
+    
+    def map(self, func, *sequences):
+        """
+        Apply func to *sequences elementwise.  Like Python's builtin map.
+        
+        This version is not load balanced.
+        """
+        max_len = max(len(s) for s in sequences)
+        for s in sequences:
+            if len(s)!=max_len:
+                raise ValueError('all sequences must have equal length')
+        assert isinstance(func, (str, FunctionType)), "func must be a fuction or str"
+        return self.multiengine.raw_map(func, sequences, dist=self.dist,
+            targets=self.targets, block=self.block)
+
+class TaskMapper(object):
+    """
+    Make an `ITaskController` look like an `IMapper`.
+    
+    This class provides a load balanced version of `map`.
+    """
+    
+    def __init__(self, task_controller, clear_before=False, clear_after=False, retries=0, 
+            recovery_task=None, depend=None, block=True):
+        """
+        Create a `IMapper` given a `TaskController` and arguments.
+        
+        The additional arguments are those that are common to all types of 
+        tasks and are described in the documentation for 
+        `IPython.kernel.task.BaseTask`.
+        
+        :Parameters:
+            task_controller : an `IBlockingTaskClient` implementer
+                The `TaskController` to use for calls to `map`
+        """
+        self.task_controller = task_controller
+        self.clear_before = clear_before
+        self.clear_after = clear_after
+        self.retries = retries
+        self.recovery_task = recovery_task
+        self.depend = depend
+        self.block = block
+    
+    def map(self, func, *sequences):
+        """
+        Apply func to *sequences elementwise.  Like Python's builtin map.
+        
+        This version is load balanced.
+        """
+        max_len = max(len(s) for s in sequences)
+        for s in sequences:
+            if len(s)!=max_len:
+                raise ValueError('all sequences must have equal length')
+        task_args = zip(*sequences)
+        task_ids = []
+        dlist = []
+        for ta in task_args:
+            task = MapTask(func, ta, clear_before=self.clear_before,
+                clear_after=self.clear_after, retries=self.retries,
+                recovery_task=self.recovery_task, depend=self.depend)
+            dlist.append(self.task_controller.run(task))
+        dlist = gatherBoth(dlist, consumeErrors=1)
+        dlist.addCallback(collect_exceptions,'map')
+        if self.block:
+            def get_results(task_ids):
+                d = self.task_controller.barrier(task_ids)
+                d.addCallback(lambda _: gatherBoth([self.task_controller.get_task_result(tid) for tid in task_ids], consumeErrors=1))
+                d.addCallback(collect_exceptions, 'map')
+                return d
+            dlist.addCallback(get_results)
+        return dlist
+
+class SynchronousTaskMapper(object):
+    """
+    Make an `IBlockingTaskClient` look like an `IMapper`.
+    
+    This class provides a load balanced version of `map`.
+    """
+    
+    def __init__(self, task_controller, clear_before=False, clear_after=False, retries=0, 
+            recovery_task=None, depend=None, block=True):
+        """
+        Create a `IMapper` given a `IBlockingTaskClient` and arguments.
+        
+        The additional arguments are those that are common to all types of 
+        tasks and are described in the documentation for 
+        `IPython.kernel.task.BaseTask`.
+        
+        :Parameters:
+            task_controller : an `IBlockingTaskClient` implementer
+                The `TaskController` to use for calls to `map`
+        """
+        self.task_controller = task_controller
+        self.clear_before = clear_before
+        self.clear_after = clear_after
+        self.retries = retries
+        self.recovery_task = recovery_task
+        self.depend = depend
+        self.block = block
+    
+    def map(self, func, *sequences):
+        """
+        Apply func to *sequences elementwise.  Like Python's builtin map.
+        
+        This version is load balanced.
+        """
+        max_len = max(len(s) for s in sequences)
+        for s in sequences:
+            if len(s)!=max_len:
+                raise ValueError('all sequences must have equal length')
+        task_args = zip(*sequences)
+        task_ids = []
+        for ta in task_args:
+            task = MapTask(func, ta, clear_before=self.clear_before,
+                clear_after=self.clear_after, retries=self.retries,
+                recovery_task=self.recovery_task, depend=self.depend)
+            task_ids.append(self.task_controller.run(task))
+        if self.block:
+            self.task_controller.barrier(task_ids)
+            task_results = [self.task_controller.get_task_result(tid) for tid in task_ids]
+            return task_results
+        else:
+            return task_ids
  No newline at end of file

IPython/Magic.py

@@ -1,3319 +1,3318 b''
 # -*- coding: utf-8 -*-
 """Magic functions for InteractiveShell.
 
 $Id: Magic.py 2996 2008-01-30 06:31:39Z fperez $"""
 
 #*****************************************************************************
 #       Copyright (C) 2001 Janko Hauser <[e-mail]> and
 #       Copyright (C) 2001-2006 Fernando Perez <[e-mail]>
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #*****************************************************************************
 
 #****************************************************************************
 # Modules and globals
 
 from IPython import Release
 __author__  = '%s <%s>\n%s <%s>' % \
               ( Release.authors['Janko'] + Release.authors['Fernando'] )
 __license__ = Release.license
 
 # Python standard modules
 import __builtin__
 import bdb
 import inspect
 import os
 import pdb
 import pydoc
 import sys
 import re
 import tempfile
 import time
 import cPickle as pickle
 import textwrap
 from cStringIO import StringIO
 from getopt import getopt,GetoptError
 from pprint import pprint, pformat
 from sets import Set
 
 # cProfile was added in Python2.5
 try:
     import cProfile as profile
     import pstats
 except ImportError:
     # profile isn't bundled by default in Debian for license reasons
     try:
         import profile,pstats
     except ImportError:
         profile = pstats = None
 
 # Homebrewed
 import IPython
 from IPython import Debugger, OInspect, wildcard
 from IPython.FakeModule import FakeModule
 from IPython.Itpl import Itpl, itpl, printpl,itplns
 from IPython.PyColorize import Parser
 from IPython.ipstruct import Struct
 from IPython.macro import Macro
 from IPython.genutils import *
 from IPython import platutils
 import IPython.generics
 import IPython.ipapi
 from IPython.ipapi import UsageError
 #***************************************************************************
 # Utility functions
 def on_off(tag):
     """Return an ON/OFF string for a 1/0 input. Simple utility function."""
     return ['OFF','ON'][tag]
 
 class Bunch: pass
 
 def compress_dhist(dh):
     head, tail = dh[:-10], dh[-10:]
 
     newhead = []
     done = Set()
     for h in head:
         if h in done:
             continue
         newhead.append(h)
         done.add(h)
 
     return newhead + tail        
         
 
 #***************************************************************************
 # Main class implementing Magic functionality
 class Magic:
     """Magic functions for InteractiveShell.
 
     Shell functions which can be reached as %function_name. All magic
     functions should accept a string, which they can parse for their own
     needs. This can make some functions easier to type, eg `%cd ../`
     vs. `%cd("../")`
 
     ALL definitions MUST begin with the prefix magic_. The user won't need it
     at the command line, but it is is needed in the definition. """
 
     # class globals
     auto_status = ['Automagic is OFF, % prefix IS needed for magic functions.',
                    'Automagic is ON, % prefix NOT needed for magic functions.']
 
     #......................................................................
     # some utility functions
 
     def __init__(self,shell):
         
         self.options_table = {}
         if profile is None:
             self.magic_prun = self.profile_missing_notice
         self.shell = shell
 
         # namespace for holding state we may need
         self._magic_state = Bunch()
 
     def profile_missing_notice(self, *args, **kwargs):
         error("""\
 The profile module could not be found. It has been removed from the standard
 python packages because of its non-free license. To use profiling, install the
 python-profiler package from non-free.""")
 
     def default_option(self,fn,optstr):
         """Make an entry in the options_table for fn, with value optstr"""
 
         if fn not in self.lsmagic():
             error("%s is not a magic function" % fn)
         self.options_table[fn] = optstr
 
     def lsmagic(self):
         """Return a list of currently available magic functions.
 
         Gives a list of the bare names after mangling (['ls','cd', ...], not
         ['magic_ls','magic_cd',...]"""
 
         # FIXME. This needs a cleanup, in the way the magics list is built.
         
         # magics in class definition
         class_magic = lambda fn: fn.startswith('magic_') and \
                       callable(Magic.__dict__[fn])
         # in instance namespace (run-time user additions)
         inst_magic =  lambda fn: fn.startswith('magic_') and \
                      callable(self.__dict__[fn])
         # and bound magics by user (so they can access self):
         inst_bound_magic =  lambda fn: fn.startswith('magic_') and \
                            callable(self.__class__.__dict__[fn])
         magics = filter(class_magic,Magic.__dict__.keys()) + \
                  filter(inst_magic,self.__dict__.keys()) + \
                  filter(inst_bound_magic,self.__class__.__dict__.keys())
         out = []
         for fn in Set(magics):
             out.append(fn.replace('magic_','',1))
         out.sort()
         return out
     
     def extract_input_slices(self,slices,raw=False):
         """Return as a string a set of input history slices.
 
         Inputs:
 
           - slices: the set of slices is given as a list of strings (like
           ['1','4:8','9'], since this function is for use by magic functions
           which get their arguments as strings.
 
         Optional inputs:
 
           - raw(False): by default, the processed input is used.  If this is
           true, the raw input history is used instead.
 
         Note that slices can be called with two notations:
 
         N:M -> standard python form, means including items N...(M-1).
 
         N-M -> include items N..M (closed endpoint)."""
 
         if raw:
             hist = self.shell.input_hist_raw
         else:
             hist = self.shell.input_hist
 
         cmds = []
         for chunk in slices:
             if ':' in chunk:
                 ini,fin = map(int,chunk.split(':'))
             elif '-' in chunk:
                 ini,fin = map(int,chunk.split('-'))
                 fin += 1
             else:
                 ini = int(chunk)
                 fin = ini+1
             cmds.append(hist[ini:fin])
         return cmds
         
     def _ofind(self, oname, namespaces=None):
         """Find an object in the available namespaces.
 
         self._ofind(oname) -> dict with keys: found,obj,ospace,ismagic
 
         Has special code to detect magic functions.
         """
         
         oname = oname.strip()
 
         alias_ns = None
         if namespaces is None:
             # Namespaces to search in:
             # Put them in a list. The order is important so that we
             # find things in the same order that Python finds them.
             namespaces = [ ('Interactive', self.shell.user_ns),
                            ('IPython internal', self.shell.internal_ns),
                            ('Python builtin', __builtin__.__dict__),
                            ('Alias', self.shell.alias_table),
                            ]
             alias_ns = self.shell.alias_table
 
         # initialize results to 'null'
         found = 0; obj = None;  ospace = None;  ds = None;
         ismagic = 0; isalias = 0; parent = None
 
         # Look for the given name by splitting it in parts.  If the head is
         # found, then we look for all the remaining parts as members, and only
         # declare success if we can find them all.
         oname_parts = oname.split('.')
         oname_head, oname_rest = oname_parts[0],oname_parts[1:]
         for nsname,ns in namespaces:
             try:
                 obj = ns[oname_head]
             except KeyError:
                 continue
             else:
                 #print 'oname_rest:', oname_rest  # dbg
                 for part in oname_rest:
                     try:
                         parent = obj
                         obj = getattr(obj,part)
                     except:
                         # Blanket except b/c some badly implemented objects
                         # allow __getattr__ to raise exceptions other than
                         # AttributeError, which then crashes IPython.
                         break
                 else:
                     # If we finish the for loop (no break), we got all members
                     found = 1
                     ospace = nsname
                     if ns == alias_ns:
                         isalias = 1
                     break  # namespace loop
 
         # Try to see if it's magic
         if not found:
             if oname.startswith(self.shell.ESC_MAGIC):
                 oname = oname[1:]
             obj = getattr(self,'magic_'+oname,None)
             if obj is not None:
                 found = 1
                 ospace = 'IPython internal'
                 ismagic = 1
 
         # Last try: special-case some literals like '', [], {}, etc:
         if not found and oname_head in ["''",'""','[]','{}','()']:
             obj = eval(oname_head)
             found = 1
             ospace = 'Interactive'
             
         return {'found':found, 'obj':obj, 'namespace':ospace,
                 'ismagic':ismagic, 'isalias':isalias, 'parent':parent}
     
     def arg_err(self,func):
         """Print docstring if incorrect arguments were passed"""
         print 'Error in arguments:'
         print OInspect.getdoc(func)
 
     def format_latex(self,strng):
         """Format a string for latex inclusion."""
 
         # Characters that need to be escaped for latex:
         escape_re = re.compile(r'(%|_|\$|#|&)',re.MULTILINE)
         # Magic command names as headers:
         cmd_name_re = re.compile(r'^(%s.*?):' % self.shell.ESC_MAGIC,
                                  re.MULTILINE)
         # Magic commands 
         cmd_re = re.compile(r'(?P<cmd>%s.+?\b)(?!\}\}:)' % self.shell.ESC_MAGIC,
                             re.MULTILINE)
         # Paragraph continue
         par_re = re.compile(r'\\$',re.MULTILINE)
 
         # The "\n" symbol
         newline_re = re.compile(r'\\n')
 
         # Now build the string for output:
         #strng = cmd_name_re.sub(r'\n\\texttt{\\textsl{\\large \1}}:',strng)
         strng = cmd_name_re.sub(r'\n\\bigskip\n\\texttt{\\textbf{ \1}}:',
                                 strng)
         strng = cmd_re.sub(r'\\texttt{\g<cmd>}',strng)
         strng = par_re.sub(r'\\\\',strng)
         strng = escape_re.sub(r'\\\1',strng)
         strng = newline_re.sub(r'\\textbackslash{}n',strng)
         return strng
 
     def format_screen(self,strng):
         """Format a string for screen printing.
 
         This removes some latex-type format codes."""
         # Paragraph continue
         par_re = re.compile(r'\\$',re.MULTILINE)
         strng = par_re.sub('',strng)
         return strng
 
     def parse_options(self,arg_str,opt_str,*long_opts,**kw):
         """Parse options passed to an argument string.
 
         The interface is similar to that of getopt(), but it returns back a
         Struct with the options as keys and the stripped argument string still
         as a string.
 
         arg_str is quoted as a true sys.argv vector by using shlex.split.
         This allows us to easily expand variables, glob files, quote
         arguments, etc.
 
         Options:
           -mode: default 'string'. If given as 'list', the argument string is
           returned as a list (split on whitespace) instead of a string.
 
           -list_all: put all option values in lists. Normally only options
           appearing more than once are put in a list.
 
           -posix (True): whether to split the input line in POSIX mode or not,
           as per the conventions outlined in the shlex module from the
           standard library."""
  
         # inject default options at the beginning of the input line
         caller = sys._getframe(1).f_code.co_name.replace('magic_','')
         arg_str = '%s %s' % (self.options_table.get(caller,''),arg_str)
         
         mode = kw.get('mode','string')
         if mode not in ['string','list']:
             raise ValueError,'incorrect mode given: %s' % mode
         # Get options
         list_all = kw.get('list_all',0)
         posix = kw.get('posix',True)
 
         # Check if we have more than one argument to warrant extra processing:
         odict = {}  # Dictionary with options
         args = arg_str.split()
         if len(args) >= 1:
             # If the list of inputs only has 0 or 1 thing in it, there's no
             # need to look for options
             argv = arg_split(arg_str,posix)
             # Do regular option processing
             try:
                 opts,args = getopt(argv,opt_str,*long_opts)
             except GetoptError,e:
                 raise UsageError('%s ( allowed: "%s" %s)' % (e.msg,opt_str, 
                                         " ".join(long_opts)))
             for o,a in opts:
                 if o.startswith('--'):
                     o = o[2:]
                 else:
                     o = o[1:]
                 try:
                     odict[o].append(a)
                 except AttributeError:
                     odict[o] = [odict[o],a]
                 except KeyError:
                     if list_all:
                         odict[o] = [a]
                     else:
                         odict[o] = a
 
         # Prepare opts,args for return
         opts = Struct(odict)
         if mode == 'string':
             args = ' '.join(args)
 
         return opts,args
     
     #......................................................................
     # And now the actual magic functions
 
     # Functions for IPython shell work (vars,funcs, config, etc)
     def magic_lsmagic(self, parameter_s = ''):
         """List currently available magic functions."""
         mesc = self.shell.ESC_MAGIC
         print 'Available magic functions:\n'+mesc+\
               ('  '+mesc).join(self.lsmagic())
         print '\n' + Magic.auto_status[self.shell.rc.automagic]
         return None
         
     def magic_magic(self, parameter_s = ''):
         """Print information about the magic function system.
         
         Supported formats: -latex, -brief, -rest        
         """
 
         mode = ''
         try:
             if parameter_s.split()[0] == '-latex':
                 mode = 'latex'
             if parameter_s.split()[0] == '-brief':
                 mode = 'brief'
             if parameter_s.split()[0] == '-rest':
                 mode = 'rest'
                 rest_docs = []
         except:
             pass
 
         magic_docs = []
         for fname in self.lsmagic():
             mname = 'magic_' + fname
             for space in (Magic,self,self.__class__):
                 try:
                     fn = space.__dict__[mname]
                 except KeyError:
                     pass
                 else:
                     break
             if mode == 'brief':
                 # only first line
                 if fn.__doc__:                    
                     fndoc = fn.__doc__.split('\n',1)[0]
                 else:
                     fndoc = 'No documentation'
             else:
                 fndoc = fn.__doc__.rstrip()
                 
             if mode == 'rest':
                 rest_docs.append('**%s%s**::\n\n\t%s\n\n' %(self.shell.ESC_MAGIC,
                                                     fname,fndoc))
                 
             else:
                 magic_docs.append('%s%s:\n\t%s\n' %(self.shell.ESC_MAGIC,
                                                     fname,fndoc))
                 
         magic_docs = ''.join(magic_docs)
 
         if mode == 'rest':
             return "".join(rest_docs)
         
         if mode == 'latex':
             print self.format_latex(magic_docs)
             return
         else:
             magic_docs = self.format_screen(magic_docs)
         if mode == 'brief':
             return magic_docs
         
         outmsg = """
 IPython's 'magic' functions
 ===========================
 
 The magic function system provides a series of functions which allow you to
 control the behavior of IPython itself, plus a lot of system-type
 features. All these functions are prefixed with a % character, but parameters
 are given without parentheses or quotes.
 
 NOTE: If you have 'automagic' enabled (via the command line option or with the
 %automagic function), you don't need to type in the % explicitly.  By default,
 IPython ships with automagic on, so you should only rarely need the % escape.
 
 Example: typing '%cd mydir' (without the quotes) changes you working directory
 to 'mydir', if it exists.
 
 You can define your own magic functions to extend the system. See the supplied
 ipythonrc and example-magic.py files for details (in your ipython
 configuration directory, typically $HOME/.ipython/).
 
 You can also define your own aliased names for magic functions. In your
 ipythonrc file, placing a line like:
 
   execute __IPYTHON__.magic_pf = __IPYTHON__.magic_profile
 
 will define %pf as a new name for %profile.
 
 You can also call magics in code using the ipmagic() function, which IPython
 automatically adds to the builtin namespace.  Type 'ipmagic?' for details.
 
 For a list of the available magic functions, use %lsmagic. For a description
 of any of them, type %magic_name?, e.g. '%cd?'.
 
 Currently the magic system has the following functions:\n"""
 
         mesc = self.shell.ESC_MAGIC
         outmsg = ("%s\n%s\n\nSummary of magic functions (from %slsmagic):"
                   "\n\n%s%s\n\n%s" % (outmsg,
                                      magic_docs,mesc,mesc,
                                      ('  '+mesc).join(self.lsmagic()),
                                      Magic.auto_status[self.shell.rc.automagic] ) )
 
         page(outmsg,screen_lines=self.shell.rc.screen_length)
   
 
     def magic_autoindent(self, parameter_s = ''):
         """Toggle autoindent on/off (if available)."""
 
         self.shell.set_autoindent()
         print "Automatic indentation is:",['OFF','ON'][self.shell.autoindent]
 
 
     def magic_automagic(self, parameter_s = ''):
         """Make magic functions callable without having to type the initial %.
 
         Without argumentsl toggles on/off (when off, you must call it as
         %automagic, of course).  With arguments it sets the value, and you can
         use any of (case insensitive):
 
          - on,1,True: to activate
 
          - off,0,False: to deactivate.
 
         Note that magic functions have lowest priority, so if there's a
         variable whose name collides with that of a magic fn, automagic won't
         work for that function (you get the variable instead). However, if you
         delete the variable (del var), the previously shadowed magic function
         becomes visible to automagic again."""
 
         rc = self.shell.rc
         arg = parameter_s.lower()
         if parameter_s in ('on','1','true'):
             rc.automagic = True
         elif parameter_s in ('off','0','false'):
             rc.automagic = False
         else:
             rc.automagic = not rc.automagic
         print '\n' + Magic.auto_status[rc.automagic]
 
 
     def magic_autocall(self, parameter_s = ''):
         """Make functions callable without having to type parentheses.
 
         Usage:
 
            %autocall [mode]
 
         The mode can be one of: 0->Off, 1->Smart, 2->Full.  If not given, the
         value is toggled on and off (remembering the previous state).
 
         In more detail, these values mean:
 
         0 -> fully disabled
 
         1 -> active, but do not apply if there are no arguments on the line.
 
         In this mode, you get:
 
         In [1]: callable
         Out[1]: <built-in function callable>
 
         In [2]: callable 'hello'
         ------> callable('hello')
         Out[2]: False
 
         2 -> Active always.  Even if no arguments are present, the callable
         object is called:
 
         In [4]: callable
         ------> callable()
 
         Note that even with autocall off, you can still use '/' at the start of
         a line to treat the first argument on the command line as a function
         and add parentheses to it:
 
         In [8]: /str 43
         ------> str(43)
         Out[8]: '43'
         """
 
         rc = self.shell.rc
 
         if parameter_s:
             arg = int(parameter_s)
         else:
             arg = 'toggle'
 
         if not arg in (0,1,2,'toggle'):
             error('Valid modes: (0->Off, 1->Smart, 2->Full')
             return
 
         if arg in (0,1,2):
             rc.autocall = arg
         else: # toggle
             if rc.autocall:
                 self._magic_state.autocall_save = rc.autocall
                 rc.autocall = 0
             else:
                 try:
                     rc.autocall = self._magic_state.autocall_save
                 except AttributeError:
                     rc.autocall = self._magic_state.autocall_save = 1
 
         print "Automatic calling is:",['OFF','Smart','Full'][rc.autocall]
 
     def magic_system_verbose(self, parameter_s = ''):
         """Set verbose printing of system calls.
 
         If called without an argument, act as a toggle"""
 
         if parameter_s:
             val = bool(eval(parameter_s))
         else:
             val = None
             
         self.shell.rc_set_toggle('system_verbose',val)
         print "System verbose printing is:",\
               ['OFF','ON'][self.shell.rc.system_verbose]
 
 
     def magic_page(self, parameter_s=''):
         """Pretty print the object and display it through a pager.
         
         %page [options] OBJECT
 
         If no object is given, use _ (last output).
         
         Options:
 
           -r: page str(object), don't pretty-print it."""
 
         # After a function contributed by Olivier Aubert, slightly modified.
 
         # Process options/args
         opts,args = self.parse_options(parameter_s,'r')
         raw = 'r' in opts
 
         oname = args and args or '_'
         info = self._ofind(oname)
         if info['found']:
             txt = (raw and str or pformat)( info['obj'] )
             page(txt)
         else:
             print 'Object `%s` not found' % oname
 
     def magic_profile(self, parameter_s=''):
         """Print your currently active IPyhton profile."""
         if self.shell.rc.profile:
             printpl('Current IPython profile: $self.shell.rc.profile.')
         else:
             print 'No profile active.'
 
     def magic_pinfo(self, parameter_s='', namespaces=None):
         """Provide detailed information about an object.
     
         '%pinfo object' is just a synonym for object? or ?object."""
     
         #print 'pinfo par: <%s>' % parameter_s  # dbg
     
         
         # detail_level: 0 -> obj? , 1 -> obj??
         detail_level = 0
         # We need to detect if we got called as 'pinfo pinfo foo', which can
         # happen if the user types 'pinfo foo?' at the cmd line.
         pinfo,qmark1,oname,qmark2 = \
                re.match('(pinfo )?(\?*)(.*?)(\??$)',parameter_s).groups()
         if pinfo or qmark1 or qmark2:
             detail_level = 1
         if "*" in oname:
             self.magic_psearch(oname)
         else:
             self._inspect('pinfo', oname, detail_level=detail_level,
                           namespaces=namespaces)
 
     def magic_pdef(self, parameter_s='', namespaces=None):
         """Print the definition header for any callable object.
 
         If the object is a class, print the constructor information."""
         self._inspect('pdef',parameter_s, namespaces)
 
     def magic_pdoc(self, parameter_s='', namespaces=None):
         """Print the docstring for an object.
 
         If the given object is a class, it will print both the class and the
         constructor docstrings."""
         self._inspect('pdoc',parameter_s, namespaces)
 
     def magic_psource(self, parameter_s='', namespaces=None):
         """Print (or run through pager) the source code for an object."""
         self._inspect('psource',parameter_s, namespaces)
 
     def magic_pfile(self, parameter_s=''):
         """Print (or run through pager) the file where an object is defined.
 
         The file opens at the line where the object definition begins. IPython
         will honor the environment variable PAGER if set, and otherwise will
         do its best to print the file in a convenient form.
 
         If the given argument is not an object currently defined, IPython will
         try to interpret it as a filename (automatically adding a .py extension
         if needed). You can thus use %pfile as a syntax highlighting code
         viewer."""
 
         # first interpret argument as an object name
         out = self._inspect('pfile',parameter_s)
         # if not, try the input as a filename
         if out == 'not found':
             try:
                 filename = get_py_filename(parameter_s)
             except IOError,msg:
                 print msg
                 return
             page(self.shell.inspector.format(file(filename).read()))
 
     def _inspect(self,meth,oname,namespaces=None,**kw):
         """Generic interface to the inspector system.
 
         This function is meant to be called by pdef, pdoc & friends."""
 
         #oname = oname.strip()
         #print '1- oname: <%r>' % oname  # dbg
         try:
             oname = oname.strip().encode('ascii')
             #print '2- oname: <%r>' % oname  # dbg
         except UnicodeEncodeError:
             print 'Python identifiers can only contain ascii characters.'
             return 'not found'
             
         info = Struct(self._ofind(oname, namespaces))
         
         if info.found:
             try:
                 IPython.generics.inspect_object(info.obj)
                 return
             except IPython.ipapi.TryNext:
                 pass
             # Get the docstring of the class property if it exists.
             path = oname.split('.')
             root = '.'.join(path[:-1])
             if info.parent is not None:
                 try:
                     target = getattr(info.parent, '__class__') 
                     # The object belongs to a class instance. 
                     try: 
                         target = getattr(target, path[-1])
                         # The class defines the object. 
                         if isinstance(target, property):
                             oname = root + '.__class__.' + path[-1]
                             info = Struct(self._ofind(oname))
                     except AttributeError: pass
                 except AttributeError: pass
                         
             pmethod = getattr(self.shell.inspector,meth)
             formatter = info.ismagic and self.format_screen or None
             if meth == 'pdoc':
                 pmethod(info.obj,oname,formatter)
             elif meth == 'pinfo':
                 pmethod(info.obj,oname,formatter,info,**kw)
             else:
                 pmethod(info.obj,oname)
         else:
             print 'Object `%s` not found.' % oname
             return 'not found'  # so callers can take other action
         
     def magic_psearch(self, parameter_s=''):
         """Search for object in namespaces by wildcard.
 
         %psearch [options] PATTERN [OBJECT TYPE]
 
         Note: ? can be used as a synonym for %psearch, at the beginning or at
         the end: both a*? and ?a* are equivalent to '%psearch a*'.  Still, the
         rest of the command line must be unchanged (options come first), so
         for example the following forms are equivalent
 
         %psearch -i a* function
         -i a* function?
         ?-i a* function
 
         Arguments:
         
           PATTERN
 
           where PATTERN is a string containing * as a wildcard similar to its
           use in a shell.  The pattern is matched in all namespaces on the
           search path. By default objects starting with a single _ are not
           matched, many IPython generated objects have a single
           underscore. The default is case insensitive matching. Matching is
           also done on the attributes of objects and not only on the objects
           in a module.
 
           [OBJECT TYPE]
 
           Is the name of a python type from the types module. The name is
           given in lowercase without the ending type, ex. StringType is
           written string. By adding a type here only objects matching the
           given type are matched. Using all here makes the pattern match all
           types (this is the default).
 
         Options:
 
           -a: makes the pattern match even objects whose names start with a
           single underscore.  These names are normally ommitted from the
           search.
 
           -i/-c: make the pattern case insensitive/sensitive.  If neither of
           these options is given, the default is read from your ipythonrc
           file.  The option name which sets this value is
           'wildcards_case_sensitive'.  If this option is not specified in your
           ipythonrc file, IPython's internal default is to do a case sensitive
           search.
 
           -e/-s NAMESPACE: exclude/search a given namespace.  The pattern you
           specifiy can be searched in any of the following namespaces:
           'builtin', 'user', 'user_global','internal', 'alias', where
           'builtin' and 'user' are the search defaults.  Note that you should
           not use quotes when specifying namespaces.
 
           'Builtin' contains the python module builtin, 'user' contains all
           user data, 'alias' only contain the shell aliases and no python
           objects, 'internal' contains objects used by IPython.  The
           'user_global' namespace is only used by embedded IPython instances,
           and it contains module-level globals.  You can add namespaces to the
           search with -s or exclude them with -e (these options can be given
           more than once).
     
         Examples:
        
         %psearch a*            -> objects beginning with an a
         %psearch -e builtin a* -> objects NOT in the builtin space starting in a
         %psearch a* function   -> all functions beginning with an a
         %psearch re.e*         -> objects beginning with an e in module re
         %psearch r*.e*         -> objects that start with e in modules starting in r
         %psearch r*.* string   -> all strings in modules beginning with r
 
         Case sensitve search:
        
         %psearch -c a*         list all object beginning with lower case a
 
         Show objects beginning with a single _:
        
         %psearch -a _*         list objects beginning with a single underscore"""
         try:
             parameter_s = parameter_s.encode('ascii')
         except UnicodeEncodeError:
             print 'Python identifiers can only contain ascii characters.'
             return
 
         # default namespaces to be searched
         def_search = ['user','builtin']
 
         # Process options/args
         opts,args = self.parse_options(parameter_s,'cias:e:',list_all=True)
         opt = opts.get
         shell = self.shell
         psearch = shell.inspector.psearch
 
         # select case options
         if opts.has_key('i'):
             ignore_case = True
         elif opts.has_key('c'):
             ignore_case = False
         else:
             ignore_case = not shell.rc.wildcards_case_sensitive
 
         # Build list of namespaces to search from user options
         def_search.extend(opt('s',[]))
         ns_exclude = ns_exclude=opt('e',[])
         ns_search = [nm for nm in def_search if nm not in ns_exclude]
         
         # Call the actual search
         try:
             psearch(args,shell.ns_table,ns_search,
                     show_all=opt('a'),ignore_case=ignore_case)
         except:
             shell.showtraceback()
 
     def magic_who_ls(self, parameter_s=''):
         """Return a sorted list of all interactive variables.
 
         If arguments are given, only variables of types matching these
         arguments are returned."""
 
         user_ns = self.shell.user_ns
         internal_ns = self.shell.internal_ns
         user_config_ns = self.shell.user_config_ns
         out = []
         typelist = parameter_s.split()
 
         for i in user_ns:
             if not (i.startswith('_') or i.startswith('_i')) \
                    and not (i in internal_ns or i in user_config_ns):
                 if typelist:
                     if type(user_ns[i]).__name__ in typelist:
                         out.append(i)
                 else:
                     out.append(i)
         out.sort()
         return out
         
     def magic_who(self, parameter_s=''):
         """Print all interactive variables, with some minimal formatting.
 
         If any arguments are given, only variables whose type matches one of
         these are printed.  For example:
 
           %who function str
 
         will only list functions and strings, excluding all other types of
         variables.  To find the proper type names, simply use type(var) at a
         command line to see how python prints type names.  For example:
 
           In [1]: type('hello')\\
           Out[1]: <type 'str'>
 
         indicates that the type name for strings is 'str'.
 
         %who always excludes executed names loaded through your configuration
         file and things which are internal to IPython.
 
         This is deliberate, as typically you may load many modules and the
         purpose of %who is to show you only what you've manually defined."""
 
         varlist = self.magic_who_ls(parameter_s)
         if not varlist:
             if parameter_s:
                 print 'No variables match your requested type.'
             else:
                 print 'Interactive namespace is empty.'
             return
 
         # if we have variables, move on...
         count = 0
         for i in varlist:
             print i+'\t',
             count += 1
             if count > 8:
                 count = 0
                 print
         print
 
     def magic_whos(self, parameter_s=''):
         """Like %who, but gives some extra information about each variable.
 
         The same type filtering of %who can be applied here.
 
         For all variables, the type is printed. Additionally it prints:
         
           - For {},[],(): their length.
 
           - For numpy and Numeric arrays, a summary with shape, number of
           elements, typecode and size in memory.
 
           - Everything else: a string representation, snipping their middle if
           too long."""
         
         varnames = self.magic_who_ls(parameter_s)
         if not varnames:
             if parameter_s:
                 print 'No variables match your requested type.'
             else:
                 print 'Interactive namespace is empty.'
             return
 
         # if we have variables, move on...
 
         # for these types, show len() instead of data:
         seq_types = [types.DictType,types.ListType,types.TupleType]
 
         # for numpy/Numeric arrays, display summary info
         try:
             import numpy
         except ImportError:
             ndarray_type = None
         else:
             ndarray_type = numpy.ndarray.__name__
         try:
             import Numeric
         except ImportError:
             array_type = None
         else:
             array_type = Numeric.ArrayType.__name__
 
         # Find all variable names and types so we can figure out column sizes
         def get_vars(i):
             return self.shell.user_ns[i]
         
         # some types are well known and can be shorter
         abbrevs = {'IPython.macro.Macro' : 'Macro'}
         def type_name(v):
             tn = type(v).__name__
             return abbrevs.get(tn,tn)
             
         varlist = map(get_vars,varnames)
 
         typelist = []
         for vv in varlist:
             tt = type_name(vv)
 
             if tt=='instance':
                 typelist.append( abbrevs.get(str(vv.__class__),
                                              str(vv.__class__)))
             else:
                 typelist.append(tt)
 
         # column labels and # of spaces as separator
         varlabel = 'Variable'
         typelabel = 'Type'
         datalabel = 'Data/Info'
         colsep = 3
         # variable format strings
         vformat    = "$vname.ljust(varwidth)$vtype.ljust(typewidth)"
         vfmt_short = '$vstr[:25]<...>$vstr[-25:]'
         aformat    = "%s: %s elems, type `%s`, %s bytes"
         # find the size of the columns to format the output nicely
         varwidth = max(max(map(len,varnames)), len(varlabel)) + colsep
         typewidth = max(max(map(len,typelist)), len(typelabel)) + colsep
         # table header
         print varlabel.ljust(varwidth) + typelabel.ljust(typewidth) + \
               ' '+datalabel+'\n' + '-'*(varwidth+typewidth+len(datalabel)+1)
         # and the table itself
         kb = 1024
         Mb = 1048576  # kb**2
         for vname,var,vtype in zip(varnames,varlist,typelist):
             print itpl(vformat),
             if vtype in seq_types:
                 print len(var)
             elif vtype in [array_type,ndarray_type]:
                 vshape = str(var.shape).replace(',','').replace(' ','x')[1:-1]
                 if vtype==ndarray_type:
                     # numpy
                     vsize  = var.size
                     vbytes = vsize*var.itemsize
                     vdtype = var.dtype
                 else:
                     # Numeric
                     vsize  = Numeric.size(var)
                     vbytes = vsize*var.itemsize()
                     vdtype = var.typecode()
                     
                 if vbytes < 100000:
                     print aformat % (vshape,vsize,vdtype,vbytes)
                 else:
                     print aformat % (vshape,vsize,vdtype,vbytes),
                     if vbytes < Mb:
                         print '(%s kb)' % (vbytes/kb,)
                     else:
                         print '(%s Mb)' % (vbytes/Mb,)
             else:
                 try:
                     vstr = str(var)
                 except UnicodeEncodeError:
                     vstr = unicode(var).encode(sys.getdefaultencoding(),
                                                'backslashreplace')
                 vstr = vstr.replace('\n','\\n')
                 if len(vstr) < 50:
                     print vstr
                 else:
                     printpl(vfmt_short)
                 
     def magic_reset(self, parameter_s=''):
         """Resets the namespace by removing all names defined by the user.
 
         Input/Output history are left around in case you need them."""
 
         ans = self.shell.ask_yes_no(
           "Once deleted, variables cannot be recovered. Proceed (y/[n])? ")
         if not ans:
             print 'Nothing done.'
             return
         user_ns = self.shell.user_ns
         for i in self.magic_who_ls():
             del(user_ns[i])
             
         # Also flush the private list of module references kept for script
         # execution protection
         self.shell._user_main_modules[:] = []
 
     def magic_logstart(self,parameter_s=''):
         """Start logging anywhere in a session.
 
         %logstart [-o|-r|-t] [log_name [log_mode]]
 
         If no name is given, it defaults to a file named 'ipython_log.py' in your
         current directory, in 'rotate' mode (see below).
 
         '%logstart name' saves to file 'name' in 'backup' mode.  It saves your
         history up to that point and then continues logging.
 
         %logstart takes a second optional parameter: logging mode. This can be one
         of (note that the modes are given unquoted):\\
           append: well, that says it.\\
           backup: rename (if exists) to name~ and start name.\\
           global: single logfile in your home dir, appended to.\\
           over  : overwrite existing log.\\
           rotate: create rotating logs name.1~, name.2~, etc.
 
         Options:
 
           -o: log also IPython's output.  In this mode, all commands which
           generate an Out[NN] prompt are recorded to the logfile, right after
           their corresponding input line.  The output lines are always
           prepended with a '#[Out]# ' marker, so that the log remains valid
           Python code.
 
           Since this marker is always the same, filtering only the output from
           a log is very easy, using for example a simple awk call:
 
             awk -F'#\\[Out\\]# ' '{if($2) {print $2}}' ipython_log.py
 
           -r: log 'raw' input.  Normally, IPython's logs contain the processed
           input, so that user lines are logged in their final form, converted
           into valid Python.  For example, %Exit is logged as
           '_ip.magic("Exit").  If the -r flag is given, all input is logged
           exactly as typed, with no transformations applied.
 
           -t: put timestamps before each input line logged (these are put in
           comments)."""
         
         opts,par = self.parse_options(parameter_s,'ort')
         log_output = 'o' in opts
         log_raw_input = 'r' in opts
         timestamp = 't' in opts
 
         rc = self.shell.rc
         logger = self.shell.logger
 
         # if no args are given, the defaults set in the logger constructor by
         # ipytohn remain valid
         if par:
             try:
                 logfname,logmode = par.split()
             except:
                 logfname = par
                 logmode = 'backup'
         else:
             logfname = logger.logfname
             logmode = logger.logmode
         # put logfname into rc struct as if it had been called on the command
         # line, so it ends up saved in the log header Save it in case we need
         # to restore it...
         old_logfile = rc.opts.get('logfile','')  
         if logfname:
             logfname = os.path.expanduser(logfname)
         rc.opts.logfile = logfname
         loghead = self.shell.loghead_tpl % (rc.opts,rc.args)
         try:
             started  = logger.logstart(logfname,loghead,logmode,
                                        log_output,timestamp,log_raw_input)
         except:
             rc.opts.logfile = old_logfile
             warn("Couldn't start log: %s" % sys.exc_info()[1])
         else:
             # log input history up to this point, optionally interleaving
             # output if requested
 
             if timestamp:
                 # disable timestamping for the previous history, since we've
                 # lost those already (no time machine here).
                 logger.timestamp = False
 
             if log_raw_input:
                 input_hist = self.shell.input_hist_raw
             else:
                 input_hist = self.shell.input_hist
                 
             if log_output:
                 log_write = logger.log_write
                 output_hist = self.shell.output_hist
                 for n in range(1,len(input_hist)-1):
                     log_write(input_hist[n].rstrip())
                     if n in output_hist:
                         log_write(repr(output_hist[n]),'output')
             else:
                 logger.log_write(input_hist[1:])
             if timestamp:
                 # re-enable timestamping
                 logger.timestamp = True
                 
             print ('Activating auto-logging. '
                    'Current session state plus future input saved.')
             logger.logstate()
 
     def magic_logstop(self,parameter_s=''):
         """Fully stop logging and close log file.
 
         In order to start logging again, a new %logstart call needs to be made,
         possibly (though not necessarily) with a new filename, mode and other
         options."""
         self.logger.logstop()
 
     def magic_logoff(self,parameter_s=''):
         """Temporarily stop logging.
 
         You must have previously started logging."""
         self.shell.logger.switch_log(0)
         
     def magic_logon(self,parameter_s=''):
         """Restart logging.
 
         This function is for restarting logging which you've temporarily
         stopped with %logoff. For starting logging for the first time, you
         must use the %logstart function, which allows you to specify an
         optional log filename."""
         
         self.shell.logger.switch_log(1)
     
     def magic_logstate(self,parameter_s=''):
         """Print the status of the logging system."""
 
         self.shell.logger.logstate()
         
     def magic_pdb(self, parameter_s=''):
         """Control the automatic calling of the pdb interactive debugger.
 
         Call as '%pdb on', '%pdb 1', '%pdb off' or '%pdb 0'. If called without
         argument it works as a toggle.
 
         When an exception is triggered, IPython can optionally call the
         interactive pdb debugger after the traceback printout. %pdb toggles
         this feature on and off.
 
         The initial state of this feature is set in your ipythonrc
         configuration file (the variable is called 'pdb').
 
         If you want to just activate the debugger AFTER an exception has fired,
         without having to type '%pdb on' and rerunning your code, you can use
         the %debug magic."""
 
         par = parameter_s.strip().lower()
 
         if par:
             try:
                 new_pdb = {'off':0,'0':0,'on':1,'1':1}[par]
             except KeyError:
                 print ('Incorrect argument. Use on/1, off/0, '
                        'or nothing for a toggle.')
                 return
         else:
             # toggle
             new_pdb = not self.shell.call_pdb
 
         # set on the shell
         self.shell.call_pdb = new_pdb
         print 'Automatic pdb calling has been turned',on_off(new_pdb)
 
     def magic_debug(self, parameter_s=''):
         """Activate the interactive debugger in post-mortem mode.
 
         If an exception has just occurred, this lets you inspect its stack
         frames interactively.  Note that this will always work only on the last
         traceback that occurred, so you must call this quickly after an
         exception that you wish to inspect has fired, because if another one
         occurs, it clobbers the previous one.
 
         If you want IPython to automatically do this on every exception, see
         the %pdb magic for more details.
         """
         
         self.shell.debugger(force=True)
 
     def magic_prun(self, parameter_s ='',user_mode=1,
                    opts=None,arg_lst=None,prog_ns=None):
 
         """Run a statement through the python code profiler.
 
         Usage:\\
           %prun [options] statement
 
         The given statement (which doesn't require quote marks) is run via the
         python profiler in a manner similar to the profile.run() function.
         Namespaces are internally managed to work correctly; profile.run
         cannot be used in IPython because it makes certain assumptions about
         namespaces which do not hold under IPython.
 
         Options:
 
         -l <limit>: you can place restrictions on what or how much of the
         profile gets printed. The limit value can be:
 
           * A string: only information for function names containing this string
           is printed.
 
           * An integer: only these many lines are printed.
 
           * A float (between 0 and 1): this fraction of the report is printed
           (for example, use a limit of 0.4 to see the topmost 40% only).
 
         You can combine several limits with repeated use of the option. For
         example, '-l __init__ -l 5' will print only the topmost 5 lines of
         information about class constructors.
 
         -r: return the pstats.Stats object generated by the profiling. This
         object has all the information about the profile in it, and you can
         later use it for further analysis or in other functions.
 
        -s <key>: sort profile by given key. You can provide more than one key
         by using the option several times: '-s key1 -s key2 -s key3...'. The
         default sorting key is 'time'.
 
         The following is copied verbatim from the profile documentation
         referenced below:
 
         When more than one key is provided, additional keys are used as
         secondary criteria when the there is equality in all keys selected
         before them.
         
         Abbreviations can be used for any key names, as long as the
         abbreviation is unambiguous.  The following are the keys currently
         defined:
 
                 Valid Arg       Meaning\\
                   "calls"      call count\\
                   "cumulative" cumulative time\\
                   "file"       file name\\
                   "module"     file name\\
                   "pcalls"     primitive call count\\
                   "line"       line number\\
                   "name"       function name\\
                   "nfl"        name/file/line\\
                   "stdname"    standard name\\
                   "time"       internal time
 
         Note that all sorts on statistics are in descending order (placing
         most time consuming items first), where as name, file, and line number
         searches are in ascending order (i.e., alphabetical). The subtle
         distinction between "nfl" and "stdname" is that the standard name is a
         sort of the name as printed, which means that the embedded line
         numbers get compared in an odd way.  For example, lines 3, 20, and 40
         would (if the file names were the same) appear in the string order
         "20" "3" and "40".  In contrast, "nfl" does a numeric compare of the
         line numbers.  In fact, sort_stats("nfl") is the same as
         sort_stats("name", "file", "line").
 
         -T <filename>: save profile results as shown on screen to a text
         file. The profile is still shown on screen.
 
         -D <filename>: save (via dump_stats) profile statistics to given
         filename. This data is in a format understod by the pstats module, and
         is generated by a call to the dump_stats() method of profile
         objects. The profile is still shown on screen.
 
         If you want to run complete programs under the profiler's control, use
         '%run -p [prof_opts] filename.py [args to program]' where prof_opts
         contains profiler specific options as described here.
         
         You can read the complete documentation for the profile module with:\\
           In [1]: import profile; profile.help() """
 
         opts_def = Struct(D=[''],l=[],s=['time'],T=[''])
         # protect user quote marks
         parameter_s = parameter_s.replace('"',r'\"').replace("'",r"\'")
         
         if user_mode:  # regular user call
             opts,arg_str = self.parse_options(parameter_s,'D:l:rs:T:',
                                               list_all=1)
             namespace = self.shell.user_ns
         else:  # called to run a program by %run -p
             try:
                 filename = get_py_filename(arg_lst[0])
             except IOError,msg:
                 error(msg)
                 return
 
             arg_str = 'execfile(filename,prog_ns)'
             namespace = locals()
 
         opts.merge(opts_def)
         
         prof = profile.Profile()
         try:
             prof = prof.runctx(arg_str,namespace,namespace)
             sys_exit = ''
         except SystemExit:
             sys_exit = """*** SystemExit exception caught in code being profiled."""
 
         stats = pstats.Stats(prof).strip_dirs().sort_stats(*opts.s)
 
         lims = opts.l
         if lims:
             lims = []  # rebuild lims with ints/floats/strings
             for lim in opts.l:
                 try:
                     lims.append(int(lim))
                 except ValueError:
                     try:
                         lims.append(float(lim))
                     except ValueError:
                         lims.append(lim)
                     
         # Trap output.
         stdout_trap = StringIO()
 
         if hasattr(stats,'stream'):
             # In newer versions of python, the stats object has a 'stream'
             # attribute to write into.
             stats.stream = stdout_trap
             stats.print_stats(*lims)
         else:
             # For older versions, we manually redirect stdout during printing
             sys_stdout = sys.stdout
             try:
                 sys.stdout = stdout_trap
                 stats.print_stats(*lims)
             finally:
                 sys.stdout = sys_stdout
                 
         output = stdout_trap.getvalue()
         output = output.rstrip()
 
         page(output,screen_lines=self.shell.rc.screen_length)
         print sys_exit,
 
         dump_file = opts.D[0]
         text_file = opts.T[0]
         if dump_file:
             prof.dump_stats(dump_file)
             print '\n*** Profile stats marshalled to file',\
                   `dump_file`+'.',sys_exit
         if text_file:
             pfile = file(text_file,'w')
             pfile.write(output)
             pfile.close()
             print '\n*** Profile printout saved to text file',\
                   `text_file`+'.',sys_exit
 
         if opts.has_key('r'):
             return stats
         else:
             return None
 
     def magic_run(self, parameter_s ='',runner=None):
         """Run the named file inside IPython as a program.
 
         Usage:\\
           %run [-n -i -t [-N<N>] -d [-b<N>] -p [profile options]] file [args]
         
         Parameters after the filename are passed as command-line arguments to
         the program (put in sys.argv). Then, control returns to IPython's
         prompt.
 
         This is similar to running at a system prompt:\\
           $ python file args\\
         but with the advantage of giving you IPython's tracebacks, and of
         loading all variables into your interactive namespace for further use
         (unless -p is used, see below).
 
         The file is executed in a namespace initially consisting only of
         __name__=='__main__' and sys.argv constructed as indicated. It thus
         sees its environment as if it were being run as a stand-alone program
         (except for sharing global objects such as previously imported
         modules). But after execution, the IPython interactive namespace gets
         updated with all variables defined in the program (except for __name__
         and sys.argv). This allows for very convenient loading of code for
         interactive work, while giving each program a 'clean sheet' to run in.
 
         Options:
         
         -n: __name__ is NOT set to '__main__', but to the running file's name
         without extension (as python does under import).  This allows running
         scripts and reloading the definitions in them without calling code
         protected by an ' if __name__ == "__main__" ' clause.
 
         -i: run the file in IPython's namespace instead of an empty one. This
         is useful if you are experimenting with code written in a text editor
         which depends on variables defined interactively.
 
         -e: ignore sys.exit() calls or SystemExit exceptions in the script
         being run.  This is particularly useful if IPython is being used to
         run unittests, which always exit with a sys.exit() call.  In such
         cases you are interested in the output of the test results, not in
         seeing a traceback of the unittest module.
 
         -t: print timing information at the end of the run.  IPython will give
         you an estimated CPU time consumption for your script, which under
         Unix uses the resource module to avoid the wraparound problems of
         time.clock().  Under Unix, an estimate of time spent on system tasks
         is also given (for Windows platforms this is reported as 0.0).
 
         If -t is given, an additional -N<N> option can be given, where <N>
         must be an integer indicating how many times you want the script to
         run.  The final timing report will include total and per run results.
 
         For example (testing the script uniq_stable.py):
 
             In [1]: run -t uniq_stable
 
             IPython CPU timings (estimated):\\
               User  :    0.19597 s.\\
               System:        0.0 s.\\
 
             In [2]: run -t -N5 uniq_stable
 
             IPython CPU timings (estimated):\\
             Total runs performed: 5\\
               Times :      Total       Per run\\
               User  :   0.910862 s,  0.1821724 s.\\
               System:        0.0 s,        0.0 s.
 
         -d: run your program under the control of pdb, the Python debugger.
         This allows you to execute your program step by step, watch variables,
         etc.  Internally, what IPython does is similar to calling:
         
           pdb.run('execfile("YOURFILENAME")')
 
         with a breakpoint set on line 1 of your file.  You can change the line
         number for this automatic breakpoint to be <N> by using the -bN option
         (where N must be an integer).  For example:
 
           %run -d -b40 myscript
 
         will set the first breakpoint at line 40 in myscript.py.  Note that
         the first breakpoint must be set on a line which actually does
         something (not a comment or docstring) for it to stop execution.
 
         When the pdb debugger starts, you will see a (Pdb) prompt.  You must
         first enter 'c' (without qoutes) to start execution up to the first
         breakpoint.
 
         Entering 'help' gives information about the use of the debugger.  You
         can easily see pdb's full documentation with "import pdb;pdb.help()"
         at a prompt.
 
         -p: run program under the control of the Python profiler module (which
         prints a detailed report of execution times, function calls, etc).
 
         You can pass other options after -p which affect the behavior of the
         profiler itself. See the docs for %prun for details.
 
         In this mode, the program's variables do NOT propagate back to the
         IPython interactive namespace (because they remain in the namespace
         where the profiler executes them).
 
         Internally this triggers a call to %prun, see its documentation for
         details on the options available specifically for profiling.
 
         There is one special usage for which the text above doesn't apply:
         if the filename ends with .ipy, the file is run as ipython script,
         just as if the commands were written on IPython prompt.
         """
 
         # get arguments and set sys.argv for program to be run.
         opts,arg_lst = self.parse_options(parameter_s,'nidtN:b:pD:l:rs:T:e',
                                           mode='list',list_all=1)
 
         try:
             filename = get_py_filename(arg_lst[0])
         except IndexError:
             warn('you must provide at least a filename.')
             print '\n%run:\n',OInspect.getdoc(self.magic_run)
             return
         except IOError,msg:
             error(msg)
             return
 
         if filename.lower().endswith('.ipy'):
             self.api.runlines(open(filename).read())
             return
         
         # Control the response to exit() calls made by the script being run
         exit_ignore = opts.has_key('e')
         
         # Make sure that the running script gets a proper sys.argv as if it
         # were run from a system shell.
         save_argv = sys.argv # save it for later restoring
         sys.argv = [filename]+ arg_lst[1:]  # put in the proper filename
 
         if opts.has_key('i'):
             # Run in user's interactive namespace
             prog_ns = self.shell.user_ns
             __name__save = self.shell.user_ns['__name__']
             prog_ns['__name__'] = '__main__'
             main_mod = FakeModule(prog_ns)
         else:
             # Run in a fresh, empty namespace
             if opts.has_key('n'):
                 name = os.path.splitext(os.path.basename(filename))[0]
             else:
                 name = '__main__'
             main_mod = FakeModule()
             prog_ns = main_mod.__dict__
             prog_ns['__name__'] = name
             # The shell MUST hold a reference to main_mod so after %run exits,
             # the python deletion mechanism doesn't zero it out (leaving
             # dangling references)
             self.shell._user_main_modules.append(main_mod)
 
         # Since '%run foo' emulates 'python foo.py' at the cmd line, we must
         # set the __file__ global in the script's namespace
         prog_ns['__file__'] = filename
 
         # pickle fix.  See iplib for an explanation.  But we need to make sure
         # that, if we overwrite __main__, we replace it at the end
         if prog_ns['__name__'] == '__main__':
             restore_main = sys.modules['__main__']
         else:
             restore_main = False
 
         sys.modules[prog_ns['__name__']] = main_mod
         
         stats = None
         try:
             self.shell.savehist()
 
             if opts.has_key('p'):
                 stats = self.magic_prun('',0,opts,arg_lst,prog_ns)
             else:
                 if opts.has_key('d'):
                     deb = Debugger.Pdb(self.shell.rc.colors)
                     # reset Breakpoint state, which is moronically kept
                     # in a class
                     bdb.Breakpoint.next = 1
                     bdb.Breakpoint.bplist = {}
                     bdb.Breakpoint.bpbynumber = [None]
                     # Set an initial breakpoint to stop execution
                     maxtries = 10
                     bp = int(opts.get('b',[1])[0])
                     checkline = deb.checkline(filename,bp)
                     if not checkline:
                         for bp in range(bp+1,bp+maxtries+1):
                             if deb.checkline(filename,bp):
                                 break
                         else:
                             msg = ("\nI failed to find a valid line to set "
                                    "a breakpoint\n"
                                    "after trying up to line: %s.\n"
                                    "Please set a valid breakpoint manually "
                                    "with the -b option." % bp)
                             error(msg)
                             return
                     # if we find a good linenumber, set the breakpoint
                     deb.do_break('%s:%s' % (filename,bp))
                     # Start file run
                     print "NOTE: Enter 'c' at the",
                     print "%s prompt to start your script." % deb.prompt
                     try:
                         deb.run('execfile("%s")' % filename,prog_ns)
                         
                     except:
                         etype, value, tb = sys.exc_info()
                         # Skip three frames in the traceback: the %run one,
                         # one inside bdb.py, and the command-line typed by the
                         # user (run by exec in pdb itself).
                         self.shell.InteractiveTB(etype,value,tb,tb_offset=3)
                 else:
                     if runner is None:
                         runner = self.shell.safe_execfile
                     if opts.has_key('t'):
                         # timed execution
                         try:
                             nruns = int(opts['N'][0])
                             if nruns < 1:
                                 error('Number of runs must be >=1')
                                 return
                         except (KeyError):
                             nruns = 1
                         if nruns == 1:
                             t0 = clock2()
                             runner(filename,prog_ns,prog_ns,
                                    exit_ignore=exit_ignore)
                             t1 = clock2()
                             t_usr = t1[0]-t0[0]
                             t_sys = t1[1]-t1[1]
                             print "\nIPython CPU timings (estimated):"
                             print "  User  : %10s s." % t_usr
                             print "  System: %10s s." % t_sys
                         else:
                             runs = range(nruns)
                             t0 = clock2()
                             for nr in runs:
                                 runner(filename,prog_ns,prog_ns,
                                        exit_ignore=exit_ignore)
                             t1 = clock2()
                             t_usr = t1[0]-t0[0]
                             t_sys = t1[1]-t1[1]
                             print "\nIPython CPU timings (estimated):"
                             print "Total runs performed:",nruns
                             print "  Times : %10s    %10s" % ('Total','Per run')
                             print "  User  : %10s s, %10s s." % (t_usr,t_usr/nruns)
                             print "  System: %10s s, %10s s." % (t_sys,t_sys/nruns)
                             
                     else:
                         # regular execution
                         runner(filename,prog_ns,prog_ns,exit_ignore=exit_ignore)
                 if opts.has_key('i'):
                     self.shell.user_ns['__name__'] = __name__save
                 else:
                     # update IPython interactive namespace
                     del prog_ns['__name__']
                     self.shell.user_ns.update(prog_ns)
         finally:
             sys.argv = save_argv
             if restore_main:
                 sys.modules['__main__'] = restore_main
             self.shell.reloadhist()
                 
         return stats
 
     def magic_runlog(self, parameter_s =''):
         """Run files as logs.
 
         Usage:\\
           %runlog file1 file2 ...
 
         Run the named files (treating them as log files) in sequence inside
         the interpreter, and return to the prompt.  This is much slower than
         %run because each line is executed in a try/except block, but it
         allows running files with syntax errors in them.
 
         Normally IPython will guess when a file is one of its own logfiles, so
         you can typically use %run even for logs. This shorthand allows you to
         force any file to be treated as a log file."""
 
         for f in parameter_s.split():
             self.shell.safe_execfile(f,self.shell.user_ns,
                                      self.shell.user_ns,islog=1)
 
     def magic_timeit(self, parameter_s =''):
         """Time execution of a Python statement or expression
 
         Usage:\\
           %timeit [-n<N> -r<R> [-t|-c]] statement
 
         Time execution of a Python statement or expression using the timeit
         module.
 
         Options:
         -n<N>: execute the given statement <N> times in a loop. If this value
         is not given, a fitting value is chosen. 
         
         -r<R>: repeat the loop iteration <R> times and take the best result.
         Default: 3
         
         -t: use time.time to measure the time, which is the default on Unix.
         This function measures wall time.
         
         -c: use time.clock to measure the time, which is the default on
         Windows and measures wall time. On Unix, resource.getrusage is used
         instead and returns the CPU user time.
 
         -p<P>: use a precision of <P> digits to display the timing result.
         Default: 3
 
         
         Examples:\\
           In [1]: %timeit pass
           10000000 loops, best of 3: 53.3 ns per loop
 
           In [2]: u = None
 
           In [3]: %timeit u is None
           10000000 loops, best of 3: 184 ns per loop
 
           In [4]: %timeit -r 4 u == None
           1000000 loops, best of 4: 242 ns per loop
 
           In [5]: import time
 
           In [6]: %timeit -n1 time.sleep(2)
           1 loops, best of 3: 2 s per loop
           
 
         The times reported by %timeit will be slightly higher than those
         reported by the timeit.py script when variables are accessed. This is
         due to the fact that %timeit executes the statement in the namespace
         of the shell, compared with timeit.py, which uses a single setup
         statement to import function or create variables. Generally, the bias
         does not matter as long as results from timeit.py are not mixed with
         those from %timeit."""
 
         import timeit
         import math
 
         units = ["s", "ms", "\xc2\xb5s", "ns"]
         scaling = [1, 1e3, 1e6, 1e9]
 
         opts, stmt = self.parse_options(parameter_s,'n:r:tcp:',
                                         posix=False)
         if stmt == "":
             return
         timefunc = timeit.default_timer
         number = int(getattr(opts, "n", 0))
         repeat = int(getattr(opts, "r", timeit.default_repeat))
         precision = int(getattr(opts, "p", 3))
         if hasattr(opts, "t"):
             timefunc = time.time
         if hasattr(opts, "c"):
             timefunc = clock
 
         timer = timeit.Timer(timer=timefunc)
         # this code has tight coupling to the inner workings of timeit.Timer,
         # but is there a better way to achieve that the code stmt has access
         # to the shell namespace?
 
         src = timeit.template % {'stmt': timeit.reindent(stmt, 8),
                                  'setup': "pass"}
         # Track compilation time so it can be reported if too long
         # Minimum time above which compilation time will be reported
         tc_min = 0.1
         
         t0 = clock()
         code = compile(src, "<magic-timeit>", "exec")
         tc = clock()-t0
         
         ns = {}
         exec code in self.shell.user_ns, ns
         timer.inner = ns["inner"]
         
         if number == 0:
             # determine number so that 0.2 <= total time < 2.0
             number = 1
             for i in range(1, 10):
                 number *= 10
                 if timer.timeit(number) >= 0.2:
                     break
         
         best = min(timer.repeat(repeat, number)) / number
 
         if best > 0.0:
             order = min(-int(math.floor(math.log10(best)) // 3), 3)
         else:
             order = 3
         print "%d loops, best of %d: %.*g %s per loop" % (number, repeat,
                                                           precision,
                                                           best * scaling[order],
                                                           units[order])
         if tc > tc_min:
             print "Compiler time: %.2f s" % tc
         
     def magic_time(self,parameter_s = ''):
         """Time execution of a Python statement or expression.
 
         The CPU and wall clock times are printed, and the value of the
         expression (if any) is returned.  Note that under Win32, system time
         is always reported as 0, since it can not be measured.
 
         This function provides very basic timing functionality.  In Python
         2.3, the timeit module offers more control and sophistication, so this
         could be rewritten to use it (patches welcome).
         
         Some examples:
 
           In [1]: time 2**128
           CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
           Wall time: 0.00
           Out[1]: 340282366920938463463374607431768211456L
 
           In [2]: n = 1000000
 
           In [3]: time sum(range(n))
           CPU times: user 1.20 s, sys: 0.05 s, total: 1.25 s
           Wall time: 1.37
           Out[3]: 499999500000L
 
           In [4]: time print 'hello world'
           hello world
           CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
           Wall time: 0.00
 
           Note that the time needed by Python to compile the given expression
           will be reported if it is more than 0.1s.  In this example, the
           actual exponentiation is done by Python at compilation time, so while
           the expression can take a noticeable amount of time to compute, that
           time is purely due to the compilation:
 
           In [5]: time 3**9999;
           CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
           Wall time: 0.00 s
 
           In [6]: time 3**999999;
           CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
           Wall time: 0.00 s
           Compiler : 0.78 s
           """
         
         # fail immediately if the given expression can't be compiled
         
         expr = self.shell.prefilter(parameter_s,False)
 
         # Minimum time above which compilation time will be reported
         tc_min = 0.1
         
         try:
             mode = 'eval'
             t0 = clock()
             code = compile(expr,'<timed eval>',mode)
             tc = clock()-t0
         except SyntaxError:
             mode = 'exec'
             t0 = clock()
             code = compile(expr,'<timed exec>',mode)
             tc = clock()-t0
         # skew measurement as little as possible
         glob = self.shell.user_ns
         clk = clock2
         wtime = time.time
         # time execution
         wall_st = wtime()
         if mode=='eval':
             st = clk()
             out = eval(code,glob)
             end = clk()
         else:
             st = clk()
             exec code in glob
             end = clk()
             out = None
         wall_end = wtime()
         # Compute actual times and report
         wall_time = wall_end-wall_st
         cpu_user = end[0]-st[0]
         cpu_sys = end[1]-st[1]
         cpu_tot = cpu_user+cpu_sys
         print "CPU times: user %.2f s, sys: %.2f s, total: %.2f s" % \
               (cpu_user,cpu_sys,cpu_tot)
         print "Wall time: %.2f s" % wall_time
         if tc > tc_min:
             print "Compiler : %.2f s" % tc
         return out
 
     def magic_macro(self,parameter_s = ''):
         """Define a set of input lines as a macro for future re-execution.
 
         Usage:\\
           %macro [options] name n1-n2 n3-n4 ... n5 .. n6 ...
 
         Options:
         
           -r: use 'raw' input.  By default, the 'processed' history is used,
           so that magics are loaded in their transformed version to valid
           Python.  If this option is given, the raw input as typed as the
           command line is used instead.
 
         This will define a global variable called `name` which is a string
         made of joining the slices and lines you specify (n1,n2,... numbers
         above) from your input history into a single string. This variable
         acts like an automatic function which re-executes those lines as if
         you had typed them. You just type 'name' at the prompt and the code
         executes.
 
         The notation for indicating number ranges is: n1-n2 means 'use line
         numbers n1,...n2' (the endpoint is included).  That is, '5-7' means
         using the lines numbered 5,6 and 7.
 
         Note: as a 'hidden' feature, you can also use traditional python slice
         notation, where N:M means numbers N through M-1.
 
         For example, if your history contains (%hist prints it):
         
           44: x=1\\
           45: y=3\\
           46: z=x+y\\
           47: print x\\
           48: a=5\\
           49: print 'x',x,'y',y\\
 
         you can create a macro with lines 44 through 47 (included) and line 49
         called my_macro with:
 
           In [51]: %macro my_macro 44-47 49
 
         Now, typing `my_macro` (without quotes) will re-execute all this code
         in one pass.
 
         You don't need to give the line-numbers in order, and any given line
         number can appear multiple times. You can assemble macros with any
         lines from your input history in any order.
 
         The macro is a simple object which holds its value in an attribute,
         but IPython's display system checks for macros and executes them as
         code instead of printing them when you type their name.
 
         You can view a macro's contents by explicitly printing it with:
         
           'print macro_name'.
 
         For one-off cases which DON'T contain magic function calls in them you
         can obtain similar results by explicitly executing slices from your
         input history with:
 
           In [60]: exec In[44:48]+In[49]"""
 
         opts,args = self.parse_options(parameter_s,'r',mode='list')
         if not args:
             macs = [k for k,v in self.shell.user_ns.items() if isinstance(v, Macro)]
             macs.sort()
             return macs
         if len(args) == 1:
             raise UsageError(
                 "%macro insufficient args; usage '%macro name n1-n2 n3-4...")
         name,ranges = args[0], args[1:]
         
         #print 'rng',ranges  # dbg
         lines = self.extract_input_slices(ranges,opts.has_key('r'))
         macro = Macro(lines)
         self.shell.user_ns.update({name:macro})
         print 'Macro `%s` created. To execute, type its name (without quotes).' % name
         print 'Macro contents:'
         print macro,
 
     def magic_save(self,parameter_s = ''):
         """Save a set of lines to a given filename.
 
         Usage:\\
           %save [options] filename n1-n2 n3-n4 ... n5 .. n6 ...
 
         Options:
         
           -r: use 'raw' input.  By default, the 'processed' history is used,
           so that magics are loaded in their transformed version to valid
           Python.  If this option is given, the raw input as typed as the
           command line is used instead.
 
         This function uses the same syntax as %macro for line extraction, but
         instead of creating a macro it saves the resulting string to the
         filename you specify.
 
         It adds a '.py' extension to the file if you don't do so yourself, and
         it asks for confirmation before overwriting existing files."""
 
         opts,args = self.parse_options(parameter_s,'r',mode='list')
         fname,ranges = args[0], args[1:]
         if not fname.endswith('.py'):
             fname += '.py'
         if os.path.isfile(fname):
             ans = raw_input('File `%s` exists. Overwrite (y/[N])? ' % fname)
             if ans.lower() not in ['y','yes']:
                 print 'Operation cancelled.'
                 return
         cmds = ''.join(self.extract_input_slices(ranges,opts.has_key('r')))
         f = file(fname,'w')
         f.write(cmds)
         f.close()
         print 'The following commands were written to file `%s`:' % fname
         print cmds
 
     def _edit_macro(self,mname,macro):
         """open an editor with the macro data in a file"""
         filename = self.shell.mktempfile(macro.value)
         self.shell.hooks.editor(filename)
 
         # and make a new macro object, to replace the old one
         mfile = open(filename)
         mvalue = mfile.read()
         mfile.close()
         self.shell.user_ns[mname] = Macro(mvalue)
 
     def magic_ed(self,parameter_s=''):
         """Alias to %edit."""
         return self.magic_edit(parameter_s)
 
     def magic_edit(self,parameter_s='',last_call=['','']):
         """Bring up an editor and execute the resulting code.
 
         Usage:
           %edit [options] [args]
 
         %edit runs IPython's editor hook.  The default version of this hook is
         set to call the __IPYTHON__.rc.editor command.  This is read from your
         environment variable $EDITOR.  If this isn't found, it will default to
         vi under Linux/Unix and to notepad under Windows.  See the end of this
         docstring for how to change the editor hook.
 
         You can also set the value of this editor via the command line option
         '-editor' or in your ipythonrc file. This is useful if you wish to use
         specifically for IPython an editor different from your typical default
         (and for Windows users who typically don't set environment variables).
 
         This command allows you to conveniently edit multi-line code right in
         your IPython session.
         
         If called without arguments, %edit opens up an empty editor with a
         temporary file and will execute the contents of this file when you
         close it (don't forget to save it!).
 
 
         Options:
 
         -n <number>: open the editor at a specified line number.  By default,
         the IPython editor hook uses the unix syntax 'editor +N filename', but
         you can configure this by providing your own modified hook if your
         favorite editor supports line-number specifications with a different
         syntax.
         
         -p: this will call the editor with the same data as the previous time
         it was used, regardless of how long ago (in your current session) it
         was.
 
         -r: use 'raw' input.  This option only applies to input taken from the
         user's history.  By default, the 'processed' history is used, so that
         magics are loaded in their transformed version to valid Python.  If
         this option is given, the raw input as typed as the command line is
         used instead.  When you exit the editor, it will be executed by
         IPython's own processor.
         
         -x: do not execute the edited code immediately upon exit. This is
         mainly useful if you are editing programs which need to be called with
         command line arguments, which you can then do using %run.
 
 
         Arguments:
 
         If arguments are given, the following possibilites exist:
 
         - The arguments are numbers or pairs of colon-separated numbers (like
         1 4:8 9). These are interpreted as lines of previous input to be
         loaded into the editor. The syntax is the same of the %macro command.
 
         - If the argument doesn't start with a number, it is evaluated as a
         variable and its contents loaded into the editor. You can thus edit
         any string which contains python code (including the result of
         previous edits).
 
         - If the argument is the name of an object (other than a string),
         IPython will try to locate the file where it was defined and open the
         editor at the point where it is defined. You can use `%edit function`
         to load an editor exactly at the point where 'function' is defined,
         edit it and have the file be executed automatically.
 
         If the object is a macro (see %macro for details), this opens up your
         specified editor with a temporary file containing the macro's data.
         Upon exit, the macro is reloaded with the contents of the file.
 
         Note: opening at an exact line is only supported under Unix, and some
         editors (like kedit and gedit up to Gnome 2.8) do not understand the
         '+NUMBER' parameter necessary for this feature. Good editors like
         (X)Emacs, vi, jed, pico and joe all do.
 
         - If the argument is not found as a variable, IPython will look for a
         file with that name (adding .py if necessary) and load it into the
         editor. It will execute its contents with execfile() when you exit,
         loading any code in the file into your interactive namespace.
 
         After executing your code, %edit will return as output the code you
         typed in the editor (except when it was an existing file). This way
         you can reload the code in further invocations of %edit as a variable,
         via _<NUMBER> or Out[<NUMBER>], where <NUMBER> is the prompt number of
         the output.
 
         Note that %edit is also available through the alias %ed.
 
         This is an example of creating a simple function inside the editor and
         then modifying it. First, start up the editor:
 
         In [1]: ed\\
         Editing... done. Executing edited code...\\
         Out[1]: 'def foo():\\n    print "foo() was defined in an editing session"\\n'
 
         We can then call the function foo():
         
         In [2]: foo()\\
         foo() was defined in an editing session
 
         Now we edit foo.  IPython automatically loads the editor with the
         (temporary) file where foo() was previously defined:
         
         In [3]: ed foo\\
         Editing... done. Executing edited code...
 
         And if we call foo() again we get the modified version:
         
         In [4]: foo()\\
         foo() has now been changed!
 
         Here is an example of how to edit a code snippet successive
         times. First we call the editor:
 
         In [8]: ed\\
         Editing... done. Executing edited code...\\
         hello\\
         Out[8]: "print 'hello'\\n"
 
         Now we call it again with the previous output (stored in _):
 
         In [9]: ed _\\
         Editing... done. Executing edited code...\\
         hello world\\
         Out[9]: "print 'hello world'\\n"
 
         Now we call it with the output #8 (stored in _8, also as Out[8]):
 
         In [10]: ed _8\\
         Editing... done. Executing edited code...\\
         hello again\\
         Out[10]: "print 'hello again'\\n"
 
 
         Changing the default editor hook:
 
         If you wish to write your own editor hook, you can put it in a
         configuration file which you load at startup time.  The default hook
         is defined in the IPython.hooks module, and you can use that as a
         starting example for further modifications.  That file also has
         general instructions on how to set a new hook for use once you've
         defined it."""
         
         # FIXME: This function has become a convoluted mess.  It needs a
         # ground-up rewrite with clean, simple logic.
 
         def make_filename(arg):
             "Make a filename from the given args"
             try:
                 filename = get_py_filename(arg)
             except IOError:
                 if args.endswith('.py'):
                     filename = arg
                 else:
                     filename = None
             return filename
 
         # custom exceptions
         class DataIsObject(Exception): pass
 
         opts,args = self.parse_options(parameter_s,'prxn:')
         # Set a few locals from the options for convenience:
         opts_p = opts.has_key('p')
         opts_r = opts.has_key('r')
         
         # Default line number value
         lineno = opts.get('n',None)
 
         if opts_p:
             args = '_%s' % last_call[0]
             if not self.shell.user_ns.has_key(args):
                 args = last_call[1]
             
         # use last_call to remember the state of the previous call, but don't
         # let it be clobbered by successive '-p' calls.
         try:
             last_call[0] = self.shell.outputcache.prompt_count
             if not opts_p:
                 last_call[1] = parameter_s
         except:
             pass
 
         # by default this is done with temp files, except when the given
         # arg is a filename
         use_temp = 1
 
         if re.match(r'\d',args):
             # Mode where user specifies ranges of lines, like in %macro.
             # This means that you can't edit files whose names begin with
             # numbers this way. Tough.
             ranges = args.split()
             data = ''.join(self.extract_input_slices(ranges,opts_r))
         elif args.endswith('.py'):
             filename = make_filename(args)
             data = ''
             use_temp = 0
         elif args:
             try:
                 # Load the parameter given as a variable. If not a string,
                 # process it as an object instead (below)
 
                 #print '*** args',args,'type',type(args)  # dbg
                 data = eval(args,self.shell.user_ns)
                 if not type(data) in StringTypes:
                     raise DataIsObject
 
             except (NameError,SyntaxError):
                 # given argument is not a variable, try as a filename
                 filename = make_filename(args)
                 if filename is None:
                     warn("Argument given (%s) can't be found as a variable "
                          "or as a filename." % args)
                     return
 
                 data = ''
                 use_temp = 0
             except DataIsObject:
 
                 # macros have a special edit function
                 if isinstance(data,Macro):
                     self._edit_macro(args,data)
                     return
                                 
                 # For objects, try to edit the file where they are defined
                 try:
                     filename = inspect.getabsfile(data)
                     if 'fakemodule' in filename.lower() and inspect.isclass(data):                     
                         # class created by %edit? Try to find source
                         # by looking for method definitions instead, the
                         # __module__ in those classes is FakeModule.
                         attrs = [getattr(data, aname) for aname in dir(data)]
                         for attr in attrs:
                             if not inspect.ismethod(attr):
                                 continue
                             filename = inspect.getabsfile(attr)
                             if filename and 'fakemodule' not in filename.lower():
                                 # change the attribute to be the edit target instead
                                 data = attr 
                                 break
                     
                     datafile = 1
                 except TypeError:
                     filename = make_filename(args)
                     datafile = 1
                     warn('Could not find file where `%s` is defined.\n'
                          'Opening a file named `%s`' % (args,filename))
                 # Now, make sure we can actually read the source (if it was in
                 # a temp file it's gone by now).
                 if datafile:
                     try:
                         if lineno is None:
                             lineno = inspect.getsourcelines(data)[1]
                     except IOError:
                         filename = make_filename(args)
                         if filename is None:
                             warn('The file `%s` where `%s` was defined cannot '
                                  'be read.' % (filename,data))
                             return
                 use_temp = 0
         else:
             data = ''
 
         if use_temp:
             filename = self.shell.mktempfile(data)
             print 'IPython will make a temporary file named:',filename
 
         # do actual editing here
         print 'Editing...',
         sys.stdout.flush()
         self.shell.hooks.editor(filename,lineno)
         if opts.has_key('x'):  # -x prevents actual execution
             print
         else:
             print 'done. Executing edited code...'
             if opts_r:
                 self.shell.runlines(file_read(filename))
             else:
                 self.shell.safe_execfile(filename,self.shell.user_ns,
                                          self.shell.user_ns)
         if use_temp:
             try:
                 return open(filename).read()
             except IOError,msg:
                 if msg.filename == filename:
                     warn('File not found. Did you forget to save?')
                     return
                 else:
                     self.shell.showtraceback()
 
     def magic_xmode(self,parameter_s = ''):
         """Switch modes for the exception handlers.
 
         Valid modes: Plain, Context and Verbose.
 
         If called without arguments, acts as a toggle."""
 
         def xmode_switch_err(name):
             warn('Error changing %s exception modes.\n%s' %
                  (name,sys.exc_info()[1]))
 
         shell = self.shell
         new_mode = parameter_s.strip().capitalize()
         try:
             shell.InteractiveTB.set_mode(mode=new_mode)
             print 'Exception reporting mode:',shell.InteractiveTB.mode
         except:
             xmode_switch_err('user')
 
         # threaded shells use a special handler in sys.excepthook
         if shell.isthreaded:
             try:
                 shell.sys_excepthook.set_mode(mode=new_mode)
             except:
                 xmode_switch_err('threaded')
             
     def magic_colors(self,parameter_s = ''):
         """Switch color scheme for prompts, info system and exception handlers.
 
         Currently implemented schemes: NoColor, Linux, LightBG.
 
         Color scheme names are not case-sensitive."""
 
         def color_switch_err(name):
             warn('Error changing %s color schemes.\n%s' %
                  (name,sys.exc_info()[1]))
             
         
         new_scheme = parameter_s.strip()
         if not new_scheme:
             raise UsageError(
                 "%colors: you must specify a color scheme. See '%colors?'")
             return
         # local shortcut
         shell = self.shell
 
         import IPython.rlineimpl as readline
 
         if not readline.have_readline and sys.platform == "win32":
             msg = """\
 Proper color support under MS Windows requires the pyreadline library.
 You can find it at:
 http://ipython.scipy.org/moin/PyReadline/Intro
 Gary's readline needs the ctypes module, from:
 http://starship.python.net/crew/theller/ctypes
 (Note that ctypes is already part of Python versions 2.5 and newer).
 
 Defaulting color scheme to 'NoColor'"""
             new_scheme = 'NoColor'
             warn(msg)
         
         # readline option is 0
         if not shell.has_readline:
             new_scheme = 'NoColor'
             
         # Set prompt colors
         try:
             shell.outputcache.set_colors(new_scheme)
         except:
             color_switch_err('prompt')
         else:
             shell.rc.colors = \
                        shell.outputcache.color_table.active_scheme_name
         # Set exception colors
         try:
             shell.InteractiveTB.set_colors(scheme = new_scheme)
             shell.SyntaxTB.set_colors(scheme = new_scheme)
         except:
             color_switch_err('exception')
 
         # threaded shells use a verbose traceback in sys.excepthook
         if shell.isthreaded:
             try:
                 shell.sys_excepthook.set_colors(scheme=new_scheme)
             except:
                 color_switch_err('system exception handler')
         
         # Set info (for 'object?') colors
         if shell.rc.color_info:
             try:
                 shell.inspector.set_active_scheme(new_scheme)
             except:
                 color_switch_err('object inspector')
         else:
             shell.inspector.set_active_scheme('NoColor')
                 
     def magic_color_info(self,parameter_s = ''):
         """Toggle color_info.
 
         The color_info configuration parameter controls whether colors are
         used for displaying object details (by things like %psource, %pfile or
         the '?' system). This function toggles this value with each call.
 
         Note that unless you have a fairly recent pager (less works better
         than more) in your system, using colored object information displays
         will not work properly. Test it and see."""
         
         self.shell.rc.color_info = 1 - self.shell.rc.color_info
         self.magic_colors(self.shell.rc.colors)
         print 'Object introspection functions have now coloring:',
         print ['OFF','ON'][self.shell.rc.color_info]
 
     def magic_Pprint(self, parameter_s=''):
         """Toggle pretty printing on/off."""
         
         self.shell.rc.pprint = 1 - self.shell.rc.pprint
         print 'Pretty printing has been turned', \
               ['OFF','ON'][self.shell.rc.pprint]
         
     def magic_exit(self, parameter_s=''):
         """Exit IPython, confirming if configured to do so.
 
         You can configure whether IPython asks for confirmation upon exit by
         setting the confirm_exit flag in the ipythonrc file."""
 
         self.shell.exit()
 
     def magic_quit(self, parameter_s=''):
         """Exit IPython, confirming if configured to do so (like %exit)"""
 
         self.shell.exit()
         
     def magic_Exit(self, parameter_s=''):
         """Exit IPython without confirmation."""
 
         self.shell.exit_now = True
 
     #......................................................................
     # Functions to implement unix shell-type things
     
     def magic_alias(self, parameter_s = ''):
         """Define an alias for a system command.
 
         '%alias alias_name cmd' defines 'alias_name' as an alias for 'cmd'
 
         Then, typing 'alias_name params' will execute the system command 'cmd
         params' (from your underlying operating system).
 
         Aliases have lower precedence than magic functions and Python normal
         variables, so if 'foo' is both a Python variable and an alias, the
         alias can not be executed until 'del foo' removes the Python variable.
 
         You can use the %l specifier in an alias definition to represent the
         whole line when the alias is called.  For example:
 
           In [2]: alias all echo "Input in brackets: <%l>"\\
           In [3]: all hello world\\
           Input in brackets: <hello world>
 
         You can also define aliases with parameters using %s specifiers (one
         per parameter):
         
           In [1]: alias parts echo first %s second %s\\
           In [2]: %parts A B\\
           first A second B\\
           In [3]: %parts A\\
           Incorrect number of arguments: 2 expected.\\
           parts is an alias to: 'echo first %s second %s'
 
         Note that %l and %s are mutually exclusive.  You can only use one or
         the other in your aliases.
 
         Aliases expand Python variables just like system calls using ! or !! 
         do: all expressions prefixed with '$' get expanded.  For details of
         the semantic rules, see PEP-215:
         http://www.python.org/peps/pep-0215.html.  This is the library used by
         IPython for variable expansion.  If you want to access a true shell
         variable, an extra $ is necessary to prevent its expansion by IPython:
 
         In [6]: alias show echo\\
         In [7]: PATH='A Python string'\\
         In [8]: show $PATH\\
         A Python string\\
         In [9]: show $$PATH\\
         /usr/local/lf9560/bin:/usr/local/intel/compiler70/ia32/bin:...
 
         You can use the alias facility to acess all of $PATH.  See the %rehash
         and %rehashx functions, which automatically create aliases for the
         contents of your $PATH.
 
         If called with no parameters, %alias prints the current alias table."""
 
         par = parameter_s.strip()
         if not par:
             stored = self.db.get('stored_aliases', {} )
             atab = self.shell.alias_table
             aliases = atab.keys()
             aliases.sort()
             res = []
             showlast = []
             for alias in aliases:
                 special = False
                 try:
                     tgt = atab[alias][1]
                 except (TypeError, AttributeError):
                     # unsubscriptable? probably a callable
                     tgt = atab[alias]
                     special = True
                 # 'interesting' aliases
                 if (alias in stored or
                     special or 
                     alias.lower() != os.path.splitext(tgt)[0].lower() or
                     ' ' in tgt):
                     showlast.append((alias, tgt))
                 else:
                     res.append((alias, tgt ))                
             
             # show most interesting aliases last
             res.extend(showlast)
             print "Total number of aliases:",len(aliases)
             return res
         try:
             alias,cmd = par.split(None,1)
         except:
             print OInspect.getdoc(self.magic_alias)
         else:
             nargs = cmd.count('%s')
             if nargs>0 and cmd.find('%l')>=0:
                 error('The %s and %l specifiers are mutually exclusive '
                       'in alias definitions.')
             else:  # all looks OK
                 self.shell.alias_table[alias] = (nargs,cmd)
                 self.shell.alias_table_validate(verbose=0)
     # end magic_alias
 
     def magic_unalias(self, parameter_s = ''):
         """Remove an alias"""
 
         aname = parameter_s.strip()
         if aname in self.shell.alias_table:
             del self.shell.alias_table[aname]
         stored = self.db.get('stored_aliases', {} )
         if aname in stored:
             print "Removing %stored alias",aname
             del stored[aname]
             self.db['stored_aliases'] = stored
             
 
     def magic_rehashx(self, parameter_s = ''):
         """Update the alias table with all executable files in $PATH.
 
         This version explicitly checks that every entry in $PATH is a file
         with execute access (os.X_OK), so it is much slower than %rehash.
 
         Under Windows, it checks executability as a match agains a
         '|'-separated string of extensions, stored in the IPython config
         variable win_exec_ext.  This defaults to 'exe|com|bat'.
         
         This function also resets the root module cache of module completer,
         used on slow filesystems.
         """
         
         
         ip = self.api
 
         # for the benefit of module completer in ipy_completers.py
         del ip.db['rootmodules']
         
         path = [os.path.abspath(os.path.expanduser(p)) for p in 
             os.environ.get('PATH','').split(os.pathsep)]
         path = filter(os.path.isdir,path)
         
         alias_table = self.shell.alias_table
         syscmdlist = []
         if os.name == 'posix':
             isexec = lambda fname:os.path.isfile(fname) and \
                      os.access(fname,os.X_OK)
         else:
 
             try:
                 winext = os.environ['pathext'].replace(';','|').replace('.','')
             except KeyError:
                 winext = 'exe|com|bat|py'
             if 'py' not in winext:
                 winext += '|py'
             execre = re.compile(r'(.*)\.(%s)$' % winext,re.IGNORECASE)
             isexec = lambda fname:os.path.isfile(fname) and execre.match(fname)
         savedir = os.getcwd()
         try:
             # write the whole loop for posix/Windows so we don't have an if in
             # the innermost part
             if os.name == 'posix':
                 for pdir in path:
                     os.chdir(pdir)
                     for ff in os.listdir(pdir):
                         if isexec(ff) and ff not in self.shell.no_alias:
                             # each entry in the alias table must be (N,name),
                             # where N is the number of positional arguments of the
                             # alias.
                             alias_table[ff] = (0,ff)
                             syscmdlist.append(ff)
             else:
                 for pdir in path:
                     os.chdir(pdir)
                     for ff in os.listdir(pdir):
                         base, ext = os.path.splitext(ff)
                         if isexec(ff) and base.lower() not in self.shell.no_alias:
                             if ext.lower() == '.exe':
                                 ff = base
                             alias_table[base.lower()] = (0,ff)
                             syscmdlist.append(ff)
             # Make sure the alias table doesn't contain keywords or builtins
             self.shell.alias_table_validate()
             # Call again init_auto_alias() so we get 'rm -i' and other
             # modified aliases since %rehashx will probably clobber them
             
             # no, we don't want them. if %rehashx clobbers them, good,
             # we'll probably get better versions
             # self.shell.init_auto_alias()
             db = ip.db
             db['syscmdlist'] = syscmdlist
         finally:
             os.chdir(savedir)
         
     def magic_pwd(self, parameter_s = ''):
         """Return the current working directory path."""
         return os.getcwd()
 
     def magic_cd(self, parameter_s=''):
         """Change the current working directory.
 
         This command automatically maintains an internal list of directories
         you visit during your IPython session, in the variable _dh. The
         command %dhist shows this history nicely formatted. You can also
         do 'cd -<tab>' to see directory history conveniently.
 
         Usage:
 
           cd 'dir': changes to directory 'dir'.
 
           cd -: changes to the last visited directory.
 
           cd -<n>: changes to the n-th directory in the directory history.
 
           cd -b <bookmark_name>: jump to a bookmark set by %bookmark
              (note: cd <bookmark_name> is enough if there is no
               directory <bookmark_name>, but a bookmark with the name exists.)
               'cd -b <tab>' allows you to tab-complete bookmark names. 
 
         Options:
 
         -q: quiet.  Do not print the working directory after the cd command is
         executed.  By default IPython's cd command does print this directory,
         since the default prompts do not display path information.
         
         Note that !cd doesn't work for this purpose because the shell where
         !command runs is immediately discarded after executing 'command'."""
 
         parameter_s = parameter_s.strip()
         #bkms = self.shell.persist.get("bookmarks",{})
 
         oldcwd = os.getcwd()
         numcd = re.match(r'(-)(\d+)$',parameter_s)
         # jump in directory history by number
         if numcd:
             nn = int(numcd.group(2))
             try:
                 ps = self.shell.user_ns['_dh'][nn]
             except IndexError:
                 print 'The requested directory does not exist in history.'
                 return
             else:
                 opts = {}
         else:
             #turn all non-space-escaping backslashes to slashes, 
             # for c:\windows\directory\names\
             parameter_s = re.sub(r'\\(?! )','/', parameter_s)            
             opts,ps = self.parse_options(parameter_s,'qb',mode='string')
         # jump to previous
         if ps == '-':
             try:
                 ps = self.shell.user_ns['_dh'][-2]
             except IndexError:
                 raise UsageError('%cd -: No previous directory to change to.')
         # jump to bookmark if needed
         else:
             if not os.path.isdir(ps) or opts.has_key('b'):
                 bkms = self.db.get('bookmarks', {})
             
                 if bkms.has_key(ps):
                     target = bkms[ps]
                     print '(bookmark:%s) -> %s' % (ps,target)
                     ps = target
                 else:
                     if opts.has_key('b'):
                         raise UsageError("Bookmark '%s' not found.  "
                               "Use '%%bookmark -l' to see your bookmarks." % ps)
             
         # at this point ps should point to the target dir
         if ps:
             try:                
                 os.chdir(os.path.expanduser(ps))
                 if self.shell.rc.term_title:
                     #print 'set term title:',self.shell.rc.term_title  # dbg
-                    ttitle = 'IPy ' + abbrev_cwd()
-                    platutils.set_term_title(ttitle)
+                    platutils.set_term_title('IPy ' + abbrev_cwd())
             except OSError:
                 print sys.exc_info()[1]
             else:
                 cwd = os.getcwd()
                 dhist = self.shell.user_ns['_dh']
                 if oldcwd != cwd:
                     dhist.append(cwd)
                     self.db['dhist'] = compress_dhist(dhist)[-100:]
                 
         else:
             os.chdir(self.shell.home_dir)
             if self.shell.rc.term_title:
                 platutils.set_term_title("IPy ~")
             cwd = os.getcwd()
             dhist = self.shell.user_ns['_dh']
             
             if oldcwd != cwd:
                 dhist.append(cwd)
                 self.db['dhist'] = compress_dhist(dhist)[-100:]
         if not 'q' in opts and self.shell.user_ns['_dh']:
             print self.shell.user_ns['_dh'][-1]
 
 
     def magic_env(self, parameter_s=''):
         """List environment variables."""
         
         return os.environ.data
 
     def magic_pushd(self, parameter_s=''):
         """Place the current dir on stack and change directory.
         
         Usage:\\
           %pushd ['dirname']
         """
         
         dir_s = self.shell.dir_stack
         tgt = os.path.expanduser(parameter_s)
         cwd = os.getcwd().replace(self.home_dir,'~')
         if tgt:
             self.magic_cd(parameter_s)
         dir_s.insert(0,cwd)
         return self.magic_dirs()
 
     def magic_popd(self, parameter_s=''):
         """Change to directory popped off the top of the stack.
         """
         if not self.shell.dir_stack:
             raise UsageError("%popd on empty stack")
         top = self.shell.dir_stack.pop(0)
         self.magic_cd(top)
         print "popd ->",top
 
     def magic_dirs(self, parameter_s=''):
         """Return the current directory stack."""
 
         return self.shell.dir_stack
 
     def magic_dhist(self, parameter_s=''):
         """Print your history of visited directories.
 
         %dhist       -> print full history\\
         %dhist n     -> print last n entries only\\
         %dhist n1 n2 -> print entries between n1 and n2 (n1 not included)\\
 
         This history is automatically maintained by the %cd command, and
         always available as the global list variable _dh. You can use %cd -<n>
         to go to directory number <n>.
         
         Note that most of time, you should view directory history by entering
         cd -<TAB>.
         
         """
 
         dh = self.shell.user_ns['_dh']
         if parameter_s:
             try:
                 args = map(int,parameter_s.split())
             except:
                 self.arg_err(Magic.magic_dhist)
                 return
             if len(args) == 1:
                 ini,fin = max(len(dh)-(args[0]),0),len(dh)
             elif len(args) == 2:
                 ini,fin = args
             else:
                 self.arg_err(Magic.magic_dhist)
                 return
         else:
             ini,fin = 0,len(dh)
         nlprint(dh,
                 header = 'Directory history (kept in _dh)',
                 start=ini,stop=fin)
 
 
     def magic_sc(self, parameter_s=''):
         """Shell capture - execute a shell command and capture its output.
 
         DEPRECATED. Suboptimal, retained for backwards compatibility.
         
         You should use the form 'var = !command' instead. Example:
          
          "%sc -l myfiles = ls ~" should now be written as
             
          "myfiles = !ls ~"
          
         myfiles.s, myfiles.l and myfiles.n still apply as documented
         below.
 
         --
         %sc [options] varname=command
 
         IPython will run the given command using commands.getoutput(), and
         will then update the user's interactive namespace with a variable
         called varname, containing the value of the call.  Your command can
         contain shell wildcards, pipes, etc.
 
         The '=' sign in the syntax is mandatory, and the variable name you
         supply must follow Python's standard conventions for valid names.
         
         (A special format without variable name exists for internal use)
 
         Options:
 
           -l: list output.  Split the output on newlines into a list before
           assigning it to the given variable.  By default the output is stored
           as a single string.
 
           -v: verbose.  Print the contents of the variable.
 
         In most cases you should not need to split as a list, because the
         returned value is a special type of string which can automatically
         provide its contents either as a list (split on newlines) or as a
         space-separated string.  These are convenient, respectively, either
         for sequential processing or to be passed to a shell command.
 
         For example:
 
             # Capture into variable a
             In [9]: sc a=ls *py
 
             # a is a string with embedded newlines
             In [10]: a
             Out[10]: 'setup.py\nwin32_manual_post_install.py'
 
             # which can be seen as a list:
             In [11]: a.l
             Out[11]: ['setup.py', 'win32_manual_post_install.py']
 
             # or as a whitespace-separated string:
             In [12]: a.s
             Out[12]: 'setup.py win32_manual_post_install.py'
 
             # a.s is useful to pass as a single command line:
             In [13]: !wc -l $a.s
               146 setup.py
               130 win32_manual_post_install.py
               276 total
 
             # while the list form is useful to loop over:
             In [14]: for f in a.l:
                ....:      !wc -l $f
                ....:
             146 setup.py
             130 win32_manual_post_install.py
 
         Similiarly, the lists returned by the -l option are also special, in
         the sense that you can equally invoke the .s attribute on them to
         automatically get a whitespace-separated string from their contents:
 
             In [1]: sc -l b=ls *py
 
             In [2]: b
             Out[2]: ['setup.py', 'win32_manual_post_install.py']
 
             In [3]: b.s
             Out[3]: 'setup.py win32_manual_post_install.py'
 
         In summary, both the lists and strings used for ouptut capture have
         the following special attributes:
 
             .l (or .list) : value as list.
             .n (or .nlstr): value as newline-separated string.
             .s (or .spstr): value as space-separated string.
         """
 
         opts,args = self.parse_options(parameter_s,'lv')
         # Try to get a variable name and command to run
         try:
             # the variable name must be obtained from the parse_options
             # output, which uses shlex.split to strip options out.
             var,_ = args.split('=',1)
             var = var.strip()
             # But the the command has to be extracted from the original input
             # parameter_s, not on what parse_options returns, to avoid the
             # quote stripping which shlex.split performs on it.
             _,cmd = parameter_s.split('=',1)
         except ValueError:
             var,cmd = '',''
         # If all looks ok, proceed
         out,err = self.shell.getoutputerror(cmd)
         if err:
             print >> Term.cerr,err
         if opts.has_key('l'):
             out = SList(out.split('\n'))
         else:
             out = LSString(out)
         if opts.has_key('v'):
             print '%s ==\n%s' % (var,pformat(out))
         if var:
             self.shell.user_ns.update({var:out})
         else:
             return out
 
     def magic_sx(self, parameter_s=''):
         """Shell execute - run a shell command and capture its output.
 
         %sx command
 
         IPython will run the given command using commands.getoutput(), and
         return the result formatted as a list (split on '\\n').  Since the
         output is _returned_, it will be stored in ipython's regular output
         cache Out[N] and in the '_N' automatic variables.
 
         Notes:
 
         1) If an input line begins with '!!', then %sx is automatically
         invoked.  That is, while:
           !ls
         causes ipython to simply issue system('ls'), typing
           !!ls
         is a shorthand equivalent to:
           %sx ls
         
         2) %sx differs from %sc in that %sx automatically splits into a list,
         like '%sc -l'.  The reason for this is to make it as easy as possible
         to process line-oriented shell output via further python commands.
         %sc is meant to provide much finer control, but requires more
         typing.
 
         3) Just like %sc -l, this is a list with special attributes:
 
           .l (or .list) : value as list.
           .n (or .nlstr): value as newline-separated string.
           .s (or .spstr): value as whitespace-separated string.
 
         This is very useful when trying to use such lists as arguments to
         system commands."""
 
         if parameter_s:
             out,err = self.shell.getoutputerror(parameter_s)
             if err:
                 print >> Term.cerr,err
             return SList(out.split('\n'))
 
     def magic_bg(self, parameter_s=''):
         """Run a job in the background, in a separate thread.
 
         For example,
 
           %bg myfunc(x,y,z=1)
 
         will execute 'myfunc(x,y,z=1)' in a background thread.  As soon as the
         execution starts, a message will be printed indicating the job
         number.  If your job number is 5, you can use
 
           myvar = jobs.result(5)  or  myvar = jobs[5].result
 
         to assign this result to variable 'myvar'.
 
         IPython has a job manager, accessible via the 'jobs' object.  You can
         type jobs? to get more information about it, and use jobs.<TAB> to see
         its attributes.  All attributes not starting with an underscore are
         meant for public use.
 
         In particular, look at the jobs.new() method, which is used to create
         new jobs.  This magic %bg function is just a convenience wrapper
         around jobs.new(), for expression-based jobs.  If you want to create a
         new job with an explicit function object and arguments, you must call
         jobs.new() directly.
 
         The jobs.new docstring also describes in detail several important
         caveats associated with a thread-based model for background job
         execution.  Type jobs.new? for details.
 
         You can check the status of all jobs with jobs.status().
 
         The jobs variable is set by IPython into the Python builtin namespace.
         If you ever declare a variable named 'jobs', you will shadow this
         name.  You can either delete your global jobs variable to regain
         access to the job manager, or make a new name and assign it manually
         to the manager (stored in IPython's namespace).  For example, to
         assign the job manager to the Jobs name, use:
 
           Jobs = __builtins__.jobs"""
         
         self.shell.jobs.new(parameter_s,self.shell.user_ns)
         
     def magic_r(self, parameter_s=''):
         """Repeat previous input.
 
         Note: Consider using the more powerfull %rep instead!
         
         If given an argument, repeats the previous command which starts with
         the same string, otherwise it just repeats the previous input.
 
         Shell escaped commands (with ! as first character) are not recognized
         by this system, only pure python code and magic commands.
         """
 
         start = parameter_s.strip()
         esc_magic = self.shell.ESC_MAGIC
         # Identify magic commands even if automagic is on (which means
         # the in-memory version is different from that typed by the user).
         if self.shell.rc.automagic:
             start_magic = esc_magic+start
         else:
             start_magic = start
         # Look through the input history in reverse
         for n in range(len(self.shell.input_hist)-2,0,-1):
             input = self.shell.input_hist[n]
             # skip plain 'r' lines so we don't recurse to infinity
             if input != '_ip.magic("r")\n' and \
                    (input.startswith(start) or input.startswith(start_magic)):
                 #print 'match',`input`  # dbg
                 print 'Executing:',input,
                 self.shell.runlines(input)
                 return
         print 'No previous input matching `%s` found.' % start
 
     
     def magic_bookmark(self, parameter_s=''):
         """Manage IPython's bookmark system.
 
         %bookmark <name>       - set bookmark to current dir
         %bookmark <name> <dir> - set bookmark to <dir>
         %bookmark -l           - list all bookmarks
         %bookmark -d <name>    - remove bookmark
         %bookmark -r           - remove all bookmarks
 
         You can later on access a bookmarked folder with:
           %cd -b <name>
         or simply '%cd <name>' if there is no directory called <name> AND
         there is such a bookmark defined.
 
         Your bookmarks persist through IPython sessions, but they are
         associated with each profile."""
 
         opts,args = self.parse_options(parameter_s,'drl',mode='list')
         if len(args) > 2:
             raise UsageError("%bookmark: too many arguments")
 
         bkms = self.db.get('bookmarks',{})
             
         if opts.has_key('d'):
             try:
                 todel = args[0]
             except IndexError:
                 raise UsageError(
                     "%bookmark -d: must provide a bookmark to delete")
             else:
                 try:
                     del bkms[todel]
                 except KeyError:
                     raise UsageError(
                         "%%bookmark -d: Can't delete bookmark '%s'" % todel)
 
         elif opts.has_key('r'):
             bkms = {}
         elif opts.has_key('l'):
             bks = bkms.keys()
             bks.sort()
             if bks:
                 size = max(map(len,bks))
             else:
                 size = 0
             fmt = '%-'+str(size)+'s -> %s'
             print 'Current bookmarks:'
             for bk in bks:
                 print fmt % (bk,bkms[bk])
         else:
             if not args:
                 raise UsageError("%bookmark: You must specify the bookmark name")
             elif len(args)==1:
                 bkms[args[0]] = os.getcwd()
             elif len(args)==2:
                 bkms[args[0]] = args[1]
         self.db['bookmarks'] = bkms
 
     def magic_pycat(self, parameter_s=''):
         """Show a syntax-highlighted file through a pager.
 
         This magic is similar to the cat utility, but it will assume the file
         to be Python source and will show it with syntax highlighting. """
         
         try:
             filename = get_py_filename(parameter_s)
             cont = file_read(filename)
         except IOError:
             try:
                 cont = eval(parameter_s,self.user_ns)
             except NameError:
                 cont = None
         if cont is None:
             print "Error: no such file or variable"
             return
             
         page(self.shell.pycolorize(cont),
              screen_lines=self.shell.rc.screen_length)
 
     def magic_cpaste(self, parameter_s=''):
         """Allows you to paste & execute a pre-formatted code block from clipboard.
         
         You must terminate the block with '--' (two minus-signs) alone on the
         line. You can also provide your own sentinel with '%paste -s %%' ('%%' 
         is the new sentinel for this operation)
         
         The block is dedented prior to execution to enable execution of method
         definitions. '>' and '+' characters at the beginning of a line are
         ignored, to allow pasting directly from e-mails, diff files and
         doctests (the '...' continuation prompt is also stripped).  The
         executed block is also assigned to variable named 'pasted_block' for
         later editing with '%edit pasted_block'.
         
         You can also pass a variable name as an argument, e.g. '%cpaste foo'.
         This assigns the pasted block to variable 'foo' as string, without 
         dedenting or executing it (preceding >>> and + is still stripped)
         
         Do not be alarmed by garbled output on Windows (it's a readline bug). 
         Just press enter and type -- (and press enter again) and the block 
         will be what was just pasted.
         
         IPython statements (magics, shell escapes) are not supported (yet).
         """
         opts,args = self.parse_options(parameter_s,'s:',mode='string')
         par = args.strip()
         sentinel = opts.get('s','--')
 
         # Regular expressions that declare text we strip from the input:
         strip_re =  [r'^\s*In \[\d+\]:', # IPython input prompt
                      r'^\s*(\s?>)+', # Python input prompt
                      r'^\s*\.{3,}', # Continuation prompts
                      r'^\++',
                      ]
 
         strip_from_start = map(re.compile,strip_re)
         
         from IPython import iplib
         lines = []
         print "Pasting code; enter '%s' alone on the line to stop." % sentinel
         while 1:
             l = iplib.raw_input_original(':')
             if l ==sentinel:
                 break
             
             for pat in strip_from_start: 
                 l = pat.sub('',l)
             lines.append(l)
                          
         block = "\n".join(lines) + '\n'
         #print "block:\n",block
         if not par:
             b = textwrap.dedent(block)
             exec b in self.user_ns            
             self.user_ns['pasted_block'] = b
         else:
-            self.user_ns[par] = block
+            self.user_ns[par] = SList(block.splitlines())
             print "Block assigned to '%s'" % par
             
     def magic_quickref(self,arg):
         """ Show a quick reference sheet """
         import IPython.usage
         qr = IPython.usage.quick_reference + self.magic_magic('-brief')
         
         page(qr)
         
     def magic_upgrade(self,arg):
         """ Upgrade your IPython installation
         
         This will copy the config files that don't yet exist in your 
         ipython dir from the system config dir. Use this after upgrading 
         IPython if you don't wish to delete your .ipython dir.
 
         Call with -nolegacy to get rid of ipythonrc* files (recommended for
         new users)
 
         """
         ip = self.getapi()
         ipinstallation = path(IPython.__file__).dirname()
         upgrade_script = '%s "%s"' % (sys.executable,ipinstallation / 'upgrade_dir.py')
         src_config = ipinstallation / 'UserConfig'
         userdir = path(ip.options.ipythondir)
         cmd = '%s "%s" "%s"' % (upgrade_script, src_config, userdir)
         print ">",cmd
         shell(cmd)
         if arg == '-nolegacy':
             legacy = userdir.files('ipythonrc*')
             print "Nuking legacy files:",legacy
             
             [p.remove() for p in legacy]
             suffix = (sys.platform == 'win32' and '.ini' or '')
             (userdir / ('ipythonrc' + suffix)).write_text('# Empty, see ipy_user_conf.py\n')
 
 
     def magic_doctest_mode(self,parameter_s=''):
         """Toggle doctest mode on and off.
 
         This mode allows you to toggle the prompt behavior between normal
         IPython prompts and ones that are as similar to the default IPython
         interpreter as possible.
 
         It also supports the pasting of code snippets that have leading '>>>'
         and '...' prompts in them.  This means that you can paste doctests from
         files or docstrings (even if they have leading whitespace), and the
         code will execute correctly.  You can then use '%history -tn' to see
         the translated history without line numbers; this will give you the
         input after removal of all the leading prompts and whitespace, which
         can be pasted back into an editor.
 
         With these features, you can switch into this mode easily whenever you
         need to do testing and changes to doctests, without having to leave
         your existing IPython session.
         """
 
         # XXX - Fix this to have cleaner activate/deactivate calls.
         from IPython.Extensions import InterpreterPasteInput as ipaste
         from IPython.ipstruct import Struct
 
         # Shorthands
         shell = self.shell
         oc = shell.outputcache
         rc = shell.rc
         meta = shell.meta
         # dstore is a data store kept in the instance metadata bag to track any
         # changes we make, so we can undo them later.
         dstore = meta.setdefault('doctest_mode',Struct())
         save_dstore = dstore.setdefault
 
         # save a few values we'll need to recover later
         mode = save_dstore('mode',False)
         save_dstore('rc_pprint',rc.pprint)
         save_dstore('xmode',shell.InteractiveTB.mode)
         save_dstore('rc_separate_out',rc.separate_out)
         save_dstore('rc_separate_out2',rc.separate_out2)
         save_dstore('rc_prompts_pad_left',rc.prompts_pad_left)
 
         if mode == False:
             # turn on
             ipaste.activate_prefilter()
 
             oc.prompt1.p_template = '>>> '
             oc.prompt2.p_template = '... '
             oc.prompt_out.p_template = ''
 
             oc.output_sep = ''
             oc.output_sep2 = ''
 
             oc.prompt1.pad_left = oc.prompt2.pad_left = \
                                   oc.prompt_out.pad_left = False
 
             rc.pprint = False
 
             shell.magic_xmode('Plain')
 
         else:
             # turn off
             ipaste.deactivate_prefilter()
 
             oc.prompt1.p_template = rc.prompt_in1
             oc.prompt2.p_template = rc.prompt_in2
             oc.prompt_out.p_template = rc.prompt_out
 
             oc.output_sep = dstore.rc_separate_out
             oc.output_sep2 = dstore.rc_separate_out2
 
             oc.prompt1.pad_left = oc.prompt2.pad_left = \
                          oc.prompt_out.pad_left = dstore.rc_prompts_pad_left
 
             rc.pprint = dstore.rc_pprint
 
             shell.magic_xmode(dstore.xmode)
 
         # Store new mode and inform
         dstore.mode = bool(1-int(mode))
         print 'Doctest mode is:',
         print ['OFF','ON'][dstore.mode]
 
 # end Magic

IPython/Release.py

@@ -1,95 +1,99 b''
 # -*- coding: utf-8 -*-
 """Release data for the IPython project.
 
 $Id: Release.py 3002 2008-02-01 07:17:00Z fperez $"""
 
 #*****************************************************************************
 #       Copyright (C) 2001-2006 Fernando Perez <[e-mail]>
 #
 #       Copyright (c) 2001 Janko Hauser <[e-mail]> and Nathaniel Gray
 #       <[e-mail]>
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #*****************************************************************************
 
 # Name of the package for release purposes.  This is the name which labels
 # the tarballs and RPMs made by distutils, so it's best to lowercase it.
 name = 'ipython'
 
 # For versions with substrings (like 0.6.16.svn), use an extra . to separate
 # the new substring.  We have to avoid using either dashes or underscores,
 # because bdist_rpm does not accept dashes (an RPM) convention, and
 # bdist_deb does not accept underscores (a Debian convention).
 
-revision = '1016'
+development = True    # change this to False to do a release
+version_base = '0.9.0'
 branch = 'ipython'
+revision = '1016'
 
+if development:
 if branch == 'ipython':
-    version = '0.9.0.bzr.r' + revision
+        version = '%s.bzr.r%s' % (version_base, revision)
+    else:
+        version = '%s.bzr.r%s.%s' % (version_base, revision, branch)
 else:
-    version = '0.9.0.bzr.r%s.%s'  % (revision,branch)
+    version = version_base
 
-# version = '0.8.4'
 
 description = "Tools for interactive development in Python."
 
 long_description = \
 """
 IPython provides a replacement for the interactive Python interpreter with
 extra functionality.
 
 Main features:
 
  * Comprehensive object introspection.
 
  * Input history, persistent across sessions.
 
  * Caching of output results during a session with automatically generated
    references.
 
  * Readline based name completion.
 
  * Extensible system of 'magic' commands for controlling the environment and
    performing many tasks related either to IPython or the operating system.
 
  * Configuration system with easy switching between different setups (simpler
    than changing $PYTHONSTARTUP environment variables every time).
 
  * Session logging and reloading.
 
  * Extensible syntax processing for special purpose situations.
 
  * Access to the system shell with user-extensible alias system.
 
  * Easily embeddable in other Python programs.
 
  * Integrated access to the pdb debugger and the Python profiler. 
 
  The latest development version is always available at the IPython subversion
  repository_.
 
 .. _repository: http://ipython.scipy.org/svn/ipython/ipython/trunk#egg=ipython-dev
  """
 
 license = 'BSD'
 
 authors = {'Fernando' : ('Fernando Perez','[e-mail]'),
            'Janko'    : ('Janko Hauser','[e-mail]'),
            'Nathan'   : ('Nathaniel Gray','[e-mail]'),
            'Ville'    : ('Ville Vainio','[e-mail]'),
            'Brian'    : ('Brian E Granger', '[e-mail]'),
            'Min'      : ('Min Ragan-Kelley', '[e-mail]')
            }
 
 author = 'The IPython Development Team'
 
 author_email = '[e-mail]'
 
 url = 'http://ipython.scipy.org'
 
 download_url = 'http://ipython.scipy.org/dist'
 
 platforms = ['Linux','Mac OSX','Windows XP/2000/NT','Windows 95/98/ME']
 
 keywords = ['Interactive','Interpreter','Shell','Parallel','Distributed']

IPython/external/mglob.py

@@ -1,203 +1,231 b''
 #!/usr/bin/env python
 
 r""" mglob - enhanced file list expansion module
 
 Use as stand-alone utility (for xargs, `backticks` etc.), 
 or a globbing library for own python programs. Globbing the sys.argv is something 
 that almost every Windows script has to perform manually, and this module is here
 to help with that task. Also Unix users will benefit from enhanced modes 
 such as recursion, exclusion, directory omission...
 
 Unlike glob.glob, directories are not included in the glob unless specified 
 with 'dir:'
 
 'expand' is the function to use in python programs. Typical use
 to expand argv (esp. in windows)::
 
     try:
         import mglob 
         files = mglob.expand(sys.argv[1:])
     except ImportError:
         print "mglob not found; try 'easy_install mglob' for extra features"
         files = sys.argv[1:] 
 
 Note that for unix, shell expands *normal* wildcards (*.cpp, etc.) in argv.
 Therefore, you might want to use quotes with normal wildcards to prevent this 
 expansion, in order for mglob to see the wildcards and get the wanted behaviour.
 Not quoting the wildcards is harmless and typically has equivalent results, though.
 
 Author: Ville Vainio <[e-mail]>
 License: MIT Open Source license
 
 """
 
 #Assigned in variable for "usage" printing convenience"
 
 globsyntax = """\
     This program allows specifying filenames with "mglob" mechanism.
     Supported syntax in globs (wilcard matching patterns)::
     
      *.cpp ?ellowo*                
          - obvious. Differs from normal glob in that dirs are not included.
            Unix users might want to write this as: "*.cpp" "?ellowo*"
      rec:/usr/share=*.txt,*.doc    
          - get all *.txt and *.doc under /usr/share, 
            recursively
      rec:/usr/share
          - All files under /usr/share, recursively
      rec:*.py
          - All .py files under current working dir, recursively
      foo                           
          - File or dir foo
      !*.bak readme*                   
          - readme*, exclude files ending with .bak
      !.svn/ !.hg/ !*_Data/ rec:.
          - Skip .svn, .hg, foo_Data dirs (and their subdirs) in recurse.
-           Trailing / is the key, \ does not work!
+           Trailing / is the key, \ does not work! Use !.*/ for all hidden.
      dir:foo                       
          - the directory foo if it exists (not files in foo)
      dir:*                         
          - all directories in current folder
      foo.py bar.* !h* rec:*.py
          - Obvious. !h* exclusion only applies for rec:*.py.
            foo.py is *not* included twice.
      @filelist.txt
          - All files listed in 'filelist.txt' file, on separate lines.
+     "cont:class \wak:" rec:*.py
+         - Match files containing regexp. Applies to subsequent files.
+           note quotes because of whitespace.
  """
 
 
 __version__ = "0.2"
 
 
-import os,glob,fnmatch,sys
+import os,glob,fnmatch,sys,re
 from sets import Set as set
 
                 
 def expand(flist,exp_dirs = False):
     """ Expand the glob(s) in flist.
     
     flist may be either a whitespace-separated list of globs/files
     or an array of globs/files.
     
     if exp_dirs is true, directory names in glob are expanded to the files
     contained in them - otherwise, directory names are returned as is.
     
     """
     if isinstance(flist, basestring):
-        flist = flist.split()
+        import shlex
+        flist = shlex.split(flist)
     done_set = set()
     denied_set = set()
+    cont_set = set()
+    cur_rejected_dirs = set()
 
     def recfind(p, pats = ["*"]):
-        denied_dirs = ["*" + d+"*" for d in denied_set if d.endswith("/")]
-        #print "de", denied_dirs
+        denied_dirs = [os.path.dirname(d) for d in denied_set if d.endswith("/")]
         for (dp,dnames,fnames) in os.walk(p):
             # see if we should ignore the whole directory
             dp_norm = dp.replace("\\","/") + "/"
             deny = False
+            # do not traverse under already rejected dirs
+            for d in cur_rejected_dirs:
+                if dp.startswith(d):
+                    deny = True
+                    break
+            if deny:
+                continue
+            
+
             #print "dp",dp
+            bname = os.path.basename(dp)
             for deny_pat in denied_dirs:
-                if fnmatch.fnmatch( dp_norm, deny_pat):
+                if fnmatch.fnmatch( bname, deny_pat):
                     deny = True
+                    cur_rejected_dirs.add(dp)
                     break
             if deny:
                 continue
 
                     
             for f in fnames:
                 matched = False
                 for p in pats:
                     if fnmatch.fnmatch(f,p):
                         matched = True
                         break
                 if matched:
                     yield os.path.join(dp,f)            
 
     def once_filter(seq):
         for it in seq:
             p = os.path.abspath(it)
             if p in done_set:
                 continue
             done_set.add(p)
             deny = False
             for deny_pat in denied_set:
                 if fnmatch.fnmatch(os.path.basename(p), deny_pat):
                     deny = True
                     break
+            if cont_set:
+                try:
+                    cont = open(p).read()
+                except IOError:
+                    # deny
+                    continue
+                for pat in cont_set:
+                    if not re.search(pat,cont, re.IGNORECASE):
+                        deny = True
+                        break
+                    
             if not deny:
                 yield it
         return
             
     res = []
 
     for ent in flist:
         ent = os.path.expanduser(os.path.expandvars(ent))
         if ent.lower().startswith('rec:'):
             fields = ent[4:].split('=')            
             if len(fields) == 2:
                 pth, patlist = fields
             elif len(fields) == 1:
                 if os.path.isdir(fields[0]):
                     # single arg is dir
                     pth, patlist = fields[0], '*'
                 else: 
                     # single arg is pattern
                     pth, patlist = '.', fields[0]
                     
             elif len(fields) == 0:
                 pth, pathlist = '.','*'
                 
             pats = patlist.split(',')
             res.extend(once_filter(recfind(pth, pats)))
         # filelist
         elif ent.startswith('@') and os.path.isfile(ent[1:]):
             res.extend(once_filter(open(ent[1:]).read().splitlines()))
         # exclusion
         elif ent.startswith('!'):
             denied_set.add(ent[1:])
         # glob only dirs
         elif ent.lower().startswith('dir:'):
             res.extend(once_filter(filter(os.path.isdir,glob.glob(ent[4:]))))
-            
+        elif ent.lower().startswith('cont:'):
+            cont_set.add(ent[5:])
         # get all files in the specified dir
         elif os.path.isdir(ent) and exp_dirs:
             res.extend(once_filter(filter(os.path.isfile,glob.glob(ent + os.sep+"*"))))
             
         # glob only files
 
         elif '*' in ent or '?' in ent:
             res.extend(once_filter(filter(os.path.isfile,glob.glob(ent))))
 
         else:
             res.extend(once_filter([ent]))
     return res
             
             
 def test():
     assert (
         expand("*.py ~/.ipython/*.py rec:/usr/share/doc-base") == 
         expand( ['*.py', '~/.ipython/*.py', 'rec:/usr/share/doc-base'] ) 
         )
     
 def main():
     if len(sys.argv) < 2:
         print globsyntax
         return
     
     print "\n".join(expand(sys.argv[1:])),
 
 def mglob_f(self, arg):
     from IPython.genutils import SList
     if arg.strip():
         return SList(expand(arg))
     print "Please specify pattern!"
     print globsyntax
 
 def init_ipython(ip):
     """ register %mglob for IPython """
     mglob_f.__doc__ = globsyntax
     ip.expose_magic("mglob",mglob_f)  
     
 # test()
 if __name__ == "__main__":
     main()

IPython/genutils.py

@@ -1,2097 +1,2124 b''
 # -*- coding: utf-8 -*-
 """
 General purpose utilities.
 
 This is a grab-bag of stuff I find useful in most programs I write. Some of
 these things are also convenient when working at the command line.
 
 $Id: genutils.py 2998 2008-01-31 10:06:04Z vivainio $"""
 
 #*****************************************************************************
 #       Copyright (C) 2001-2006 Fernando Perez. <[e-mail]>
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #*****************************************************************************
 
 from IPython import Release
 __author__  = '%s <%s>' % Release.authors['Fernando']
 __license__ = Release.license
 
 #****************************************************************************
 # required modules from the Python standard library
 import __main__
 import commands
 try:
     import doctest
 except ImportError:
     pass
 import os
 import platform
 import re
 import shlex
 import shutil
 import subprocess
 import sys
 import tempfile
 import time
 import types
 import warnings
 
 # Curses and termios are Unix-only modules
 try:
     import curses
     # We need termios as well, so if its import happens to raise, we bail on
     # using curses altogether.
     import termios
 except ImportError:
     USE_CURSES = False
 else:
     # Curses on Solaris may not be complete, so we can't use it there
     USE_CURSES = hasattr(curses,'initscr')
 
 # Other IPython utilities
 import IPython
 from IPython.Itpl import Itpl,itpl,printpl
 from IPython import DPyGetOpt, platutils
 from IPython.generics import result_display
 import IPython.ipapi
 from IPython.external.path import path
 if os.name == "nt":
     from IPython.winconsole import get_console_size
 
 try:
     set
 except:
     from sets import Set as set
 
 
 #****************************************************************************
 # Exceptions
 class Error(Exception):
     """Base class for exceptions in this module."""
     pass
 
 #----------------------------------------------------------------------------
 class IOStream:
     def __init__(self,stream,fallback):
         if not hasattr(stream,'write') or not hasattr(stream,'flush'):
             stream = fallback
         self.stream = stream
         self._swrite = stream.write
         self.flush = stream.flush
 
     def write(self,data):
         try:
             self._swrite(data)
         except:
             try:
                 # print handles some unicode issues which may trip a plain
                 # write() call.  Attempt to emulate write() by using a
                 # trailing comma
                 print >> self.stream, data,
             except:
                 # if we get here, something is seriously broken.
                 print >> sys.stderr, \
                       'ERROR - failed to write data to stream:', self.stream
 
     def close(self):
         pass
                 
         
 class IOTerm:
     """ Term holds the file or file-like objects for handling I/O operations.
 
     These are normally just sys.stdin, sys.stdout and sys.stderr but for
     Windows they can can replaced to allow editing the strings before they are
     displayed."""
 
     # In the future, having IPython channel all its I/O operations through
     # this class will make it easier to embed it into other environments which
     # are not a normal terminal (such as a GUI-based shell)
     def __init__(self,cin=None,cout=None,cerr=None):
         self.cin  = IOStream(cin,sys.stdin)
         self.cout = IOStream(cout,sys.stdout)
         self.cerr = IOStream(cerr,sys.stderr)
         
 # Global variable to be used for all I/O
 Term = IOTerm()
 
 import IPython.rlineimpl as readline
 # Remake Term to use the readline i/o facilities
 if sys.platform == 'win32' and readline.have_readline:
     
     Term = IOTerm(cout=readline._outputfile,cerr=readline._outputfile)
     
 
 #****************************************************************************
 # Generic warning/error printer, used by everything else
 def warn(msg,level=2,exit_val=1):
     """Standard warning printer. Gives formatting consistency.
 
     Output is sent to Term.cerr (sys.stderr by default).
     
     Options:
     
     -level(2): allows finer control:
       0 -> Do nothing, dummy function.
       1 -> Print message.
       2 -> Print 'WARNING:' + message. (Default level).
       3 -> Print 'ERROR:' + message.
       4 -> Print 'FATAL ERROR:' + message and trigger a sys.exit(exit_val).
 
     -exit_val (1): exit value returned by sys.exit() for a level 4
     warning. Ignored for all other levels."""
     
     if level>0:
         header = ['','','WARNING: ','ERROR: ','FATAL ERROR: ']
         print >> Term.cerr, '%s%s' % (header[level],msg)
         if level == 4:
             print >> Term.cerr,'Exiting.\n'
             sys.exit(exit_val)
 
 def info(msg):
     """Equivalent to warn(msg,level=1)."""
 
     warn(msg,level=1)
 
 def error(msg):
     """Equivalent to warn(msg,level=3)."""
 
     warn(msg,level=3)
 
 def fatal(msg,exit_val=1):
     """Equivalent to warn(msg,exit_val=exit_val,level=4)."""
 
     warn(msg,exit_val=exit_val,level=4)
 
 #---------------------------------------------------------------------------
 # Debugging routines
 # 
 def debugx(expr,pre_msg=''):
     """Print the value of an expression from the caller's frame.
     
     Takes an expression, evaluates it in the caller's frame and prints both
     the given expression and the resulting value (as well as a debug mark
     indicating the name of the calling function.  The input must be of a form
     suitable for eval().
 
     An optional message can be passed, which will be prepended to the printed
     expr->value pair."""
     
     cf = sys._getframe(1)
     print '[DBG:%s] %s%s -> %r' % (cf.f_code.co_name,pre_msg,expr,
                                    eval(expr,cf.f_globals,cf.f_locals))
 
 # deactivate it by uncommenting the following line, which makes it a no-op
 #def debugx(expr,pre_msg=''): pass
 
 #----------------------------------------------------------------------------
 StringTypes = types.StringTypes
 
 # Basic timing functionality
 
 # If possible (Unix), use the resource module instead of time.clock()
 try:
     import resource
     def clocku():
         """clocku() -> floating point number
 
         Return the *USER* CPU time in seconds since the start of the process.
         This is done via a call to resource.getrusage, so it avoids the
         wraparound problems in time.clock()."""
         
         return resource.getrusage(resource.RUSAGE_SELF)[0]
     
     def clocks():
         """clocks() -> floating point number
 
         Return the *SYSTEM* CPU time in seconds since the start of the process.
         This is done via a call to resource.getrusage, so it avoids the
         wraparound problems in time.clock()."""
         
         return resource.getrusage(resource.RUSAGE_SELF)[1]
     
     def clock():
         """clock() -> floating point number
 
         Return the *TOTAL USER+SYSTEM* CPU time in seconds since the start of
         the process.  This is done via a call to resource.getrusage, so it
         avoids the wraparound problems in time.clock()."""
 
         u,s = resource.getrusage(resource.RUSAGE_SELF)[:2] 
         return u+s
     
     def clock2():
         """clock2() -> (t_user,t_system)
 
         Similar to clock(), but return a tuple of user/system times."""
         return resource.getrusage(resource.RUSAGE_SELF)[:2]
 
 except ImportError:
     # There is no distinction of user/system time under windows, so we just use
     # time.clock() for everything...
     clocku = clocks = clock = time.clock
     def clock2():
         """Under windows, system CPU time can't be measured.
 
         This just returns clock() and zero."""
         return time.clock(),0.0
 
 def timings_out(reps,func,*args,**kw):
     """timings_out(reps,func,*args,**kw) -> (t_total,t_per_call,output)
 
     Execute a function reps times, return a tuple with the elapsed total
     CPU time in seconds, the time per call and the function's output.
 
     Under Unix, the return value is the sum of user+system time consumed by
     the process, computed via the resource module.  This prevents problems
     related to the wraparound effect which the time.clock() function has.
     
     Under Windows the return value is in wall clock seconds. See the
     documentation for the time module for more details."""
 
     reps = int(reps)
     assert reps >=1, 'reps must be >= 1'
     if reps==1:
         start = clock()
         out = func(*args,**kw)
         tot_time = clock()-start
     else:
         rng = xrange(reps-1) # the last time is executed separately to store output
         start = clock()
         for dummy in rng: func(*args,**kw)
         out = func(*args,**kw)  # one last time
         tot_time = clock()-start
     av_time = tot_time / reps
     return tot_time,av_time,out
 
 def timings(reps,func,*args,**kw):
     """timings(reps,func,*args,**kw) -> (t_total,t_per_call)
 
     Execute a function reps times, return a tuple with the elapsed total CPU
     time in seconds and the time per call. These are just the first two values
     in timings_out()."""
 
     return timings_out(reps,func,*args,**kw)[0:2]
 
 def timing(func,*args,**kw):
     """timing(func,*args,**kw) -> t_total
 
     Execute a function once, return the elapsed total CPU time in
     seconds. This is just the first value in timings_out()."""
 
     return timings_out(1,func,*args,**kw)[0]
 
 #****************************************************************************
 # file and system
 
 def arg_split(s,posix=False):
     """Split a command line's arguments in a shell-like manner.
 
     This is a modified version of the standard library's shlex.split()
     function, but with a default of posix=False for splitting, so that quotes
     in inputs are respected."""
 
     # XXX - there may be unicode-related problems here!!!  I'm not sure that
     # shlex is truly unicode-safe, so it might be necessary to do
     #
     # s = s.encode(sys.stdin.encoding)
     #
     # first, to ensure that shlex gets a normal string.  Input from anyone who
     # knows more about unicode and shlex than I would be good to have here...
     lex = shlex.shlex(s, posix=posix)
     lex.whitespace_split = True
     return list(lex)
 
 def system(cmd,verbose=0,debug=0,header=''):
     """Execute a system command, return its exit status.
 
     Options:
 
     - verbose (0): print the command to be executed.
     
     - debug (0): only print, do not actually execute.
 
     - header (''): Header to print on screen prior to the executed command (it
     is only prepended to the command, no newlines are added).
 
     Note: a stateful version of this function is available through the
     SystemExec class."""
 
     stat = 0
     if verbose or debug: print header+cmd
     sys.stdout.flush()
     if not debug: stat = os.system(cmd)
     return stat
 
 def abbrev_cwd():
     """ Return abbreviated version of cwd, e.g. d:mydir """
     cwd = os.getcwd().replace('\\','/')
     drivepart = ''
     tail = cwd
     if sys.platform == 'win32':
         if len(cwd) < 4:
             return cwd
         drivepart,tail = os.path.splitdrive(cwd)
     
 
     parts = tail.split('/')    
     if len(parts) > 2:
         tail = '/'.join(parts[-2:])
         
     return (drivepart + (
         cwd == '/' and '/' or tail))
     
 
 # This function is used by ipython in a lot of places to make system calls.
 # We need it to be slightly different under win32, due to the vagaries of
 # 'network shares'.  A win32 override is below.
 
 def shell(cmd,verbose=0,debug=0,header=''):
     """Execute a command in the system shell, always return None.
 
     Options:
 
     - verbose (0): print the command to be executed.
     
     - debug (0): only print, do not actually execute.
 
     - header (''): Header to print on screen prior to the executed command (it
     is only prepended to the command, no newlines are added).
 
     Note: this is similar to genutils.system(), but it returns None so it can
     be conveniently used in interactive loops without getting the return value
     (typically 0) printed many times."""
 
     stat = 0
     if verbose or debug: print header+cmd
     # flush stdout so we don't mangle python's buffering
     sys.stdout.flush()
     
     if not debug:
         platutils.set_term_title("IPy " + cmd)
         os.system(cmd)
         platutils.set_term_title("IPy " + abbrev_cwd())
 
 # override shell() for win32 to deal with network shares
 if os.name in ('nt','dos'):
 
     shell_ori = shell
 
     def shell(cmd,verbose=0,debug=0,header=''):
         if os.getcwd().startswith(r"\\"):
             path = os.getcwd()
             # change to c drive (cannot be on UNC-share when issuing os.system,
             # as cmd.exe cannot handle UNC addresses)
             os.chdir("c:")
             # issue pushd to the UNC-share and then run the command
             try:
                 shell_ori('"pushd %s&&"'%path+cmd,verbose,debug,header)
             finally:
                 os.chdir(path)
         else:
             shell_ori(cmd,verbose,debug,header)
 
     shell.__doc__ = shell_ori.__doc__
 
 def getoutput(cmd,verbose=0,debug=0,header='',split=0):
     """Dummy substitute for perl's backquotes.
 
     Executes a command and returns the output.
 
     Accepts the same arguments as system(), plus:
 
     - split(0): if true, the output is returned as a list split on newlines.
 
     Note: a stateful version of this function is available through the
     SystemExec class.
     
     This is pretty much deprecated and rarely used, 
     genutils.getoutputerror may be what you need.
     
     """
 
     if verbose or debug: print header+cmd
     if not debug:
         output = os.popen(cmd).read()
         # stipping last \n is here for backwards compat.
         if output.endswith('\n'):
             output = output[:-1]
         if split:
             return output.split('\n')
         else:
             return output
 
 def getoutputerror(cmd,verbose=0,debug=0,header='',split=0):
     """Return (standard output,standard error) of executing cmd in a shell.
 
     Accepts the same arguments as system(), plus:
 
     - split(0): if true, each of stdout/err is returned as a list split on
     newlines.
 
     Note: a stateful version of this function is available through the
     SystemExec class."""
 
     if verbose or debug: print header+cmd
     if not cmd:
         if split:
             return [],[]
         else:
             return '',''
     if not debug:
         pin,pout,perr = os.popen3(cmd)
         tout = pout.read().rstrip()
         terr = perr.read().rstrip()
         pin.close()
         pout.close()
         perr.close()
         if split:
             return tout.split('\n'),terr.split('\n')
         else:
             return tout,terr
 
 # for compatibility with older naming conventions
 xsys = system
 bq = getoutput
 
 class SystemExec:
     """Access the system and getoutput functions through a stateful interface.
 
     Note: here we refer to the system and getoutput functions from this
     library, not the ones from the standard python library.
     
     This class offers the system and getoutput functions as methods, but the
     verbose, debug and header parameters can be set for the instance (at
     creation time or later) so that they don't need to be specified on each
     call.
 
     For efficiency reasons, there's no way to override the parameters on a
     per-call basis other than by setting instance attributes. If you need
     local overrides, it's best to directly call system() or getoutput().
 
     The following names are provided as alternate options:
      - xsys: alias to system
      - bq: alias to getoutput
 
     An instance can then be created as:
      >>> sysexec = SystemExec(verbose=1,debug=0,header='Calling: ')
 
     And used as:
      >>> sysexec.xsys('pwd')
      >>> dirlist = sysexec.bq('ls -l')
     """
     
     def __init__(self,verbose=0,debug=0,header='',split=0):
         """Specify the instance's values for verbose, debug and header."""
         setattr_list(self,'verbose debug header split')
 
     def system(self,cmd):
         """Stateful interface to system(), with the same keyword parameters."""
 
         system(cmd,self.verbose,self.debug,self.header)
 
     def shell(self,cmd):
         """Stateful interface to shell(), with the same keyword parameters."""
 
         shell(cmd,self.verbose,self.debug,self.header)
 
     xsys = system  # alias
 
     def getoutput(self,cmd):
         """Stateful interface to getoutput()."""
 
         return getoutput(cmd,self.verbose,self.debug,self.header,self.split)
 
     def getoutputerror(self,cmd):
         """Stateful interface to getoutputerror()."""
 
         return getoutputerror(cmd,self.verbose,self.debug,self.header,self.split)
 
     bq = getoutput  # alias
 
 #-----------------------------------------------------------------------------
 def mutex_opts(dict,ex_op):
     """Check for presence of mutually exclusive keys in a dict.
 
     Call: mutex_opts(dict,[[op1a,op1b],[op2a,op2b]...]"""
     for op1,op2 in ex_op:
         if op1 in dict and op2 in dict:
             raise ValueError,'\n*** ERROR in Arguments *** '\
                   'Options '+op1+' and '+op2+' are mutually exclusive.'
 
 #-----------------------------------------------------------------------------
 def get_py_filename(name):
     """Return a valid python filename in the current directory.
 
     If the given name is not a file, it adds '.py' and searches again.
     Raises IOError with an informative message if the file isn't found."""
 
     name = os.path.expanduser(name)
     if not os.path.isfile(name) and not name.endswith('.py'):
         name += '.py'
     if os.path.isfile(name):
         return name
     else:
         raise IOError,'File `%s` not found.' % name
 
 #-----------------------------------------------------------------------------
 def filefind(fname,alt_dirs = None):
     """Return the given filename either in the current directory, if it
     exists, or in a specified list of directories.
 
     ~ expansion is done on all file and directory names.
 
     Upon an unsuccessful search, raise an IOError exception."""
 
     if alt_dirs is None:
         try:
             alt_dirs = get_home_dir()
         except HomeDirError:
             alt_dirs = os.getcwd()
     search = [fname] + list_strings(alt_dirs)
     search = map(os.path.expanduser,search)
     #print 'search list for',fname,'list:',search  # dbg
     fname = search[0]
     if os.path.isfile(fname):
         return fname
     for direc in search[1:]:
         testname = os.path.join(direc,fname)
         #print 'testname',testname  # dbg
         if os.path.isfile(testname):
             return testname
     raise IOError,'File' + `fname` + \
           ' not found in current or supplied directories:' + `alt_dirs`
 
 #----------------------------------------------------------------------------
 def file_read(filename):
     """Read a file and close it.  Returns the file source."""
     fobj = open(filename,'r');
     source = fobj.read();
     fobj.close()
     return source
 
 def file_readlines(filename):
     """Read a file and close it.  Returns the file source using readlines()."""
     fobj = open(filename,'r');
     lines = fobj.readlines();
     fobj.close()
     return lines
 
 #----------------------------------------------------------------------------
 def target_outdated(target,deps):
     """Determine whether a target is out of date.
 
     target_outdated(target,deps) -> 1/0
 
     deps: list of filenames which MUST exist.
     target: single filename which may or may not exist.
 
     If target doesn't exist or is older than any file listed in deps, return
     true, otherwise return false.
     """
     try:
         target_time = os.path.getmtime(target)
     except os.error:
         return 1
     for dep in deps:
         dep_time = os.path.getmtime(dep)
         if dep_time > target_time:
             #print "For target",target,"Dep failed:",dep # dbg
             #print "times (dep,tar):",dep_time,target_time # dbg
             return 1
     return 0
 
 #-----------------------------------------------------------------------------
 def target_update(target,deps,cmd):
     """Update a target with a given command given a list of dependencies.
 
     target_update(target,deps,cmd) -> runs cmd if target is outdated.
 
     This is just a wrapper around target_outdated() which calls the given
     command if target is outdated."""
 
     if target_outdated(target,deps):
         xsys(cmd)
 
 #----------------------------------------------------------------------------
 def unquote_ends(istr):
     """Remove a single pair of quotes from the endpoints of a string."""
 
     if not istr:
         return istr
     if (istr[0]=="'" and istr[-1]=="'") or \
        (istr[0]=='"' and istr[-1]=='"'):
         return istr[1:-1]
     else:
         return istr
 
 #----------------------------------------------------------------------------
 def process_cmdline(argv,names=[],defaults={},usage=''):
     """ Process command-line options and arguments.
 
     Arguments:
 
     - argv: list of arguments, typically sys.argv.
     
     - names: list of option names. See DPyGetOpt docs for details on options
     syntax.
 
     - defaults: dict of default values.
 
     - usage: optional usage notice to print if a wrong argument is passed.
 
     Return a dict of options and a list of free arguments."""
 
     getopt = DPyGetOpt.DPyGetOpt()
     getopt.setIgnoreCase(0)
     getopt.parseConfiguration(names)
 
     try:
         getopt.processArguments(argv)
     except DPyGetOpt.ArgumentError, exc:
         print usage
         warn('"%s"' % exc,level=4)
 
     defaults.update(getopt.optionValues)
     args = getopt.freeValues
     
     return defaults,args
 
 #----------------------------------------------------------------------------
 def optstr2types(ostr):
     """Convert a string of option names to a dict of type mappings.
 
     optstr2types(str) -> {None:'string_opts',int:'int_opts',float:'float_opts'}
 
     This is used to get the types of all the options in a string formatted
     with the conventions of DPyGetOpt. The 'type' None is used for options
     which are strings (they need no further conversion). This function's main
     use is to get a typemap for use with read_dict().
     """
 
     typeconv = {None:'',int:'',float:''}
     typemap = {'s':None,'i':int,'f':float}
     opt_re = re.compile(r'([\w]*)([^:=]*:?=?)([sif]?)')
 
     for w in ostr.split():
         oname,alias,otype = opt_re.match(w).groups()
         if otype == '' or alias == '!':   # simple switches are integers too
             otype = 'i'
         typeconv[typemap[otype]] += oname + ' '
     return typeconv
 
 #----------------------------------------------------------------------------
 def read_dict(filename,type_conv=None,**opt):
 
     """Read a dictionary of key=value pairs from an input file, optionally
     performing conversions on the resulting values.
 
     read_dict(filename,type_conv,**opt) -> dict
 
     Only one value per line is accepted, the format should be
      # optional comments are ignored
      key value\n
 
     Args:
 
       - type_conv: A dictionary specifying which keys need to be converted to
       which types. By default all keys are read as strings. This dictionary
       should have as its keys valid conversion functions for strings
       (int,long,float,complex, or your own).  The value for each key
       (converter) should be a whitespace separated string containing the names
       of all the entries in the file to be converted using that function. For
       keys to be left alone, use None as the conversion function (only needed
       with purge=1, see below).
 
       - opt: dictionary with extra options as below (default in parens)
 
         purge(0): if set to 1, all keys *not* listed in type_conv are purged out
         of the dictionary to be returned. If purge is going to be used, the
         set of keys to be left as strings also has to be explicitly specified
         using the (non-existent) conversion function None.
 
         fs(None): field separator. This is the key/value separator to be used
         when parsing the file. The None default means any whitespace [behavior
         of string.split()].
 
         strip(0): if 1, strip string values of leading/trailinig whitespace.
 
         warn(1): warning level if requested keys are not found in file.
           - 0: silently ignore.
           - 1: inform but proceed.
           - 2: raise KeyError exception.
 
         no_empty(0): if 1, remove keys with whitespace strings as a value.
 
         unique([]): list of keys (or space separated string) which can't be
         repeated. If one such key is found in the file, each new instance
         overwrites the previous one. For keys not listed here, the behavior is
         to make a list of all appearances.
 
     Example:
     If the input file test.ini has:
       i 3
       x 4.5
       y 5.5
       s hi ho
     Then:
 
     >>> type_conv={int:'i',float:'x',None:'s'}
     >>> read_dict('test.ini')
     {'i': '3', 's': 'hi ho', 'x': '4.5', 'y': '5.5'}
     >>> read_dict('test.ini',type_conv)
     {'i': 3, 's': 'hi ho', 'x': 4.5, 'y': '5.5'}
     >>> read_dict('test.ini',type_conv,purge=1)
     {'i': 3, 's': 'hi ho', 'x': 4.5}
     """
 
     # starting config
     opt.setdefault('purge',0)
     opt.setdefault('fs',None)  # field sep defaults to any whitespace
     opt.setdefault('strip',0)
     opt.setdefault('warn',1)
     opt.setdefault('no_empty',0)
     opt.setdefault('unique','')
     if type(opt['unique']) in StringTypes:
         unique_keys = qw(opt['unique'])
     elif type(opt['unique']) in (types.TupleType,types.ListType):
         unique_keys = opt['unique']
     else:
         raise ValueError, 'Unique keys must be given as a string, List or Tuple'
 
     dict = {}
     # first read in table of values as strings
     file = open(filename,'r')
     for line in file.readlines():
         line = line.strip()
         if len(line) and line[0]=='#': continue
         if len(line)>0:
             lsplit = line.split(opt['fs'],1)
             try:
                 key,val = lsplit
             except ValueError:
                 key,val = lsplit[0],''
             key = key.strip()
             if opt['strip']: val = val.strip()
             if val == "''" or val == '""': val = ''
             if opt['no_empty'] and (val=='' or val.isspace()):
                 continue
             # if a key is found more than once in the file, build a list
             # unless it's in the 'unique' list. In that case, last found in file
             # takes precedence. User beware.
             try:
                 if dict[key] and key in unique_keys:
                     dict[key] = val
                 elif type(dict[key]) is types.ListType:
                     dict[key].append(val)
                 else:
                     dict[key] = [dict[key],val]
             except KeyError:
                 dict[key] = val
     # purge if requested
     if opt['purge']:
         accepted_keys = qwflat(type_conv.values())
         for key in dict.keys():
             if key in accepted_keys: continue
             del(dict[key])
     # now convert if requested
     if type_conv==None: return dict
     conversions = type_conv.keys()
     try: conversions.remove(None)
     except: pass
     for convert in conversions:
         for val in qw(type_conv[convert]):
             try:
                 dict[val] = convert(dict[val])
             except KeyError,e:
                 if opt['warn'] == 0:
                     pass
                 elif opt['warn'] == 1:
                     print >>sys.stderr, 'Warning: key',val,\
                           'not found in file',filename
                 elif opt['warn'] == 2:
                     raise KeyError,e
                 else:
                     raise ValueError,'Warning level must be 0,1 or 2'
 
     return dict
 
 #----------------------------------------------------------------------------
 def flag_calls(func):
     """Wrap a function to detect and flag when it gets called.
 
     This is a decorator which takes a function and wraps it in a function with
     a 'called' attribute. wrapper.called is initialized to False.
 
     The wrapper.called attribute is set to False right before each call to the
     wrapped function, so if the call fails it remains False.  After the call
     completes, wrapper.called is set to True and the output is returned.
 
     Testing for truth in wrapper.called allows you to determine if a call to
     func() was attempted and succeeded."""
     
     def wrapper(*args,**kw):
         wrapper.called = False
         out = func(*args,**kw)
         wrapper.called = True
         return out
 
     wrapper.called = False
     wrapper.__doc__ = func.__doc__
     return wrapper
 
 #----------------------------------------------------------------------------
 def dhook_wrap(func,*a,**k):
     """Wrap a function call in a sys.displayhook controller.
 
     Returns a wrapper around func which calls func, with all its arguments and
     keywords unmodified, using the default sys.displayhook.  Since IPython
     modifies sys.displayhook, it breaks the behavior of certain systems that
     rely on the default behavior, notably doctest.
     """
 
     def f(*a,**k):
     
         dhook_s = sys.displayhook
         sys.displayhook = sys.__displayhook__
         try:
             out = func(*a,**k)
         finally:
             sys.displayhook = dhook_s
 
         return out
 
     f.__doc__ = func.__doc__
     return f
 
 #----------------------------------------------------------------------------
 def doctest_reload():
     """Properly reload doctest to reuse it interactively.
 
     This routine:
 
       - reloads doctest
       
       - resets its global 'master' attribute to None, so that multiple uses of
       the module interactively don't produce cumulative reports.
     
       - Monkeypatches its core test runner method to protect it from IPython's
       modified displayhook.  Doctest expects the default displayhook behavior
       deep down, so our modification breaks it completely.  For this reason, a
       hard monkeypatch seems like a reasonable solution rather than asking
       users to manually use a different doctest runner when under IPython."""
 
     import doctest
     reload(doctest)
     doctest.master=None
 
     try:
         doctest.DocTestRunner
     except AttributeError:
         # This is only for python 2.3 compatibility, remove once we move to
         # 2.4 only.
         pass
     else:
         doctest.DocTestRunner.run = dhook_wrap(doctest.DocTestRunner.run)
 
 #----------------------------------------------------------------------------
 class HomeDirError(Error):
     pass
 
 def get_home_dir():
     """Return the closest possible equivalent to a 'home' directory.
 
     We first try $HOME.  Absent that, on NT it's $HOMEDRIVE\$HOMEPATH.
 
     Currently only Posix and NT are implemented, a HomeDirError exception is
     raised for all other OSes. """
 
     isdir = os.path.isdir
     env = os.environ
     
     # first, check py2exe distribution root directory for _ipython.
     # This overrides all. Normally does not exist.
     
     if '\\library.zip\\' in IPython.__file__.lower():
         root, rest = IPython.__file__.lower().split('library.zip')
         if isdir(root + '_ipython'):
             os.environ["IPYKITROOT"] = root.rstrip('\\')
             return root
     
     try:
         homedir = env['HOME']
         if not isdir(homedir):
             # in case a user stuck some string which does NOT resolve to a
             # valid path, it's as good as if we hadn't foud it
             raise KeyError
         return homedir
     except KeyError:
         if os.name == 'posix':
             raise HomeDirError,'undefined $HOME, IPython can not proceed.'
         elif os.name == 'nt':
             # For some strange reason, win9x returns 'nt' for os.name.
             try:
                 homedir = os.path.join(env['HOMEDRIVE'],env['HOMEPATH'])
                 if not isdir(homedir):
                     homedir = os.path.join(env['USERPROFILE'])
                     if not isdir(homedir):
                         raise HomeDirError
                 return homedir
             except:
                 try:
                     # Use the registry to get the 'My Documents' folder.
                     import _winreg as wreg
                     key = wreg.OpenKey(wreg.HKEY_CURRENT_USER,
                                        "Software\Microsoft\Windows\CurrentVersion\Explorer\Shell Folders")
                     homedir = wreg.QueryValueEx(key,'Personal')[0]
                     key.Close()
                     if not isdir(homedir):
                         e = ('Invalid "Personal" folder registry key '
                              'typically "My Documents".\n'
                              'Value: %s\n'
                              'This is not a valid directory on your system.' %
                              homedir)
                         raise HomeDirError(e)
                     return homedir
                 except HomeDirError:
                     raise
                 except:
                     return 'C:\\'
         elif os.name == 'dos':
             # Desperate, may do absurd things in classic MacOS. May work under DOS.
             return 'C:\\'
         else:
             raise HomeDirError,'support for your operating system not implemented.'
 
 #****************************************************************************
 # strings and text
 
 class LSString(str):
     """String derivative with a special access attributes.
 
     These are normal strings, but with the special attributes:
 
         .l (or .list) : value as list (split on newlines).
         .n (or .nlstr): original value (the string itself).
         .s (or .spstr): value as whitespace-separated string.
         .p (or .paths): list of path objects
         
     Any values which require transformations are computed only once and
     cached.
 
     Such strings are very useful to efficiently interact with the shell, which
     typically only understands whitespace-separated options for commands."""
 
     def get_list(self):
         try:
             return self.__list
         except AttributeError:
             self.__list = self.split('\n')
             return self.__list
 
     l = list = property(get_list)
 
     def get_spstr(self):
         try:
             return self.__spstr
         except AttributeError:
             self.__spstr = self.replace('\n',' ')
             return self.__spstr
 
     s = spstr = property(get_spstr)
 
     def get_nlstr(self):
         return self
 
     n = nlstr = property(get_nlstr)
 
     def get_paths(self):
         try:
             return self.__paths
         except AttributeError:
             self.__paths = [path(p) for p in self.split('\n') if os.path.exists(p)]
             return self.__paths
     
     p = paths = property(get_paths)
 
 def print_lsstring(arg):
     """ Prettier (non-repr-like) and more informative printer for LSString """
     print "LSString (.p, .n, .l, .s available). Value:"
     print arg
     
 print_lsstring = result_display.when_type(LSString)(print_lsstring)
 
 #----------------------------------------------------------------------------
 class SList(list):
     """List derivative with a special access attributes.
 
     These are normal lists, but with the special attributes:
 
         .l (or .list) : value as list (the list itself).
         .n (or .nlstr): value as a string, joined on newlines.
         .s (or .spstr): value as a string, joined on spaces.
         .p (or .paths): list of path objects
         
     Any values which require transformations are computed only once and
     cached."""
 
     def get_list(self):
         return self
 
     l = list = property(get_list)
 
     def get_spstr(self):
         try:
             return self.__spstr
         except AttributeError:
             self.__spstr = ' '.join(self)
             return self.__spstr
 
     s = spstr = property(get_spstr)
 
     def get_nlstr(self):
         try:
             return self.__nlstr
         except AttributeError:
             self.__nlstr = '\n'.join(self)
             return self.__nlstr
 
     n = nlstr = property(get_nlstr)
     
     def get_paths(self):
         try:
             return self.__paths
         except AttributeError:
             self.__paths = [path(p) for p in self if os.path.exists(p)]
             return self.__paths
     
     p = paths = property(get_paths)
 
     def grep(self, pattern, prune = False, field = None):
         """ Return all strings matching 'pattern' (a regex or callable) 
         
         This is case-insensitive. If prune is true, return all items
         NOT matching the pattern.
         
         If field is specified, the match must occur in the specified 
         whitespace-separated field.
                
         Examples::
         
             a.grep( lambda x: x.startswith('C') )
             a.grep('Cha.*log', prune=1)
             a.grep('chm', field=-1)
         """
         
         def match_target(s):
             if field is None:
                 return s
             parts = s.split()
             try:
                 tgt = parts[field]
                 return tgt
             except IndexError:
                 return ""
         
         if isinstance(pattern, basestring):
             pred = lambda x : re.search(pattern, x, re.IGNORECASE)
         else:
             pred = pattern
         if not prune:
             return SList([el for el in self if pred(match_target(el))])
         else:
             return SList([el for el in self if not pred(match_target(el))])
     def fields(self, *fields):
         """ Collect whitespace-separated fields from string list
         
         Allows quick awk-like usage of string lists.
         
         Example data (in var a, created by 'a = !ls -l')::
             -rwxrwxrwx  1 ville None      18 Dec 14  2006 ChangeLog
             drwxrwxrwx+ 6 ville None       0 Oct 24 18:05 IPython        
         
         a.fields(0) is ['-rwxrwxrwx', 'drwxrwxrwx+']
         a.fields(1,0) is ['1 -rwxrwxrwx', '6 drwxrwxrwx+']
         (note the joining by space).
         a.fields(-1) is ['ChangeLog', 'IPython']
         
         IndexErrors are ignored.
         
         Without args, fields() just split()'s the strings.
         """
         if len(fields) == 0:
             return [el.split() for el in self]
         
         res = SList()
         for el in [f.split() for f in self]:
             lineparts = []
 
             for fd in fields:
                 try:
                     lineparts.append(el[fd])
                 except IndexError:
                     pass
             if lineparts:
                 res.append(" ".join(lineparts))
             
         return res        
+    def sort(self,field= None,  nums = False):
+        """ sort by specified fields (see fields())
                 
+        Example::
+            a.sort(1, nums = True)
             
+        Sorts a by second field, in numerical order (so that 21 > 3)
+        
+        """ 
+        
+        #decorate, sort, undecorate
+        if field is not None:
+            dsu = [[SList([line]).fields(field),  line] for line in self]
+        else:
+            dsu = [[line,  line] for line in self]
+        if nums:
+            for i in range(len(dsu)):
+                numstr = "".join([ch for ch in dsu[i][0] if ch.isdigit()])
+                try:
+                    n = int(numstr)
+                except ValueError:
+                    n = 0;
+                dsu[i][0] = n
             
         
+        dsu.sort()
+        return SList([t[1] for t in dsu])
 
 def print_slist(arg):
     """ Prettier (non-repr-like) and more informative printer for SList """
-    print "SList (.p, .n, .l, .s, .grep(), .fields() available). Value:"
+    print "SList (.p, .n, .l, .s, .grep(), .fields(), sort() available):"
+    if hasattr(arg,  'hideonce') and arg.hideonce:
+        arg.hideonce = False
+        return
+        
     nlprint(arg)
     
 print_slist = result_display.when_type(SList)(print_slist)
 
 
 
 #----------------------------------------------------------------------------
 def esc_quotes(strng):
     """Return the input string with single and double quotes escaped out"""
 
     return strng.replace('"','\\"').replace("'","\\'")
 
 #----------------------------------------------------------------------------
 def make_quoted_expr(s):
     """Return string s in appropriate quotes, using raw string if possible.
     
     Effectively this turns string: cd \ao\ao\
     to: r"cd \ao\ao\_"[:-1]
     
     Note the use of raw string and padding at the end to allow trailing backslash.
     
     """
     
     tail = ''
     tailpadding = ''
     raw  = ''
     if "\\" in s:
         raw = 'r'
         if s.endswith('\\'):
             tail = '[:-1]'
             tailpadding = '_'
     if '"' not in s:
         quote = '"'
     elif "'" not in s:
         quote = "'"
     elif '"""' not in s and not s.endswith('"'):
         quote = '"""'
     elif "'''" not in s and not s.endswith("'"):
         quote = "'''"
     else:
         # give up, backslash-escaped string will do
         return '"%s"' % esc_quotes(s)
     res = raw + quote + s + tailpadding + quote + tail
     return res
 
 
 #----------------------------------------------------------------------------
 def raw_input_multi(header='', ps1='==> ', ps2='..> ',terminate_str = '.'):
     """Take multiple lines of input.
 
     A list with each line of input as a separate element is returned when a
     termination string is entered (defaults to a single '.'). Input can also
     terminate via EOF (^D in Unix, ^Z-RET in Windows).
 
     Lines of input which end in \\ are joined into single entries (and a
     secondary continuation prompt is issued as long as the user terminates
     lines with \\). This allows entering very long strings which are still
     meant to be treated as single entities.
     """
 
     try:
         if header:
             header += '\n'
         lines = [raw_input(header + ps1)]
     except EOFError:
         return []
     terminate = [terminate_str]
     try:
         while lines[-1:] != terminate:
             new_line = raw_input(ps1)
             while new_line.endswith('\\'):
                 new_line = new_line[:-1] + raw_input(ps2)
             lines.append(new_line)
                 
         return lines[:-1]  # don't return the termination command
     except EOFError:
         print
         return lines
 
 #----------------------------------------------------------------------------
 def raw_input_ext(prompt='',  ps2='... '):
     """Similar to raw_input(), but accepts extended lines if input ends with \\."""
 
     line = raw_input(prompt)
     while line.endswith('\\'):
         line = line[:-1] + raw_input(ps2)
     return line
 
 #----------------------------------------------------------------------------
 def ask_yes_no(prompt,default=None):
     """Asks a question and returns a boolean (y/n) answer.
 
     If default is given (one of 'y','n'), it is used if the user input is
     empty. Otherwise the question is repeated until an answer is given.
 
     An EOF is treated as the default answer.  If there is no default, an
     exception is raised to prevent infinite loops.
 
     Valid answers are: y/yes/n/no (match is not case sensitive)."""
 
     answers = {'y':True,'n':False,'yes':True,'no':False}
     ans = None
     while ans not in answers.keys():
         try:
             ans = raw_input(prompt+' ').lower()
             if not ans:  # response was an empty string
                 ans = default
         except KeyboardInterrupt:
             pass
         except EOFError:
             if default in answers.keys():
                 ans = default
                 print
             else:
                 raise
             
     return answers[ans]
 
 #----------------------------------------------------------------------------
 def marquee(txt='',width=78,mark='*'):
     """Return the input string centered in a 'marquee'."""
     if not txt:
         return (mark*width)[:width]
     nmark = (width-len(txt)-2)/len(mark)/2
     if nmark < 0: nmark =0
     marks = mark*nmark
     return '%s %s %s' % (marks,txt,marks)
 
 #----------------------------------------------------------------------------
 class EvalDict:
     """
     Emulate a dict which evaluates its contents in the caller's frame.
 
     Usage:
     >>>number = 19
     >>>text = "python"
     >>>print "%(text.capitalize())s %(number/9.0).1f rules!" % EvalDict()
     """
 
     # This version is due to [e-mail] on c.l.py, and is basically a
     # modified (shorter) version of:
     # http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/66018 by
     # Skip Montanaro ([e-mail]).
 
     def __getitem__(self, name):
         frame = sys._getframe(1)
         return eval(name, frame.f_globals, frame.f_locals)
 
 EvalString = EvalDict  # for backwards compatibility
 #----------------------------------------------------------------------------
 def qw(words,flat=0,sep=None,maxsplit=-1):
     """Similar to Perl's qw() operator, but with some more options.
 
     qw(words,flat=0,sep=' ',maxsplit=-1) -> words.split(sep,maxsplit)
 
     words can also be a list itself, and with flat=1, the output will be
     recursively flattened. Examples:
 
     >>> qw('1 2')
     ['1', '2']
     >>> qw(['a b','1 2',['m n','p q']])
     [['a', 'b'], ['1', '2'], [['m', 'n'], ['p', 'q']]]
     >>> qw(['a b','1 2',['m n','p q']],flat=1)
     ['a', 'b', '1', '2', 'm', 'n', 'p', 'q']    """
 
     if type(words) in StringTypes:
         return [word.strip() for word in words.split(sep,maxsplit)
                 if word and not word.isspace() ]
     if flat:
         return flatten(map(qw,words,[1]*len(words)))
     return map(qw,words)
 
 #----------------------------------------------------------------------------
 def qwflat(words,sep=None,maxsplit=-1):
     """Calls qw(words) in flat mode. It's just a convenient shorthand."""
     return qw(words,1,sep,maxsplit)
 
 #----------------------------------------------------------------------------
 def qw_lol(indata):
     """qw_lol('a b') -> [['a','b']],
     otherwise it's just a call to qw().
 
     We need this to make sure the modules_some keys *always* end up as a
     list of lists."""
 
     if type(indata) in StringTypes:
         return [qw(indata)]
     else:
         return qw(indata)
 
 #-----------------------------------------------------------------------------
 def list_strings(arg):
     """Always return a list of strings, given a string or list of strings
     as input."""
 
     if type(arg) in StringTypes: return [arg]
     else: return arg
 
 #----------------------------------------------------------------------------
 def grep(pat,list,case=1):
     """Simple minded grep-like function.
     grep(pat,list) returns occurrences of pat in list, None on failure.
 
     It only does simple string matching, with no support for regexps. Use the
     option case=0 for case-insensitive matching."""
 
     # This is pretty crude. At least it should implement copying only references
     # to the original data in case it's big. Now it copies the data for output.
     out=[]
     if case:
         for term in list:
             if term.find(pat)>-1: out.append(term)
     else:
         lpat=pat.lower()
         for term in list:
             if term.lower().find(lpat)>-1: out.append(term)
 
     if len(out): return out
     else: return None
 
 #----------------------------------------------------------------------------
 def dgrep(pat,*opts):
     """Return grep() on dir()+dir(__builtins__).
 
     A very common use of grep() when working interactively."""
 
     return grep(pat,dir(__main__)+dir(__main__.__builtins__),*opts)
 
 #----------------------------------------------------------------------------
 def idgrep(pat):
     """Case-insensitive dgrep()"""
 
     return dgrep(pat,0)
 
 #----------------------------------------------------------------------------
 def igrep(pat,list):
     """Synonym for case-insensitive grep."""
 
     return grep(pat,list,case=0)
 
 #----------------------------------------------------------------------------
 def indent(str,nspaces=4,ntabs=0):
     """Indent a string a given number of spaces or tabstops.
     
     indent(str,nspaces=4,ntabs=0) -> indent str by ntabs+nspaces.
     """
     if str is None:
         return
     ind = '\t'*ntabs+' '*nspaces
     outstr = '%s%s' % (ind,str.replace(os.linesep,os.linesep+ind))
     if outstr.endswith(os.linesep+ind):
         return outstr[:-len(ind)]
     else:
         return outstr
     
 #-----------------------------------------------------------------------------
 def native_line_ends(filename,backup=1):
     """Convert (in-place) a file to line-ends native to the current OS.
 
     If the optional backup argument is given as false, no backup of the
     original file is left.  """
 
     backup_suffixes = {'posix':'~','dos':'.bak','nt':'.bak','mac':'.bak'}
 
     bak_filename = filename + backup_suffixes[os.name]
 
     original = open(filename).read()
     shutil.copy2(filename,bak_filename)
     try:
         new = open(filename,'wb')
         new.write(os.linesep.join(original.splitlines()))
         new.write(os.linesep) # ALWAYS put an eol at the end of the file
         new.close()
     except:
         os.rename(bak_filename,filename)
     if not backup:
         try:
             os.remove(bak_filename)
         except:
             pass
     
 #----------------------------------------------------------------------------
 def get_pager_cmd(pager_cmd = None):
     """Return a pager command.
 
     Makes some attempts at finding an OS-correct one."""
     
     if os.name == 'posix':
         default_pager_cmd = 'less -r'  # -r for color control sequences
     elif os.name in ['nt','dos']:
         default_pager_cmd = 'type'
 
     if pager_cmd is None:
         try:
             pager_cmd = os.environ['PAGER']
         except:
             pager_cmd = default_pager_cmd
     return pager_cmd
 
 #-----------------------------------------------------------------------------
 def get_pager_start(pager,start):
     """Return the string for paging files with an offset.
 
     This is the '+N' argument which less and more (under Unix) accept.
     """
 
     if pager in ['less','more']:
         if start:
             start_string = '+' + str(start)
         else:
             start_string = ''
     else:
         start_string = ''
     return start_string
 
 #----------------------------------------------------------------------------
 # (X)emacs on W32 doesn't like to be bypassed with msvcrt.getch()
 if os.name == 'nt' and os.environ.get('TERM','dumb') != 'emacs':
     import msvcrt
     def page_more():
         """ Smart pausing between pages
 
         @return:    True if need print more lines, False if quit
         """
         Term.cout.write('---Return to continue, q to quit--- ')
         ans = msvcrt.getch()
         if ans in ("q", "Q"):
             result = False
         else:
             result = True
         Term.cout.write("\b"*37 + " "*37 + "\b"*37)
         return result
 else:
     def page_more():
         ans = raw_input('---Return to continue, q to quit--- ')
         if ans.lower().startswith('q'):
             return False
         else:
             return True
 
 esc_re = re.compile(r"(\x1b[^m]+m)")
 
 def page_dumb(strng,start=0,screen_lines=25):
     """Very dumb 'pager' in Python, for when nothing else works.
 
     Only moves forward, same interface as page(), except for pager_cmd and
     mode."""
 
     out_ln  = strng.splitlines()[start:]
     screens = chop(out_ln,screen_lines-1)
     if len(screens) == 1:
         print >>Term.cout, os.linesep.join(screens[0])
     else:
         last_escape = ""
         for scr in screens[0:-1]:
             hunk = os.linesep.join(scr)
             print >>Term.cout, last_escape + hunk
             if not page_more():
                 return
             esc_list = esc_re.findall(hunk)
             if len(esc_list) > 0:
                 last_escape = esc_list[-1]
         print >>Term.cout, last_escape + os.linesep.join(screens[-1])
 
 #----------------------------------------------------------------------------
 def page(strng,start=0,screen_lines=0,pager_cmd = None):
     """Print a string, piping through a pager after a certain length.
 
     The screen_lines parameter specifies the number of *usable* lines of your
     terminal screen (total lines minus lines you need to reserve to show other
     information).
 
     If you set screen_lines to a number <=0, page() will try to auto-determine
     your screen size and will only use up to (screen_size+screen_lines) for
     printing, paging after that. That is, if you want auto-detection but need
     to reserve the bottom 3 lines of the screen, use screen_lines = -3, and for
     auto-detection without any lines reserved simply use screen_lines = 0.
 
     If a string won't fit in the allowed lines, it is sent through the
     specified pager command. If none given, look for PAGER in the environment,
     and ultimately default to less.
 
     If no system pager works, the string is sent through a 'dumb pager'
     written in python, very simplistic.
     """
 
     # Some routines may auto-compute start offsets incorrectly and pass a
     # negative value.  Offset to 0 for robustness.
     start = max(0,start)
 
     # first, try the hook
     ip = IPython.ipapi.get()
     if ip:
         try:
             ip.IP.hooks.show_in_pager(strng)
             return
         except IPython.ipapi.TryNext:
             pass
         
     # Ugly kludge, but calling curses.initscr() flat out crashes in emacs
     TERM = os.environ.get('TERM','dumb')
     if TERM in ['dumb','emacs'] and os.name != 'nt':
         print strng
         return
     # chop off the topmost part of the string we don't want to see
     str_lines = strng.split(os.linesep)[start:]
     str_toprint = os.linesep.join(str_lines)
     num_newlines = len(str_lines)
     len_str = len(str_toprint)
     
     # Dumb heuristics to guesstimate number of on-screen lines the string
     # takes.  Very basic, but good enough for docstrings in reasonable
     # terminals. If someone later feels like refining it, it's not hard.
     numlines = max(num_newlines,int(len_str/80)+1)
 
     if os.name == "nt":
         screen_lines_def = get_console_size(defaulty=25)[1]
     else:
         screen_lines_def = 25 # default value if we can't auto-determine
 
     # auto-determine screen size
     if screen_lines <= 0:
         if TERM=='xterm':
             use_curses = USE_CURSES
         else:
             # curses causes problems on many terminals other than xterm.
             use_curses = False
         if use_curses:
             # There is a bug in curses, where *sometimes* it fails to properly
             # initialize, and then after the endwin() call is made, the
             # terminal is left in an unusable state.  Rather than trying to
             # check everytime for this (by requesting and comparing termios
             # flags each time), we just save the initial terminal state and
             # unconditionally reset it every time.  It's cheaper than making
             # the checks.
             term_flags = termios.tcgetattr(sys.stdout)
             scr = curses.initscr()
             screen_lines_real,screen_cols = scr.getmaxyx()
             curses.endwin()
             # Restore terminal state in case endwin() didn't.
             termios.tcsetattr(sys.stdout,termios.TCSANOW,term_flags)
             # Now we have what we needed: the screen size in rows/columns
             screen_lines += screen_lines_real
             #print '***Screen size:',screen_lines_real,'lines x',\
             #screen_cols,'columns.' # dbg
         else:
             screen_lines += screen_lines_def
 
     #print 'numlines',numlines,'screenlines',screen_lines  # dbg
     if numlines <= screen_lines :
         #print '*** normal print'  # dbg
         print >>Term.cout, str_toprint
     else:
         # Try to open pager and default to internal one if that fails.
         # All failure modes are tagged as 'retval=1', to match the return
         # value of a failed system command.  If any intermediate attempt
         # sets retval to 1, at the end we resort to our own page_dumb() pager.
         pager_cmd = get_pager_cmd(pager_cmd)
         pager_cmd += ' ' + get_pager_start(pager_cmd,start)
         if os.name == 'nt':
             if pager_cmd.startswith('type'):
                 # The default WinXP 'type' command is failing on complex strings.
                 retval = 1
             else:
                 tmpname = tempfile.mktemp('.txt')
                 tmpfile = file(tmpname,'wt')
                 tmpfile.write(strng)
                 tmpfile.close()
                 cmd = "%s < %s" % (pager_cmd,tmpname)
                 if os.system(cmd):
                   retval = 1
                 else:
                   retval = None
                 os.remove(tmpname)
         else:
             try:
                 retval = None
                 # if I use popen4, things hang. No idea why.
                 #pager,shell_out = os.popen4(pager_cmd)
                 pager = os.popen(pager_cmd,'w')
                 pager.write(strng)
                 pager.close()
                 retval = pager.close()  # success returns None
             except IOError,msg:  # broken pipe when user quits
                 if msg.args == (32,'Broken pipe'):
                     retval = None
                 else:
                     retval = 1
             except OSError:
                 # Other strange problems, sometimes seen in Win2k/cygwin
                 retval = 1
         if retval is not None:
             page_dumb(strng,screen_lines=screen_lines)
 
 #----------------------------------------------------------------------------
 def page_file(fname,start = 0, pager_cmd = None):
     """Page a file, using an optional pager command and starting line.
     """
 
     pager_cmd = get_pager_cmd(pager_cmd)
     pager_cmd += ' ' + get_pager_start(pager_cmd,start)
 
     try:
         if os.environ['TERM'] in ['emacs','dumb']:
             raise EnvironmentError
         xsys(pager_cmd + ' ' + fname)
     except:
         try:
             if start > 0:
                 start -= 1
             page(open(fname).read(),start)
         except:
             print 'Unable to show file',`fname`
 
 
 #----------------------------------------------------------------------------
 def snip_print(str,width = 75,print_full = 0,header = ''):
     """Print a string snipping the midsection to fit in width.
 
     print_full: mode control:
       - 0: only snip long strings
       - 1: send to page() directly.
       - 2: snip long strings and ask for full length viewing with page()
     Return 1 if snipping was necessary, 0 otherwise."""
 
     if print_full == 1:
         page(header+str)
         return 0
 
     print header,
     if len(str) < width:
         print str
         snip = 0
     else:
         whalf = int((width -5)/2)
         print str[:whalf] + ' <...> ' + str[-whalf:]
         snip = 1
     if snip and print_full == 2:
         if raw_input(header+' Snipped. View (y/n)? [N]').lower() == 'y':
             page(str)
     return snip
 
 #****************************************************************************
 # lists, dicts and structures
 
 def belong(candidates,checklist):
     """Check whether a list of items appear in a given list of options.
 
     Returns a list of 1 and 0, one for each candidate given."""
 
     return [x in checklist for x in candidates]
 
 #----------------------------------------------------------------------------
 def uniq_stable(elems):
     """uniq_stable(elems) -> list
 
     Return from an iterable, a list of all the unique elements in the input,
     but maintaining the order in which they first appear.
 
     A naive solution to this problem which just makes a dictionary with the
     elements as keys fails to respect the stability condition, since
     dictionaries are unsorted by nature.
 
     Note: All elements in the input must be valid dictionary keys for this
     routine to work, as it internally uses a dictionary for efficiency
     reasons."""
     
     unique = []
     unique_dict = {}
     for nn in elems:
         if nn not in unique_dict:
             unique.append(nn)
             unique_dict[nn] = None
     return unique
 
 #----------------------------------------------------------------------------
 class NLprinter:
     """Print an arbitrarily nested list, indicating index numbers.
 
     An instance of this class called nlprint is available and callable as a
     function.
     
     nlprint(list,indent=' ',sep=': ') -> prints indenting each level by 'indent'
     and using 'sep' to separate the index from the value. """
 
     def __init__(self):
         self.depth = 0
     
     def __call__(self,lst,pos='',**kw):
         """Prints the nested list numbering levels."""
         kw.setdefault('indent',' ')
         kw.setdefault('sep',': ')
         kw.setdefault('start',0)
         kw.setdefault('stop',len(lst))
         # we need to remove start and stop from kw so they don't propagate
         # into a recursive call for a nested list.
         start = kw['start']; del kw['start']
         stop = kw['stop']; del kw['stop']
         if self.depth == 0 and 'header' in kw.keys():
             print kw['header']
             
         for idx in range(start,stop):
             elem = lst[idx]
             if type(elem)==type([]):
                 self.depth += 1
                 self.__call__(elem,itpl('$pos$idx,'),**kw)
                 self.depth -= 1
             else:
                 printpl(kw['indent']*self.depth+'$pos$idx$kw["sep"]$elem')
 
 nlprint = NLprinter()
 #----------------------------------------------------------------------------
 def all_belong(candidates,checklist):
     """Check whether a list of items ALL appear in a given list of options.
 
     Returns a single 1 or 0 value."""
 
     return 1-(0 in [x in checklist for x in candidates])
 
 #----------------------------------------------------------------------------
 def sort_compare(lst1,lst2,inplace = 1):
     """Sort and compare two lists.
 
     By default it does it in place, thus modifying the lists. Use inplace = 0
     to avoid that (at the cost of temporary copy creation)."""
     if not inplace:
         lst1 = lst1[:]
         lst2 = lst2[:]
     lst1.sort(); lst2.sort()
     return lst1 == lst2
 
 #----------------------------------------------------------------------------
 def mkdict(**kwargs):
     """Return a dict from a keyword list.
 
     It's just syntactic sugar for making ditcionary creation more convenient:
     # the standard way
     >>>data = { 'red' : 1, 'green' : 2, 'blue' : 3 }
     # a cleaner way
     >>>data = dict(red=1, green=2, blue=3)
 
     If you need more than this, look at the Struct() class."""
 
     return kwargs
 
 #----------------------------------------------------------------------------
 def list2dict(lst):
     """Takes a list of (key,value) pairs and turns it into a dict."""
 
     dic = {}
     for k,v in lst: dic[k] = v
     return dic
 
 #----------------------------------------------------------------------------
 def list2dict2(lst,default=''):
     """Takes a list and turns it into a dict.
     Much slower than list2dict, but more versatile. This version can take
     lists with sublists of arbitrary length (including sclars)."""
 
     dic = {}
     for elem in lst:
         if type(elem) in (types.ListType,types.TupleType):
             size = len(elem)
             if  size == 0:
                 pass
             elif size == 1:
                 dic[elem] = default
             else:
                 k,v = elem[0], elem[1:]
                 if len(v) == 1: v = v[0]
                 dic[k] = v
         else:
             dic[elem] = default
     return dic
 
 #----------------------------------------------------------------------------
 def flatten(seq):
     """Flatten a list of lists (NOT recursive, only works for 2d lists)."""
 
     return [x for subseq in seq for x in subseq]
 
 #----------------------------------------------------------------------------
 def get_slice(seq,start=0,stop=None,step=1):
     """Get a slice of a sequence with variable step. Specify start,stop,step."""
     if stop == None:
         stop = len(seq)
     item = lambda i: seq[i]
     return map(item,xrange(start,stop,step))
 
 #----------------------------------------------------------------------------
 def chop(seq,size):
     """Chop a sequence into chunks of the given size."""
     chunk = lambda i: seq[i:i+size]
     return map(chunk,xrange(0,len(seq),size))
 
 #----------------------------------------------------------------------------
 # with is a keyword as of python 2.5, so this function is renamed to withobj
 # from its old 'with' name.
 def with_obj(object, **args):
     """Set multiple attributes for an object, similar to Pascal's with.
 
     Example:
     with_obj(jim,
              born = 1960,
              haircolour = 'Brown',
              eyecolour = 'Green')
 
     Credit: Greg Ewing, in
     http://mail.python.org/pipermail/python-list/2001-May/040703.html.
 
     NOTE: up until IPython 0.7.2, this was called simply 'with', but 'with'
     has become a keyword for Python 2.5, so we had to rename it."""
 
     object.__dict__.update(args)
 
 #----------------------------------------------------------------------------
 def setattr_list(obj,alist,nspace = None):
     """Set a list of attributes for an object taken from a namespace.
 
     setattr_list(obj,alist,nspace) -> sets in obj all the attributes listed in
     alist with their values taken from nspace, which must be a dict (something
     like locals() will often do) If nspace isn't given, locals() of the
     *caller* is used, so in most cases you can omit it.
 
     Note that alist can be given as a string, which will be automatically
     split into a list on whitespace. If given as a list, it must be a list of
     *strings* (the variable names themselves), not of variables."""
 
     # this grabs the local variables from the *previous* call frame -- that is
     # the locals from the function that called setattr_list().
     # - snipped from weave.inline()
     if nspace is None:
         call_frame = sys._getframe().f_back
         nspace = call_frame.f_locals
 
     if type(alist) in StringTypes:
         alist = alist.split()
     for attr in alist:
         val = eval(attr,nspace)
         setattr(obj,attr,val)
 
 #----------------------------------------------------------------------------
 def getattr_list(obj,alist,*args):
     """getattr_list(obj,alist[, default]) -> attribute list.
 
     Get a list of named attributes for an object. When a default argument is
     given, it is returned when the attribute doesn't exist; without it, an
     exception is raised in that case.
 
     Note that alist can be given as a string, which will be automatically
     split into a list on whitespace. If given as a list, it must be a list of
     *strings* (the variable names themselves), not of variables."""
 
     if type(alist) in StringTypes:
         alist = alist.split()
     if args:
         if len(args)==1:
             default = args[0]
             return map(lambda attr: getattr(obj,attr,default),alist)
         else:
             raise ValueError,'getattr_list() takes only one optional argument'
     else:
         return map(lambda attr: getattr(obj,attr),alist)
     
 #----------------------------------------------------------------------------
 def map_method(method,object_list,*argseq,**kw):
     """map_method(method,object_list,*args,**kw) -> list
 
     Return a list of the results of applying the methods to the items of the
     argument sequence(s).  If more than one sequence is given, the method is
     called with an argument list consisting of the corresponding item of each
     sequence. All sequences must be of the same length.
 
     Keyword arguments are passed verbatim to all objects called.
 
     This is Python code, so it's not nearly as fast as the builtin map()."""
 
     out_list = []
     idx = 0
     for object in object_list:
         try:
             handler = getattr(object, method)
         except AttributeError:
             out_list.append(None)
         else:
             if argseq:
                 args = map(lambda lst:lst[idx],argseq)
                 #print 'ob',object,'hand',handler,'ar',args # dbg
                 out_list.append(handler(args,**kw))
             else:
                 out_list.append(handler(**kw))
         idx += 1
     return out_list
 
 #----------------------------------------------------------------------------
 def get_class_members(cls):
     ret = dir(cls)
     if hasattr(cls,'__bases__'):
         for base in cls.__bases__:
             ret.extend(get_class_members(base))
     return ret
 
 #----------------------------------------------------------------------------
 def dir2(obj):
     """dir2(obj) -> list of strings
 
     Extended version of the Python builtin dir(), which does a few extra
     checks, and supports common objects with unusual internals that confuse
     dir(), such as Traits and PyCrust.
 
     This version is guaranteed to return only a list of true strings, whereas
     dir() returns anything that objects inject into themselves, even if they
     are later not really valid for attribute access (many extension libraries
     have such bugs).
     """
     
     # Start building the attribute list via dir(), and then complete it
     # with a few extra special-purpose calls.
     words = dir(obj)
 
     if hasattr(obj,'__class__'):
         words.append('__class__')
         words.extend(get_class_members(obj.__class__))
     #if '__base__' in words: 1/0
 
     # Some libraries (such as traits) may introduce duplicates, we want to
     # track and clean this up if it happens
     may_have_dupes = False
 
     # this is the 'dir' function for objects with Enthought's traits
     if hasattr(obj, 'trait_names'):
         try:
             words.extend(obj.trait_names())
             may_have_dupes = True
         except TypeError:
             # This will happen if `obj` is a class and not an instance.
             pass
 
     # Support for PyCrust-style _getAttributeNames magic method.
     if hasattr(obj, '_getAttributeNames'):
         try:
             words.extend(obj._getAttributeNames())
             may_have_dupes = True
         except TypeError:
             # `obj` is a class and not an instance.  Ignore
             # this error.
             pass
 
     if may_have_dupes:
         # eliminate possible duplicates, as some traits may also
         # appear as normal attributes in the dir() call.
         words = list(set(words))
         words.sort()
 
     # filter out non-string attributes which may be stuffed by dir() calls
     # and poor coding in third-party modules
     return [w for w in words if isinstance(w, basestring)]
 
 #----------------------------------------------------------------------------
 def import_fail_info(mod_name,fns=None):
     """Inform load failure for a module."""
 
     if fns == None:
         warn("Loading of %s failed.\n" % (mod_name,))
     else:
         warn("Loading of %s from %s failed.\n" % (fns,mod_name))
 
 #----------------------------------------------------------------------------
 # Proposed popitem() extension, written as a method
 
 
 class NotGiven: pass
 
 def popkey(dct,key,default=NotGiven):
     """Return dct[key] and delete dct[key].
 
     If default is given, return it if dct[key] doesn't exist, otherwise raise
     KeyError.  """
 
     try:
         val = dct[key]
     except KeyError:
         if default is NotGiven:
             raise
         else:
             return default
     else:
         del dct[key]
         return val
 
 def wrap_deprecated(func, suggest = '<nothing>'):
     def newFunc(*args, **kwargs):
         warnings.warn("Call to deprecated function %s, use %s instead" % 
                       ( func.__name__, suggest),
                       category=DeprecationWarning,
                       stacklevel = 2)
         return func(*args, **kwargs)
     return newFunc
 
 
 def _num_cpus_unix():
     """Return the number of active CPUs on a Unix system."""
     return os.sysconf("SC_NPROCESSORS_ONLN")
 
 
 def _num_cpus_darwin():
     """Return the number of active CPUs on a Darwin system."""
     p = subprocess.Popen(['sysctl','-n','hw.ncpu'],stdout=subprocess.PIPE)
     return p.stdout.read()
 
 
 def _num_cpus_windows():
     """Return the number of active CPUs on a Windows system."""
     return os.environ.get("NUMBER_OF_PROCESSORS")
 
 
 def num_cpus():
    """Return the effective number of CPUs in the system as an integer.
 
    This cross-platform function makes an attempt at finding the total number of
    available CPUs in the system, as returned by various underlying system and
    python calls.
 
    If it can't find a sensible answer, it returns 1 (though an error *may* make
    it return a large positive number that's actually incorrect).
    """
    
    # Many thanks to the Parallel Python project (http://www.parallelpython.com)
    # for the names of the keys we needed to look up for this function.  This
    # code was inspired by their equivalent function.
 
    ncpufuncs = {'Linux':_num_cpus_unix,
                 'Darwin':_num_cpus_darwin,
                 'Windows':_num_cpus_windows,
                 # On Vista, python < 2.5.2 has a bug and returns 'Microsoft'
                 # See http://bugs.python.org/issue1082 for details.
                 'Microsoft':_num_cpus_windows,
                 }
 
    ncpufunc = ncpufuncs.get(platform.system(),
                             # default to unix version (Solaris, AIX, etc)
                             _num_cpus_unix)
 
    try:
        ncpus = max(1,int(ncpufunc()))
    except:
        ncpus = 1
    return ncpus
 
 #*************************** end of file <genutils.py> **********************

IPython/kernel/asyncclient.py

@@ -1,41 +1,41 b''
 # encoding: utf-8
 
 """Asynchronous clients for the IPython controller.
 
 This module has clients for using the various interfaces of the controller 
 in a fully asynchronous manner.  This means that you will need to run the
 Twisted reactor yourself and that all methods of the client classes return
 deferreds to the result.
 
 The main methods are are `get_*_client` and `get_client`.
 """
 
 __docformat__ = "restructuredtext en"
 
 #-------------------------------------------------------------------------------
 #  Copyright (C) 2008  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-------------------------------------------------------------------------------
 
 #-------------------------------------------------------------------------------
 # Imports
 #-------------------------------------------------------------------------------
 
 from IPython.kernel import codeutil
 from IPython.kernel.clientconnector import ClientConnector
 
 # Other things that the user will need
-from IPython.kernel.task import Task
+from IPython.kernel.task import MapTask, StringTask
 from IPython.kernel.error import CompositeError
 
 #-------------------------------------------------------------------------------
 # Code
 #-------------------------------------------------------------------------------
 
 _client_tub = ClientConnector()
 get_multiengine_client = _client_tub.get_multiengine_client
 get_task_client = _client_tub.get_task_client
 get_client = _client_tub.get_client
 

IPython/kernel/client.py

@@ -1,96 +1,96 b''
 # encoding: utf-8
 
 """This module contains blocking clients for the controller interfaces.
 
 Unlike the clients in `asyncclient.py`, the clients in this module are fully
 blocking.  This means that methods on the clients return the actual results
 rather than a deferred to the result.  Also, we manage the Twisted reactor
 for you.  This is done by running the reactor in a thread.
 
 The main classes in this module are:
 
     * MultiEngineClient
     * TaskClient
     * Task
     * CompositeError
 """
 
 __docformat__ = "restructuredtext en"
 
 #-------------------------------------------------------------------------------
 #  Copyright (C) 2008  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-------------------------------------------------------------------------------
 
 #-------------------------------------------------------------------------------
 # Imports
 #-------------------------------------------------------------------------------
 
 import sys
 
 # from IPython.tools import growl
 # growl.start("IPython1 Client")
 
 
 from twisted.internet import reactor
 from IPython.kernel.clientconnector import ClientConnector
 from IPython.kernel.twistedutil import ReactorInThread
 from IPython.kernel.twistedutil import blockingCallFromThread
 
 # These enable various things 
 from IPython.kernel import codeutil
 import IPython.kernel.magic
 
 # Other things that the user will need
-from IPython.kernel.task import Task
+from IPython.kernel.task import MapTask, StringTask
 from IPython.kernel.error import CompositeError
 
 #-------------------------------------------------------------------------------
 # Code
 #-------------------------------------------------------------------------------
 
 _client_tub = ClientConnector()
 
 
 def get_multiengine_client(furl_or_file=''):
     """Get the blocking MultiEngine client.
     
     :Parameters:
         furl_or_file : str
             A furl or a filename containing a furl.  If empty, the
             default furl_file will be used
             
     :Returns:
         The connected MultiEngineClient instance
     """
     client = blockingCallFromThread(_client_tub.get_multiengine_client, 
         furl_or_file)
     return client.adapt_to_blocking_client()
 
 def get_task_client(furl_or_file=''):
     """Get the blocking Task client.
     
     :Parameters:
         furl_or_file : str
             A furl or a filename containing a furl.  If empty, the
             default furl_file will be used
             
     :Returns:
         The connected TaskClient instance
     """
     client = blockingCallFromThread(_client_tub.get_task_client, 
         furl_or_file)
     return client.adapt_to_blocking_client()
 
 
 MultiEngineClient = get_multiengine_client
 TaskClient = get_task_client
 
 
 
 # Now we start the reactor in a thread
 rit = ReactorInThread()
 rit.setDaemon(True)
 rit.start()
  No newline at end of file

IPython/kernel/contexts.py

@@ -1,178 +1,180 b''
 # encoding: utf-8
 # -*- test-case-name: IPython.kernel.test.test_contexts -*-
 """Context managers for IPython.
 
 Python 2.5 introduced the `with` statement, which is based on the context
 manager protocol.  This module offers a few context managers for common cases,
 which can also be useful as templates for writing new, application-specific
 managers.
 """
 
 from __future__ import with_statement
 
 __docformat__ = "restructuredtext en"
 
 #-------------------------------------------------------------------------------
 #  Copyright (C) 2008  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-------------------------------------------------------------------------------
 
 #-------------------------------------------------------------------------------
 # Imports
 #-------------------------------------------------------------------------------
 
 import linecache
 import sys
 
 from twisted.internet.error import ConnectionRefusedError
 
 from IPython.ultraTB import _fixed_getinnerframes, findsource
 from IPython import ipapi
 
 from IPython.kernel import error
 
 #---------------------------------------------------------------------------
 # Utility functions needed by all context managers.
 #---------------------------------------------------------------------------
 
 def remote():
     """Raises a special exception meant to be caught by context managers.
     """
     m = 'Special exception to stop local execution of parallel code.'
     raise error.StopLocalExecution(m)
 
 
 def strip_whitespace(source,require_remote=True):
     """strip leading whitespace from input source.
 
     :Parameters:
     
     """
     remote_mark = 'remote()'
     # Expand tabs to avoid any confusion.
     wsource = [l.expandtabs(4) for l in source]
     # Detect the indentation level
     done = False
     for line in wsource:
         if line.isspace():
             continue
         for col,char in enumerate(line):
             if char != ' ':
                 done = True
                 break
         if done:
             break
     # Now we know how much leading space there is in the code.  Next, we
     # extract up to the first line that has less indentation.
     # WARNINGS: we skip comments that may be misindented, but we do NOT yet
     # detect triple quoted strings that may have flush left text.
     for lno,line in enumerate(wsource):
         lead = line[:col]
         if lead.isspace():
             continue
         else:
             if not lead.lstrip().startswith('#'):
                 break
     # The real 'with' source is up to lno
     src_lines = [l[col:] for l in wsource[:lno+1]]
 
     # Finally, check that the source's first non-comment line begins with the
     # special call 'remote()'
     if require_remote:
         for nline,line in enumerate(src_lines):
             if line.isspace() or line.startswith('#'):
                 continue
             if line.startswith(remote_mark):
                 break
             else:
                 raise ValueError('%s call missing at the start of code' %
                                  remote_mark)
         out_lines = src_lines[nline+1:]
     else:
         # If the user specified that the remote() call wasn't mandatory
         out_lines = src_lines
 
     # src = ''.join(out_lines)  # dbg
     #print 'SRC:\n<<<<<<<>>>>>>>\n%s<<<<<>>>>>>' % src  # dbg
     return ''.join(out_lines)
 
 class RemoteContextBase(object):
     def __init__(self):
         self.ip = ipapi.get()
     
     def _findsource_file(self,f):
         linecache.checkcache()
         s = findsource(f.f_code)
         lnum = f.f_lineno
         wsource = s[0][f.f_lineno:]
         return strip_whitespace(wsource)
 
     def _findsource_ipython(self,f):
         from IPython import ipapi
         self.ip = ipapi.get()
         buf = self.ip.IP.input_hist_raw[-1].splitlines()[1:]
         wsource = [l+'\n' for l in buf ] 
                    
         return strip_whitespace(wsource)
         
     def findsource(self,frame):
         local_ns = frame.f_locals
         global_ns = frame.f_globals
         if frame.f_code.co_filename == '<ipython console>':
             src = self._findsource_ipython(frame)
         else:
             src = self._findsource_file(frame)
         return src
 
     def __enter__(self):
         raise NotImplementedError
         
     def __exit__ (self, etype, value, tb):
         if issubclass(etype,error.StopLocalExecution):
             return True
 
 class RemoteMultiEngine(RemoteContextBase):
     def __init__(self,mec):
         self.mec = mec
         RemoteContextBase.__init__(self)
 
     def __enter__(self):
         src = self.findsource(sys._getframe(1))
         return self.mec.execute(src)
 
 
 # XXX - Temporary hackish testing, we'll move this into proper tests right
-# away
-
-if __name__ == '__main__':
-
-    # XXX - for now, we need a running cluster to be started separately.  The
-    # daemon work is almost finished, and will make much of this unnecessary.
-    from IPython.kernel import client
-    mec = client.MultiEngineClient(('127.0.0.1',10105))
-
-    try:
-        mec.get_ids()
-    except ConnectionRefusedError:
-        import os, time
-        os.system('ipcluster -n 2 &')
-        time.sleep(2)
-        mec = client.MultiEngineClient(('127.0.0.1',10105))
-
-    mec.block = False
-
-    import itertools
-    c = itertools.count()
-
-    parallel = RemoteMultiEngine(mec)
-
-    with parallel as pr:
-        # A comment
-        remote()  # this means the code below only runs remotely
-        print 'Hello remote world'
-        x = 3.14
-        # Comments are OK
-        # Even misindented.
-        y = x+1
+# away.  This has been commented out as it doesn't run under Python 2.4
+# because of the usage of the with statement below.  We need to protect
+# such things with a try:except.
+
+# if __name__ == '__main__':
+# 
+#     # XXX - for now, we need a running cluster to be started separately.  The
+#     # daemon work is almost finished, and will make much of this unnecessary.
+#     from IPython.kernel import client
+#     mec = client.MultiEngineClient(('127.0.0.1',10105))
+# 
+#     try:
+#         mec.get_ids()
+#     except ConnectionRefusedError:
+#         import os, time
+#         os.system('ipcluster -n 2 &')
+#         time.sleep(2)
+#         mec = client.MultiEngineClient(('127.0.0.1',10105))
+# 
+#     mec.block = False
+# 
+#     import itertools
+#     c = itertools.count()
+# 
+#     parallel = RemoteMultiEngine(mec)
+# 
+#     with parallel as pr:
+#         # A comment
+#         remote()  # this means the code below only runs remotely
+#         print 'Hello remote world'
+#         x = 3.14
+#         # Comments are OK
+#         # Even misindented.
+#         y = x+1

IPython/kernel/magic.py

@@ -1,171 +1,171 b''
 # encoding: utf-8
 
 """Magic command interface for interactive parallel work."""
 
 __docformat__ = "restructuredtext en"
 
 #-------------------------------------------------------------------------------
 #  Copyright (C) 2008  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-------------------------------------------------------------------------------
 
 #-------------------------------------------------------------------------------
 # Imports
 #-------------------------------------------------------------------------------
 
 import new
 
 from IPython.iplib import InteractiveShell
 from IPython.Shell import MTInteractiveShell
 
 from twisted.internet.defer import Deferred
 
 
 #-------------------------------------------------------------------------------
 # Definitions of magic functions for use with IPython
 #-------------------------------------------------------------------------------
 
 NO_ACTIVE_CONTROLLER = """
 Error:  No Controller is activated
 Use activate() on a RemoteController object to activate it for magics.
 """
 
 def magic_result(self,parameter_s=''):
     """Print the result of command i on all engines of the active controller.
     
     To activate a controller in IPython, first create it and then call
     the activate() method.
     
     Then you can do the following:
     
     >>> result                                # Print the latest result
     Printing result... 
     [127.0.0.1:0] In [1]: b = 10
     [127.0.0.1:1] In [1]: b = 10
     
     >>> result 0                              # Print result 0
     In [14]: result 0
     Printing result... 
     [127.0.0.1:0] In [0]: a = 5
     [127.0.0.1:1] In [0]: a = 5
     """
     try:
         activeController = __IPYTHON__.activeController
     except AttributeError:
         print NO_ACTIVE_CONTROLLER
     else:
         try:
             index = int(parameter_s)
         except:
             index = None
         result = activeController.get_result(index)
         return result
 
 def magic_px(self,parameter_s=''):
     """Executes the given python command on the active IPython Controller.
     
     To activate a Controller in IPython, first create it and then call
     the activate() method.
     
     Then you can do the following:
      
     >>> %px a = 5       # Runs a = 5 on all nodes
     """
 
     try:
         activeController = __IPYTHON__.activeController
     except AttributeError:
         print NO_ACTIVE_CONTROLLER
     else:
-        print "Executing command on Controller"
+        print "Parallel execution on engines: %s" % activeController.targets
         result = activeController.execute(parameter_s)
         return result
 
 def pxrunsource(self, source, filename="<input>", symbol="single"):
 
     try:
         code = self.compile(source, filename, symbol)
     except (OverflowError, SyntaxError, ValueError):
         # Case 1
         self.showsyntaxerror(filename)
         return None
 
     if code is None:
         # Case 2
         return True
 
     # Case 3
     # Because autopx is enabled, we now call executeAll or disable autopx if
     # %autopx or autopx has been called
     if '_ip.magic("%autopx' in source or '_ip.magic("autopx' in source:
         _disable_autopx(self)
         return False
     else:
         try:
             result = self.activeController.execute(source)
         except:
             self.showtraceback()
         else:
             print result.__repr__()
         return False
         
 def magic_autopx(self, parameter_s=''):
     """Toggles auto parallel mode for the active IPython Controller.
     
     To activate a Controller in IPython, first create it and then call
     the activate() method.
     
     Then you can do the following:
      
     >>> %autopx                    # Now all commands are executed in parallel
     Auto Parallel Enabled
     Type %autopx to disable
     ...
     >>> %autopx                    # Now all commands are locally executed
     Auto Parallel Disabled
     """
   
     if hasattr(self, 'autopx'):
         if self.autopx == True:
             _disable_autopx(self)
         else:
             _enable_autopx(self)
     else:
         _enable_autopx(self)
 
 def _enable_autopx(self):
     """Enable %autopx mode by saving the original runsource and installing 
     pxrunsource.
     """
     try:
         activeController = __IPYTHON__.activeController
     except AttributeError:
         print "No active RemoteController found, use RemoteController.activate()."
     else:
         self._original_runsource = self.runsource
         self.runsource = new.instancemethod(pxrunsource, self, self.__class__)
         self.autopx = True
         print "Auto Parallel Enabled\nType %autopx to disable"
 
 def _disable_autopx(self):
     """Disable %autopx by restoring the original runsource."""
     if hasattr(self, 'autopx'):
         if self.autopx == True:
             self.runsource = self._original_runsource
             self.autopx = False
             print "Auto Parallel Disabled"
             
 # Add the new magic function to the class dict:
 
 InteractiveShell.magic_result = magic_result
 InteractiveShell.magic_px = magic_px
 InteractiveShell.magic_autopx = magic_autopx
 
 # And remove the global name to keep global namespace clean.  Don't worry, the
 # copy bound to IPython stays, we're just removing the global name.
 del magic_result
 del magic_px
 del magic_autopx
 

IPython/kernel/map.py

@@ -1,121 +1,121 b''
 # encoding: utf-8
 
 """Classes used in scattering and gathering sequences.
 
 Scattering consists of partitioning a sequence and sending the various
 pieces to individual nodes in a cluster.
 """
 
 __docformat__ = "restructuredtext en"
 
 #-------------------------------------------------------------------------------
 #  Copyright (C) 2008  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-------------------------------------------------------------------------------
 
 #-------------------------------------------------------------------------------
 # Imports
 #-------------------------------------------------------------------------------
 
 import types
 
 from IPython.genutils import flatten as genutil_flatten
 
 #-------------------------------------------------------------------------------
 # Figure out which array packages are present and their array types
 #-------------------------------------------------------------------------------
 
 arrayModules = []
 try:
     import Numeric
 except ImportError:
     pass
 else:
     arrayModules.append({'module':Numeric, 'type':Numeric.arraytype})
 try:
     import numpy
 except ImportError:
     pass
 else:
     arrayModules.append({'module':numpy, 'type':numpy.ndarray})
 try:
     import numarray
 except ImportError:
     pass
 else:
     arrayModules.append({'module':numarray,
         'type':numarray.numarraycore.NumArray})
 
 class Map:
     """A class for partitioning a sequence using a map."""
             
     def getPartition(self, seq, p, q):
         """Returns the pth partition of q partitions of seq."""
         
         # Test for error conditions here
         if p<0 or p>=q:
           print "No partition exists."
           return
           
         remainder = len(seq)%q
         basesize = len(seq)/q
         hi = []
         lo = []
         for n in range(q):
             if n < remainder:
                 lo.append(n * (basesize + 1))
                 hi.append(lo[-1] + basesize + 1)
             else:
                 lo.append(n*basesize + remainder)
                 hi.append(lo[-1] + basesize)
 
         
         result = seq[lo[p]:hi[p]]
         return result
            
     def joinPartitions(self, listOfPartitions):
         return self.concatenate(listOfPartitions)
                     
     def concatenate(self, listOfPartitions):
         testObject = listOfPartitions[0]
         # First see if we have a known array type
         for m in arrayModules:
             #print m
             if isinstance(testObject, m['type']):
                 return m['module'].concatenate(listOfPartitions)
         # Next try for Python sequence types
         if isinstance(testObject, (types.ListType, types.TupleType)):
             return genutil_flatten(listOfPartitions)
         # If we have scalars, just return listOfPartitions
         return listOfPartitions
 
 class RoundRobinMap(Map):
     """Partitions a sequence in a roun robin fashion.
     
     This currently does not work!
     """
 
     def getPartition(self, seq, p, q):
         return seq[p:len(seq):q]
         #result = []
         #for i in range(p,len(seq),q):
         #    result.append(seq[i])
         #return result
 
     def joinPartitions(self, listOfPartitions):
         #lengths = [len(x) for x in listOfPartitions]
         #maxPartitionLength = len(listOfPartitions[0])
         #numberOfPartitions = len(listOfPartitions)
         #concat = self.concatenate(listOfPartitions)
         #totalLength = len(concat)
         #result = []
         #for i in range(maxPartitionLength):
         #    result.append(concat[i:totalLength:maxPartitionLength])
         return self.concatenate(listOfPartitions)
         
-styles = {'basic':Map}
+dists = {'b':Map}
 
     
     

IPython/kernel/multiengine.py

@@ -1,780 +1,753 b''
 # encoding: utf-8
 # -*- test-case-name: IPython.kernel.test.test_multiengine -*-
 
 """Adapt the IPython ControllerServer to IMultiEngine.
 
 This module provides classes that adapt a ControllerService to the 
 IMultiEngine interface.  This interface is a basic interactive interface
 for working with a set of engines where it is desired to have explicit 
 access to each registered engine.  
 
 The classes here are exposed to the network in files like:
 
 * multienginevanilla.py
 * multienginepb.py
 """
 
 __docformat__ = "restructuredtext en"
 
 #-------------------------------------------------------------------------------
 #  Copyright (C) 2008  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-------------------------------------------------------------------------------
 
 #-------------------------------------------------------------------------------
 # Imports
 #-------------------------------------------------------------------------------
 
 from new import instancemethod
 from types import FunctionType
 
 from twisted.application import service
 from twisted.internet import defer, reactor
 from twisted.python import log, components, failure
 from zope.interface import Interface, implements, Attribute
 
 from IPython.tools import growl
 from IPython.kernel.util import printer
 from IPython.kernel.twistedutil import gatherBoth
 from IPython.kernel import map as Map
 from IPython.kernel import error
 from IPython.kernel.pendingdeferred import PendingDeferredManager, two_phase
 from IPython.kernel.controllerservice import \
     ControllerAdapterBase, \
     ControllerService, \
     IControllerBase
 
 
 #-------------------------------------------------------------------------------
 # Interfaces for the MultiEngine representation of a controller
 #-------------------------------------------------------------------------------
 
 class IEngineMultiplexer(Interface):
     """Interface to multiple engines implementing IEngineCore/Serialized/Queued.
     
     This class simply acts as a multiplexer of methods that are in the 
     various IEngines* interfaces.  Thus the methods here are jut like those 
     in the IEngine* interfaces, but with an extra first argument, targets.  
     The targets argument can have the following forms:
     
     * targets = 10            # Engines are indexed by ints
     * targets = [0,1,2,3]     # A list of ints
     * targets = 'all'         # A string to indicate all targets
     
     If targets is bad in any way, an InvalidEngineID will be raised.  This
     includes engines not being registered.
     
     All IEngineMultiplexer multiplexer methods must return a Deferred to a list 
     with length equal to the number of targets.  The elements of the list will 
     correspond to the return of the corresponding IEngine method.
     
     Failures are aggressive, meaning that if an action fails for any target, 
     the overall action will fail immediately with that Failure.
     
     :Parameters:
         targets : int, list of ints, or 'all'
             Engine ids the action will apply to.
     
     :Returns: Deferred to a list of results for each engine.
     
     :Exception:
         InvalidEngineID
             If the targets argument is bad or engines aren't registered.
         NoEnginesRegistered
             If there are no engines registered and targets='all'
     """
         
     #---------------------------------------------------------------------------
     # Mutiplexed methods
     #---------------------------------------------------------------------------
      
     def execute(lines, targets='all'):
         """Execute lines of Python code on targets.
               
         See the class docstring for information about targets and possible
         exceptions this method can raise.
               
         :Parameters:
             lines : str
                 String of python code to be executed on targets.
         """
             
     def push(namespace, targets='all'):
         """Push dict namespace into the user's namespace on targets.
         
         See the class docstring for information about targets and possible
         exceptions this method can raise.
         
         :Parameters:
             namspace : dict
                 Dict of key value pairs to be put into the users namspace.
         """
         
     def pull(keys, targets='all'):
         """Pull values out of the user's namespace on targets by keys.
         
         See the class docstring for information about targets and possible
         exceptions this method can raise.
         
         :Parameters:
             keys : tuple of strings
                 Sequence of keys to be pulled from user's namespace.
         """
           
     def push_function(namespace, targets='all'):
         """"""
         
     def pull_function(keys, targets='all'):
         """"""
                 
     def get_result(i=None, targets='all'):
         """Get the result for command i from targets.
         
         See the class docstring for information about targets and possible
         exceptions this method can raise.
         
         :Parameters:
             i : int or None
                 Command index or None to indicate most recent command.                
         """
         
     def reset(targets='all'):
         """Reset targets.
         
         This clears the users namespace of the Engines, but won't cause
         modules to be reloaded.
         """
         
     def keys(targets='all'):
         """Get variable names defined in user's namespace on targets."""
         
     def kill(controller=False, targets='all'):
         """Kill the targets Engines and possibly the controller.
         
         :Parameters:
             controller : boolean
                 Should the controller be killed as well.  If so all the 
                 engines will be killed first no matter what targets is.
         """
         
     def push_serialized(namespace, targets='all'):
         """Push a namespace of Serialized objects to targets.
         
         :Parameters:
             namespace : dict
                 A dict whose keys are the variable names and whose values
                 are serialized version of the objects.
         """
         
     def pull_serialized(keys, targets='all'):
         """Pull Serialized objects by keys from targets.
         
         :Parameters:
             keys : tuple of strings
                 Sequence of variable names to pull as serialized objects.
         """
         
     def clear_queue(targets='all'):
         """Clear the queue of pending command for targets."""
         
     def queue_status(targets='all'):
         """Get the status of the queue on the targets."""
     
     def set_properties(properties, targets='all'):
         """set properties by key and value"""
     
     def get_properties(keys=None, targets='all'):
         """get a list of properties by `keys`, if no keys specified, get all"""
     
     def del_properties(keys, targets='all'):
         """delete properties by `keys`"""
     
     def has_properties(keys, targets='all'):
         """get a list of bool values for whether `properties` has `keys`"""
     
     def clear_properties(targets='all'):
         """clear the properties dict"""
 
 
 class IMultiEngine(IEngineMultiplexer):
     """A controller that exposes an explicit interface to all of its engines.
     
     This is the primary inteface for interactive usage.
     """
     
     def get_ids():
         """Return list of currently registered ids.
                 
         :Returns:  A Deferred to a list of registered engine ids.
         """
 
 
 
 #-------------------------------------------------------------------------------
 # Implementation of the core MultiEngine classes
 #-------------------------------------------------------------------------------
 
 class MultiEngine(ControllerAdapterBase):
     """The representation of a ControllerService as a IMultiEngine.
     
     Although it is not implemented currently, this class would be where a
     client/notification API is implemented.  It could inherit from something
     like results.NotifierParent and then use the notify method to send
     notifications.
     """
     
     implements(IMultiEngine)
     
     def __init(self, controller):
         ControllerAdapterBase.__init__(self, controller)
     
     #---------------------------------------------------------------------------
     # Helper methods
     #---------------------------------------------------------------------------
     
     def engineList(self, targets):
         """Parse the targets argument into a list of valid engine objects.
         
         :Parameters:
             targets : int, list of ints or 'all'
                 The targets argument to be parsed.
                 
         :Returns: List of engine objects.
         
         :Exception:
             InvalidEngineID
                 If targets is not valid or if an engine is not registered.
         """
         if isinstance(targets, int):
             if targets not in self.engines.keys():
                 log.msg("Engine with id %i is not registered" % targets)
                 raise error.InvalidEngineID("Engine with id %i is not registered" % targets)
             else: 
                 return [self.engines[targets]]
         elif isinstance(targets, (list, tuple)):
             for id in targets:
                 if id not in self.engines.keys():
                     log.msg("Engine with id %r is not registered" % id)
                     raise error.InvalidEngineID("Engine with id %r is not registered" % id)  
             return map(self.engines.get, targets)
         elif targets == 'all':
             eList = self.engines.values()
             if len(eList) == 0:
                 msg = """There are no engines registered.
                      Check the logs in ~/.ipython/log if you think there should have been."""
                 raise error.NoEnginesRegistered(msg)
             else:
                 return eList
         else:
             raise error.InvalidEngineID("targets argument is not an int, list of ints or 'all': %r"%targets)
     
     def _performOnEngines(self, methodName, *args, **kwargs):
         """Calls a method on engines and returns deferred to list of results.
         
         :Parameters:
             methodName : str
                 Name of the method to be called.
             targets : int, list of ints, 'all'
                 The targets argument to be parsed into a list of engine objects.
             args
                 The positional keyword arguments to be passed to the engines.
             kwargs
                 The keyword arguments passed to the method
                 
         :Returns: List of deferreds to the results on each engine
         
         :Exception:
             InvalidEngineID
                 If the targets argument is bad in any way.
             AttributeError
                 If the method doesn't exist on one of the engines.
         """
         targets = kwargs.pop('targets')
         log.msg("Performing %s on %r" % (methodName, targets))
         # log.msg("Performing %s(%r, %r) on %r" % (methodName, args, kwargs, targets))
         # This will and should raise if targets is not valid!
         engines = self.engineList(targets)
         dList = []
         for e in engines:
             meth = getattr(e, methodName, None)
             if meth is not None:
                 dList.append(meth(*args, **kwargs))
             else:
                 raise AttributeError("Engine %i does not have method %s" % (e.id, methodName))
         return dList
         
     def _performOnEnginesAndGatherBoth(self, methodName, *args, **kwargs):
         """Called _performOnEngines and wraps result/exception into deferred."""
         try:
             dList = self._performOnEngines(methodName, *args, **kwargs)
         except (error.InvalidEngineID, AttributeError, KeyError, error.NoEnginesRegistered):
             return defer.fail(failure.Failure())
         else:
             # Having fireOnOneErrback is causing problems with the determinacy
             # of the system.  Basically, once a single engine has errbacked, this
             # method returns.  In some cases, this will cause client to submit
             # another command.  Because the previous command is still running
             # on some engines, this command will be queued.  When those commands
             # then errback, the second command will raise QueueCleared.  Ahhh!
             d = gatherBoth(dList, 
                            fireOnOneErrback=0,
                            consumeErrors=1,
                            logErrors=0)
             d.addCallback(error.collect_exceptions, methodName)
             return d
               
     #---------------------------------------------------------------------------
     # General IMultiEngine methods
     #---------------------------------------------------------------------------
     
     def get_ids(self):
         return defer.succeed(self.engines.keys())
     
     #---------------------------------------------------------------------------
     # IEngineMultiplexer methods
     #---------------------------------------------------------------------------
         
     def execute(self, lines, targets='all'):
         return self._performOnEnginesAndGatherBoth('execute', lines, targets=targets)
     
     def push(self, ns, targets='all'):
         return self._performOnEnginesAndGatherBoth('push', ns, targets=targets)
         
     def pull(self, keys, targets='all'):
         return self._performOnEnginesAndGatherBoth('pull', keys, targets=targets)
     
     def push_function(self, ns, targets='all'):
         return self._performOnEnginesAndGatherBoth('push_function', ns, targets=targets)
         
     def pull_function(self, keys, targets='all'):
         return self._performOnEnginesAndGatherBoth('pull_function', keys, targets=targets)
     
     def get_result(self, i=None, targets='all'):
         return self._performOnEnginesAndGatherBoth('get_result', i, targets=targets)
     
     def reset(self, targets='all'):
         return self._performOnEnginesAndGatherBoth('reset', targets=targets)
     
     def keys(self, targets='all'):
         return self._performOnEnginesAndGatherBoth('keys', targets=targets)
     
     def kill(self, controller=False, targets='all'):
         if controller:
             targets = 'all'
         d = self._performOnEnginesAndGatherBoth('kill', targets=targets)
         if controller:
             log.msg("Killing controller")
             d.addCallback(lambda _: reactor.callLater(2.0, reactor.stop))
             # Consume any weird stuff coming back
             d.addBoth(lambda _: None)
         return d
     
     def push_serialized(self, namespace, targets='all'):
         for k, v in namespace.iteritems():
             log.msg("Pushed object %s is %f MB" % (k, v.getDataSize()))
         d = self._performOnEnginesAndGatherBoth('push_serialized', namespace, targets=targets)      
         return d
         
     def pull_serialized(self, keys, targets='all'):
         try:
             dList = self._performOnEngines('pull_serialized', keys, targets=targets)
         except (error.InvalidEngineID, AttributeError, error.NoEnginesRegistered):
             return defer.fail(failure.Failure())
         else:
             for d in dList:
                 d.addCallback(self._logSizes)
             d = gatherBoth(dList, 
                            fireOnOneErrback=0,
                            consumeErrors=1,
                            logErrors=0)
             d.addCallback(error.collect_exceptions, 'pull_serialized')
             return d  
                               
     def _logSizes(self, listOfSerialized):
         if isinstance(listOfSerialized, (list, tuple)):
             for s in listOfSerialized:
                 log.msg("Pulled object is %f MB" % s.getDataSize())
         else:
             log.msg("Pulled object is %f MB" % listOfSerialized.getDataSize())
         return listOfSerialized
     
     def clear_queue(self, targets='all'):
         return self._performOnEnginesAndGatherBoth('clear_queue', targets=targets)         
     
     def queue_status(self, targets='all'):
         log.msg("Getting queue status on %r" % targets)
         try:
             engines = self.engineList(targets)
         except (error.InvalidEngineID, AttributeError, error.NoEnginesRegistered):
             return defer.fail(failure.Failure())            
         else:
             dList = []
             for e in engines:
                 dList.append(e.queue_status().addCallback(lambda s:(e.id, s)))
             d = gatherBoth(dList, 
                            fireOnOneErrback=0,
                            consumeErrors=1,
                            logErrors=0)
             d.addCallback(error.collect_exceptions, 'queue_status')
             return d 
     
     def get_properties(self, keys=None, targets='all'):
         log.msg("Getting properties on %r" % targets)
         try:
             engines = self.engineList(targets)
         except (error.InvalidEngineID, AttributeError, error.NoEnginesRegistered):
             return defer.fail(failure.Failure())            
         else:
             dList = [e.get_properties(keys) for e in engines]
             d = gatherBoth(dList, 
                            fireOnOneErrback=0,
                            consumeErrors=1,
                            logErrors=0)
             d.addCallback(error.collect_exceptions, 'get_properties')
             return d
     
     def set_properties(self, properties, targets='all'):
         log.msg("Setting properties on %r" % targets)
         try:
             engines = self.engineList(targets)
         except (error.InvalidEngineID, AttributeError, error.NoEnginesRegistered):
             return defer.fail(failure.Failure())            
         else:
             dList = [e.set_properties(properties) for e in engines]
             d = gatherBoth(dList, 
                            fireOnOneErrback=0,
                            consumeErrors=1,
                            logErrors=0)
             d.addCallback(error.collect_exceptions, 'set_properties')
             return d
     
     def has_properties(self, keys, targets='all'):
         log.msg("Checking properties on %r" % targets)
         try:
             engines = self.engineList(targets)
         except (error.InvalidEngineID, AttributeError, error.NoEnginesRegistered):
             return defer.fail(failure.Failure())            
         else:
             dList = [e.has_properties(keys) for e in engines]
             d = gatherBoth(dList, 
                            fireOnOneErrback=0,
                            consumeErrors=1,
                            logErrors=0)
             d.addCallback(error.collect_exceptions, 'has_properties')
             return d
     
     def del_properties(self, keys, targets='all'):
         log.msg("Deleting properties on %r" % targets)
         try:
             engines = self.engineList(targets)
         except (error.InvalidEngineID, AttributeError, error.NoEnginesRegistered):
             return defer.fail(failure.Failure())            
         else:
             dList = [e.del_properties(keys) for e in engines]
             d = gatherBoth(dList, 
                            fireOnOneErrback=0,
                            consumeErrors=1,
                            logErrors=0)
             d.addCallback(error.collect_exceptions, 'del_properties')
             return d
     
     def clear_properties(self, targets='all'):
         log.msg("Clearing properties on %r" % targets)
         try:
             engines = self.engineList(targets)
         except (error.InvalidEngineID, AttributeError, error.NoEnginesRegistered):
             return defer.fail(failure.Failure())            
         else:
             dList = [e.clear_properties() for e in engines]
             d = gatherBoth(dList, 
                            fireOnOneErrback=0,
                            consumeErrors=1,
                            logErrors=0)
             d.addCallback(error.collect_exceptions, 'clear_properties')
             return d
 
 
 components.registerAdapter(MultiEngine,
                            IControllerBase, 
                            IMultiEngine)
 
 
 #-------------------------------------------------------------------------------
 # Interfaces for the Synchronous MultiEngine
 #-------------------------------------------------------------------------------
 
 class ISynchronousEngineMultiplexer(Interface):
     pass
 
 
 class ISynchronousMultiEngine(ISynchronousEngineMultiplexer):
     """Synchronous, two-phase version of IMultiEngine.
     
     Methods in this interface are identical to those of IMultiEngine, but they
     take one additional argument:
     
     execute(lines, targets='all') -> execute(lines, targets='all, block=True)
     
     :Parameters:
         block : boolean
             Should the method return a deferred to a deferredID or the 
             actual result.  If block=False a deferred to a deferredID is 
             returned and the user must call `get_pending_deferred` at a later
             point.  If block=True, a deferred to the actual result comes back.
     """
     def get_pending_deferred(deferredID, block=True):
         """"""
     
     def clear_pending_deferreds():
         """"""
 
 
 #-------------------------------------------------------------------------------
 # Implementation of the Synchronous MultiEngine
 #-------------------------------------------------------------------------------
 
 class SynchronousMultiEngine(PendingDeferredManager):
     """Adapt an `IMultiEngine` -> `ISynchronousMultiEngine`
     
     Warning, this class uses a decorator that currently uses **kwargs.  
     Because of this block must be passed as a kwarg, not positionally.
     """
     
     implements(ISynchronousMultiEngine)
     
     def __init__(self, multiengine):
         self.multiengine = multiengine
         PendingDeferredManager.__init__(self)
     
     #---------------------------------------------------------------------------
     # Decorated pending deferred methods
     #---------------------------------------------------------------------------
     
     @two_phase
     def execute(self, lines, targets='all'):
         d = self.multiengine.execute(lines, targets)
         return d
     
     @two_phase
     def push(self, namespace, targets='all'):
         return self.multiengine.push(namespace, targets)
     
     @two_phase
     def pull(self, keys, targets='all'):
         d = self.multiengine.pull(keys, targets)
         return d
     
     @two_phase
     def push_function(self, namespace, targets='all'):
         return self.multiengine.push_function(namespace, targets)
     
     @two_phase
     def pull_function(self, keys, targets='all'):
         d = self.multiengine.pull_function(keys, targets)
         return d
     
     @two_phase
     def get_result(self, i=None, targets='all'):
         return self.multiengine.get_result(i, targets='all')
     
     @two_phase
     def reset(self, targets='all'):
         return self.multiengine.reset(targets)
     
     @two_phase
     def keys(self, targets='all'):
         return self.multiengine.keys(targets)
     
     @two_phase
     def kill(self, controller=False, targets='all'):
         return self.multiengine.kill(controller, targets)
     
     @two_phase
     def push_serialized(self, namespace, targets='all'):
         return self.multiengine.push_serialized(namespace, targets)
     
     @two_phase
     def pull_serialized(self, keys, targets='all'):
         return self.multiengine.pull_serialized(keys, targets)
     
     @two_phase
     def clear_queue(self, targets='all'):
         return self.multiengine.clear_queue(targets)
     
     @two_phase
     def queue_status(self, targets='all'):
         return self.multiengine.queue_status(targets)
     
     @two_phase
     def set_properties(self, properties, targets='all'):
         return self.multiengine.set_properties(properties, targets)
     
     @two_phase
     def get_properties(self, keys=None, targets='all'):
         return self.multiengine.get_properties(keys, targets)
     
     @two_phase
     def has_properties(self, keys, targets='all'):
         return self.multiengine.has_properties(keys, targets)
     
     @two_phase
     def del_properties(self, keys, targets='all'):
         return self.multiengine.del_properties(keys, targets)
     
     @two_phase
     def clear_properties(self, targets='all'):
         return self.multiengine.clear_properties(targets)
     
     #---------------------------------------------------------------------------
     # IMultiEngine methods
     #---------------------------------------------------------------------------
     
     def get_ids(self):
         """Return a list of registered engine ids.
         
         Never use the two phase block/non-block stuff for this.
         """
         return self.multiengine.get_ids()
 
 
 components.registerAdapter(SynchronousMultiEngine, IMultiEngine, ISynchronousMultiEngine)
 
 
 #-------------------------------------------------------------------------------
 # Various high-level interfaces that can be used as MultiEngine mix-ins
 #-------------------------------------------------------------------------------
 
 #-------------------------------------------------------------------------------
 # IMultiEngineCoordinator
 #-------------------------------------------------------------------------------
 
 class IMultiEngineCoordinator(Interface):
     """Methods that work on multiple engines explicitly."""
        
-    def scatter(key, seq, style='basic', flatten=False, targets='all'):
-        """Partition and distribute a sequence to targets.
+    def scatter(key, seq, dist='b', flatten=False, targets='all'):
+        """Partition and distribute a sequence to targets."""
         
-        :Parameters:
-            key : str
-                The variable name to call the scattered sequence.
-            seq : list, tuple, array
-                The sequence to scatter.  The type should be preserved.
-            style : string
-                A specification of how the sequence is partitioned.  Currently 
-                only 'basic' is implemented.
-            flatten : boolean
-                Should single element sequences be converted to scalars.
-        """
+    def gather(key, dist='b', targets='all'):
+        """Gather object key from targets."""
         
-    def gather(key, style='basic', targets='all'):
-        """Gather object key from targets.
-    
-        :Parameters:
-            key : string
-                The name of a sequence on the targets to gather.
-            style : string
-                A specification of how the sequence is partitioned.  Currently 
-                only 'basic' is implemented.                
+    def raw_map(func, seqs, dist='b', targets='all'):
         """
+        A parallelized version of Python's builtin `map` function.
     
-    def map(func, seq, style='basic', targets='all'):
-        """A parallelized version of Python's builtin map.
+        This has a slightly different syntax than the builtin `map`.
+        This is needed because we need to have keyword arguments and thus
+        can't use *args to capture all the sequences.  Instead, they must
+        be passed in a list or tuple.
         
-        This function implements the following pattern:
+        The equivalence is:
         
-        1. The sequence seq is scattered to the given targets.
-        2. map(functionSource, seq) is called on each engine.
-        3. The resulting sequences are gathered back to the local machine.
+        raw_map(func, seqs) -> map(func, seqs[0], seqs[1], ...)
                 
-        :Parameters:
-            targets : int, list or 'all'
-                The engine ids the action will apply to.  Call `get_ids` to see
-                a list of currently available engines.
-            func : str, function
-                An actual function object or a Python string that names a 
-                callable defined on the engines.
-            seq : list, tuple or numpy array
-                The local sequence to be scattered.
-            style : str
-                Only 'basic' is supported for now.
-                
-        :Returns: A list of len(seq) with functionSource called on each element
-        of seq.
-                
-        Example
-        =======
-        
-        >>> rc.mapAll('lambda x: x*x', range(10000))
-        [0,2,4,9,25,36,...]
+        Most users will want to use parallel functions or the `mapper`
+        and `map` methods for an API that follows that of the builtin
+        `map`.
         """
 
 
 class ISynchronousMultiEngineCoordinator(IMultiEngineCoordinator):
     """Methods that work on multiple engines explicitly."""
-    pass
+
+    def scatter(key, seq, dist='b', flatten=False, targets='all', block=True):
+        """Partition and distribute a sequence to targets."""
+        
+    def gather(key, dist='b', targets='all', block=True):
+        """Gather object key from targets"""
+    
+    def raw_map(func, seqs, dist='b', targets='all', block=True):
+        """
+        A parallelized version of Python's builtin map.
+        
+        This has a slightly different syntax than the builtin `map`.
+        This is needed because we need to have keyword arguments and thus
+        can't use *args to capture all the sequences.  Instead, they must
+        be passed in a list or tuple.
+        
+        raw_map(func, seqs) -> map(func, seqs[0], seqs[1], ...)
+        
+        Most users will want to use parallel functions or the `mapper`
+        and `map` methods for an API that follows that of the builtin
+        `map`.
+        """
 
 
 #-------------------------------------------------------------------------------
 # IMultiEngineExtras
 #-------------------------------------------------------------------------------
 
 class IMultiEngineExtras(Interface):
     
-    def zip_pull(targets, *keys):
-        """Pull, but return results in a different format from `pull`.
+    def zip_pull(targets, keys):
+        """
+        Pull, but return results in a different format from `pull`.
         
         This method basically returns zip(pull(targets, *keys)), with a few 
         edge cases handled differently.  Users of chainsaw will find this format 
         familiar.
-        
-        :Parameters:
-            targets : int, list or 'all'
-                The engine ids the action will apply to.  Call `get_ids` to see
-                a list of currently available engines.
-            keys: list or tuple of str
-                A list of variable names as string of the Python objects to be pulled
-                back to the client.
-        
-        :Returns: A list of pulled Python objects for each target.
         """
     
     def run(targets, fname):
-        """Run a .py file on targets.
-        
-        :Parameters:
-            targets : int, list or 'all'
-                The engine ids the action will apply to.  Call `get_ids` to see
-                a list of currently available engines.
-            fname : str
-                The filename of a .py file on the local system to be sent to and run
-                on the engines.
-            block : boolean
-                Should I block or not.  If block=True, wait for the action to
-                complete and return the result.  If block=False, return a
-                `PendingResult` object that can be used to later get the
-                result.  If block is not specified, the block attribute 
-                will be used instead. 
-        """
+        """Run a .py file on targets."""
 
 
 class ISynchronousMultiEngineExtras(IMultiEngineExtras):
-    pass
+    def zip_pull(targets, keys, block=True):
+        """
+        Pull, but return results in a different format from `pull`.
+        
+        This method basically returns zip(pull(targets, *keys)), with a few 
+        edge cases handled differently.  Users of chainsaw will find this format 
+        familiar.
+        """
 
+    def run(targets, fname, block=True):
+        """Run a .py file on targets."""
 
 #-------------------------------------------------------------------------------
 # The full MultiEngine interface
 #-------------------------------------------------------------------------------
 
 class IFullMultiEngine(IMultiEngine, 
     IMultiEngineCoordinator, 
     IMultiEngineExtras):
     pass
 
 
 class IFullSynchronousMultiEngine(ISynchronousMultiEngine, 
     ISynchronousMultiEngineCoordinator,
     ISynchronousMultiEngineExtras):
     pass
 

IPython/kernel/multiengineclient.py

@@ -1,840 +1,896 b''
 # encoding: utf-8
 # -*- test-case-name: IPython.kernel.test.test_multiengineclient -*-
 
 """General Classes for IMultiEngine clients."""
 
 __docformat__ = "restructuredtext en"
 
 #-------------------------------------------------------------------------------
 #  Copyright (C) 2008  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-------------------------------------------------------------------------------
 
 #-------------------------------------------------------------------------------
 # Imports
 #-------------------------------------------------------------------------------
 
 import sys
 import cPickle as pickle
 from types import FunctionType
 import linecache
 
 from twisted.internet import reactor
 from twisted.python import components, log
 from twisted.python.failure import Failure
 from zope.interface import Interface, implements, Attribute
 
 from IPython.ColorANSI import TermColors
 
 from IPython.kernel.twistedutil import blockingCallFromThread
 from IPython.kernel import error
 from IPython.kernel.parallelfunction import ParallelFunction
+from IPython.kernel.mapper import (
+    MultiEngineMapper, 
+    IMultiEngineMapperFactory,
+    IMapper
+)
 from IPython.kernel import map as Map
 from IPython.kernel import multiengine as me
 from IPython.kernel.multiengine import (IFullMultiEngine,
     IFullSynchronousMultiEngine)
 
 
 #-------------------------------------------------------------------------------
 # Pending Result things
 #-------------------------------------------------------------------------------
 
 class IPendingResult(Interface):
     """A representation of a result that is pending.
     
     This class is similar to Twisted's `Deferred` object, but is designed to be
     used in a synchronous context.
     """
     
     result_id=Attribute("ID of the deferred on the other side")
     client=Attribute("A client that I came from")
     r=Attribute("An attribute that is a property that calls and returns get_result")
     
     def get_result(default=None, block=True):
         """
         Get a result that is pending.
                 
         :Parameters:
             default
                 The value to return if the result is not ready.
             block : boolean
                 Should I block for the result.
                 
         :Returns: The actual result or the default value.
         """
         
     def add_callback(f, *args, **kwargs):
         """
         Add a callback that is called with the result.
         
         If the original result is foo, adding a callback will cause
         f(foo, *args, **kwargs) to be returned instead.  If multiple
         callbacks are registered, they are chained together: the result of
         one is passed to the next and so on.  
         
         Unlike Twisted's Deferred object, there is no errback chain.  Thus
         any exception raised will not be caught and handled.  User must 
         catch these by hand when calling `get_result`.
         """
 
 
 class PendingResult(object):
     """A representation of a result that is not yet ready.
     
     A user should not create a `PendingResult` instance by hand.
     
     Methods
     =======
     
     * `get_result`
     * `add_callback`
     
     Properties
     ==========
     * `r`
     """
     
     def __init__(self, client, result_id):
         """Create a PendingResult with a result_id and a client instance.
         
         The client should implement `_getPendingResult(result_id, block)`.
         """
         self.client = client
         self.result_id = result_id
         self.called = False
         self.raised = False
         self.callbacks = []
         
     def get_result(self, default=None, block=True):
         """Get a result that is pending.
                 
         This method will connect to an IMultiEngine adapted controller
         and see if the result is ready.  If the action triggers an exception
         raise it and record it.  This method records the result/exception once it is 
         retrieved.  Calling `get_result` again will get this cached result or will
         re-raise the exception.  The .r attribute is a property that calls
         `get_result` with block=True.
         
         :Parameters:
             default
                 The value to return if the result is not ready.
             block : boolean
                 Should I block for the result.
                 
         :Returns: The actual result or the default value.
         """
         
         if self.called:
             if self.raised:
                 raise self.result[0], self.result[1], self.result[2]
             else:
                 return self.result
         try:
             result = self.client.get_pending_deferred(self.result_id, block)
         except error.ResultNotCompleted:
             return default
         except:
             # Reraise other error, but first record them so they can be reraised
             # later if .r or get_result is called again.
             self.result = sys.exc_info()
             self.called = True
             self.raised = True
             raise
         else:
             for cb in self.callbacks:
                 result = cb[0](result, *cb[1], **cb[2])
             self.result = result
             self.called = True
             return result
         
     def add_callback(self, f, *args, **kwargs):
         """Add a callback that is called with the result.
         
         If the original result is result, adding a callback will cause
         f(result, *args, **kwargs) to be returned instead.  If multiple
         callbacks are registered, they are chained together: the result of
         one is passed to the next and so on.  
         
         Unlike Twisted's Deferred object, there is no errback chain.  Thus
         any exception raised will not be caught and handled.  User must 
         catch these by hand when calling `get_result`.
         """
         assert callable(f)
         self.callbacks.append((f, args, kwargs))
         
     def __cmp__(self, other):
         if self.result_id < other.result_id:
             return -1
         else:
             return 1
             
     def _get_r(self):
         return self.get_result(block=True)
     
     r = property(_get_r)
     """This property is a shortcut to a `get_result(block=True)`."""
 
 
 #-------------------------------------------------------------------------------
 # Pretty printing wrappers for certain lists
 #-------------------------------------------------------------------------------    
     
 class ResultList(list):
     """A subclass of list that pretty prints the output of `execute`/`get_result`."""
     
     def __repr__(self):
         output = []
+        # These colored prompts were not working on Windows
+        if sys.platform == 'win32':
+            blue = normal = red = green = ''
+        else:
         blue = TermColors.Blue
         normal = TermColors.Normal
         red = TermColors.Red
         green = TermColors.Green
         output.append("<Results List>\n")
         for cmd in self:
             if isinstance(cmd, Failure):
                 output.append(cmd)
             else:
                 target = cmd.get('id',None)
                 cmd_num = cmd.get('number',None)
                 cmd_stdin = cmd.get('input',{}).get('translated','No Input')
                 cmd_stdout = cmd.get('stdout', None)
                 cmd_stderr = cmd.get('stderr', None)
                 output.append("%s[%i]%s In [%i]:%s %s\n" % \
                     (green, target,
                     blue, cmd_num, normal, cmd_stdin))
                 if cmd_stdout:
                     output.append("%s[%i]%s Out[%i]:%s %s\n" % \
                         (green, target,
                         red, cmd_num, normal, cmd_stdout))
                 if cmd_stderr:
                     output.append("%s[%i]%s Err[%i]:\n%s %s" % \
                         (green, target,
                         red, cmd_num, normal, cmd_stderr))
         return ''.join(output)
 
 
 def wrapResultList(result):
     """A function that wraps the output of `execute`/`get_result` -> `ResultList`."""
     if len(result) == 0:
         result = [result]
     return ResultList(result)
 
 
 class QueueStatusList(list):
     """A subclass of list that pretty prints the output of `queue_status`."""
     
     def __repr__(self):
         output = []
         output.append("<Queue Status List>\n")
         for e in self:
             output.append("Engine: %s\n" % repr(e[0]))
             output.append("    Pending: %s\n" % repr(e[1]['pending']))
             for q in e[1]['queue']:
                 output.append("    Command: %s\n" % repr(q))
         return ''.join(output)
 
 
 #-------------------------------------------------------------------------------
 # InteractiveMultiEngineClient
 #-------------------------------------------------------------------------------    
 
 class InteractiveMultiEngineClient(object):
     """A mixin class that add a few methods to a multiengine client.
     
     The methods in this mixin class are designed for interactive usage.
     """
                                         
     def activate(self):
         """Make this `MultiEngineClient` active for parallel magic commands.
         
         IPython has a magic command syntax to work with `MultiEngineClient` objects.
         In a given IPython session there is a single active one.  While
         there can be many `MultiEngineClient` created and used by the user, 
         there is only one active one.  The active `MultiEngineClient` is used whenever 
         the magic commands %px and %autopx are used.
         
         The activate() method is called on a given `MultiEngineClient` to make it 
         active.  Once this has been done, the magic commands can be used.
         """
         
         try:
             __IPYTHON__.activeController = self
         except NameError:
             print "The IPython Controller magics only work within IPython."
                     
     def __setitem__(self, key, value):
         """Add a dictionary interface for pushing/pulling.
         
         This functions as a shorthand for `push`.
         
         :Parameters:
             key : str 
                 What to call the remote object.
             value : object
                 The local Python object to push.
         """
         targets, block = self._findTargetsAndBlock()
         return self.push({key:value}, targets=targets, block=block)
     
     def __getitem__(self, key):
         """Add a dictionary interface for pushing/pulling.
         
         This functions as a shorthand to `pull`.
         
         :Parameters:
          - `key`: A string representing the key.
         """
         if isinstance(key, str):
             targets, block = self._findTargetsAndBlock()
             return self.pull(key, targets=targets, block=block)
         else:
             raise TypeError("__getitem__ only takes strs")
             
     def __len__(self):
         """Return the number of available engines."""
         return len(self.get_ids())
         
-    def parallelize(self, func, targets=None, block=None):
-        """Build a `ParallelFunction` object for functionName on engines.
-        
-        The returned object will implement a parallel version of functionName
-        that takes a local sequence as its only argument and calls (in 
-        parallel) functionName on each element of that sequence.  The
-        `ParallelFunction` object has a `targets` attribute that controls
-        which engines the function is run on.
-        
-        :Parameters:
-            targets : int, list or 'all'
-                The engine ids the action will apply to.  Call `get_ids` to see
-                a list of currently available engines.
-            functionName : str
-                A Python string that names a callable defined on the engines.
-        
-        :Returns:  A `ParallelFunction` object.                
-        
-        Examples
-        ========
-        
-        >>> psin = rc.parallelize('all','lambda x:sin(x)')
-        >>> psin(range(10000))
-        [0,2,4,9,25,36,...]
-        """
-        targets, block = self._findTargetsAndBlock(targets, block)
-        return ParallelFunction(func, self, targets, block)
-
     #---------------------------------------------------------------------------
     # Make this a context manager for with
     #---------------------------------------------------------------------------
     
     def findsource_file(self,f):
         linecache.checkcache()
         s = findsource(f.f_code)
         lnum = f.f_lineno
         wsource = s[0][f.f_lineno:]
         return strip_whitespace(wsource)
 
     def findsource_ipython(self,f):
         from IPython import ipapi
         self.ip = ipapi.get()
         wsource = [l+'\n' for l in
                    self.ip.IP.input_hist_raw[-1].splitlines()[1:]] 
         return strip_whitespace(wsource)
         
     def __enter__(self):
         f = sys._getframe(1)
         local_ns = f.f_locals
         global_ns = f.f_globals
         if f.f_code.co_filename == '<ipython console>':
             s = self.findsource_ipython(f)
         else:
             s = self.findsource_file(f)
 
         self._with_context_result = self.execute(s)
 
     def __exit__ (self, etype, value, tb):
         if issubclass(etype,error.StopLocalExecution):
             return True
 
 
 def remote():
     m = 'Special exception to stop local execution of parallel code.'
     raise error.StopLocalExecution(m)
 
 def strip_whitespace(source):
     # Expand tabs to avoid any confusion.
     wsource = [l.expandtabs(4) for l in source]
     # Detect the indentation level
     done = False
     for line in wsource:
         if line.isspace():
             continue
         for col,char in enumerate(line):
             if char != ' ':
                 done = True
                 break
         if done:
             break
     # Now we know how much leading space there is in the code.  Next, we
     # extract up to the first line that has less indentation.
     # WARNINGS: we skip comments that may be misindented, but we do NOT yet
     # detect triple quoted strings that may have flush left text.
     for lno,line in enumerate(wsource):
         lead = line[:col]
         if lead.isspace():
             continue
         else:
             if not lead.lstrip().startswith('#'):
                 break
     # The real 'with' source is up to lno
     src_lines = [l[col:] for l in wsource[:lno+1]]
 
     # Finally, check that the source's first non-comment line begins with the
     # special call 'remote()'
     for nline,line in enumerate(src_lines):
         if line.isspace() or line.startswith('#'):
             continue
         if 'remote()' in line:
             break
         else:
             raise ValueError('remote() call missing at the start of code')
     src = ''.join(src_lines[nline+1:])
     #print 'SRC:\n<<<<<<<>>>>>>>\n%s<<<<<>>>>>>' % src  # dbg
     return src
 
 
 #-------------------------------------------------------------------------------
 # The top-level MultiEngine client adaptor
 #-------------------------------------------------------------------------------
 
 
 class IFullBlockingMultiEngineClient(Interface):
     pass
 
 
 class FullBlockingMultiEngineClient(InteractiveMultiEngineClient):
     """
     A blocking client to the `IMultiEngine` controller interface.
     
     This class allows users to use a set of engines for a parallel
     computation through the `IMultiEngine` interface.  In this interface,
     each engine has a specific id (an int) that is used to refer to the
     engine, run code on it, etc.
     """
     
-    implements(IFullBlockingMultiEngineClient)
+    implements(
+        IFullBlockingMultiEngineClient,
+        IMultiEngineMapperFactory,
+        IMapper
+    )
     
     def __init__(self, smultiengine):
         self.smultiengine = smultiengine
         self.block = True
         self.targets = 'all'
     
     def _findBlock(self, block=None):
         if block is None:
             return self.block
         else:
             if block in (True, False):
                 return block
             else:
                 raise ValueError("block must be True or False")
     
     def _findTargets(self, targets=None):
         if targets is None:
             return self.targets
         else:
             if not isinstance(targets, (str,list,tuple,int)):
                 raise ValueError("targets must be a str, list, tuple or int")
             return targets
     
     def _findTargetsAndBlock(self, targets=None, block=None):
         return self._findTargets(targets), self._findBlock(block) 
     
     def _blockFromThread(self, function, *args, **kwargs):
         block = kwargs.get('block', None)
         if block is None:
             raise error.MissingBlockArgument("'block' keyword argument is missing")
         result = blockingCallFromThread(function, *args, **kwargs)
         if not block:
             result = PendingResult(self, result)
         return result
     
     def get_pending_deferred(self, deferredID, block):
         return blockingCallFromThread(self.smultiengine.get_pending_deferred, deferredID, block)
     
     def barrier(self, pendingResults):
         """Synchronize a set of `PendingResults`.
         
         This method is a synchronization primitive that waits for a set of
         `PendingResult` objects to complete.  More specifically, barier does
         the following.
         
         * The `PendingResult`s are sorted by result_id.
         * The `get_result` method is called for each `PendingResult` sequentially
           with block=True.
         * If a `PendingResult` gets a result that is an exception, it is 
           trapped and can be re-raised later by calling `get_result` again.
         * The `PendingResult`s are flushed from the controller.
                 
         After barrier has been called on a `PendingResult`, its results can 
         be retrieved by calling `get_result` again or accesing the `r` attribute
         of the instance.
         """
         
         # Convert to list for sorting and check class type 
         prList = list(pendingResults)
         for pr in prList:
             if not isinstance(pr, PendingResult):
                 raise error.NotAPendingResult("Objects passed to barrier must be PendingResult instances")
                             
         # Sort the PendingResults so they are in order
         prList.sort()
         # Block on each PendingResult object
         for pr in prList:
             try:
                 result = pr.get_result(block=True)
             except Exception:
                 pass
     
     def flush(self):
         """
         Clear all pending deferreds/results from the controller.
         
         For each `PendingResult` that is created by this client, the controller
         holds on to the result for that `PendingResult`.  This can be a problem
         if there are a large number of `PendingResult` objects that are created.
         
         Once the result of the `PendingResult` has been retrieved, the result
         is removed from the controller, but if a user doesn't get a result (
         they just ignore the `PendingResult`) the result is kept forever on the
         controller.  This method allows the user to clear out all un-retrieved
         results on the controller. 
         """
         r = blockingCallFromThread(self.smultiengine.clear_pending_deferreds)
         return r
     
     clear_pending_results = flush
     
     #---------------------------------------------------------------------------
     # IEngineMultiplexer related methods
     #---------------------------------------------------------------------------
     
     def execute(self, lines, targets=None, block=None):
         """
         Execute code on a set of engines.
         
         :Parameters:
             lines : str
                 The Python code to execute as a string
             targets : id or list of ids
                 The engine to use for the execution
             block : boolean
                 If False, this method will return the actual result.  If False,
                 a `PendingResult` is returned which can be used to get the result
                 at a later time.
         """
         targets, block = self._findTargetsAndBlock(targets, block)
         result = blockingCallFromThread(self.smultiengine.execute, lines,
             targets=targets, block=block)
         if block:
             result = ResultList(result)
         else:
             result = PendingResult(self, result)
             result.add_callback(wrapResultList)
         return result
     
     def push(self, namespace, targets=None, block=None):
         """
         Push a dictionary of keys and values to engines namespace.
         
         Each engine has a persistent namespace.  This method is used to push
         Python objects into that namespace.
         
         The objects in the namespace must be pickleable.
         
         :Parameters:
             namespace : dict
                 A dict that contains Python objects to be injected into
                 the engine persistent namespace.
             targets : id or list of ids
                 The engine to use for the execution
             block : boolean
                 If False, this method will return the actual result.  If False,
                 a `PendingResult` is returned which can be used to get the result
                 at a later time.
         """
         targets, block = self._findTargetsAndBlock(targets, block)
         return self._blockFromThread(self.smultiengine.push, namespace,
             targets=targets, block=block)
     
     def pull(self, keys, targets=None, block=None):
         """
         Pull Python objects by key out of engines namespaces.
         
         :Parameters:
             keys : str or list of str
                 The names of the variables to be pulled
             targets : id or list of ids
                 The engine to use for the execution
             block : boolean
                 If False, this method will return the actual result.  If False,
                 a `PendingResult` is returned which can be used to get the result
                 at a later time.
         """
         targets, block = self._findTargetsAndBlock(targets, block)
         return self._blockFromThread(self.smultiengine.pull, keys, targets=targets, block=block)
     
     def push_function(self, namespace, targets=None, block=None):
         """
         Push a Python function to an engine.
         
         This method is used to push a Python function to an engine.  This
         method can then be used in code on the engines.  Closures are not supported.
         
         :Parameters:
             namespace : dict
                 A dict whose values are the functions to be pushed.  The keys give
                 that names that the function will appear as in the engines
                 namespace.
             targets : id or list of ids
                 The engine to use for the execution
             block : boolean
                 If False, this method will return the actual result.  If False,
                 a `PendingResult` is returned which can be used to get the result
                 at a later time.
         """
         targets, block = self._findTargetsAndBlock(targets, block)
         return self._blockFromThread(self.smultiengine.push_function, namespace, targets=targets, block=block)
     
     def pull_function(self, keys, targets=None, block=None):
         """
         Pull a Python function from an engine.
         
         This method is used to pull a Python function from an engine.
         Closures are not supported.
         
         :Parameters:
             keys : str or list of str
                 The names of the functions to be pulled
             targets : id or list of ids
                 The engine to use for the execution
             block : boolean
                 If False, this method will return the actual result.  If False,
                 a `PendingResult` is returned which can be used to get the result
                 at a later time.
         """
         targets, block = self._findTargetsAndBlock(targets, block)
         return self._blockFromThread(self.smultiengine.pull_function, keys, targets=targets, block=block)
     
     def push_serialized(self, namespace, targets=None, block=None):
         targets, block = self._findTargetsAndBlock(targets, block)
         return self._blockFromThread(self.smultiengine.push_serialized, namespace, targets=targets, block=block)
     
     def pull_serialized(self, keys, targets=None, block=None):
         targets, block = self._findTargetsAndBlock(targets, block)
         return self._blockFromThread(self.smultiengine.pull_serialized, keys, targets=targets, block=block)
     
     def get_result(self, i=None, targets=None, block=None):
         """
         Get a previous result.
         
         When code is executed in an engine, a dict is created and returned.  This
         method retrieves that dict for previous commands.
         
         :Parameters:
             i : int
                 The number of the result to get
             targets : id or list of ids
                 The engine to use for the execution
             block : boolean
                 If False, this method will return the actual result.  If False,
                 a `PendingResult` is returned which can be used to get the result
                 at a later time.
         """
         targets, block = self._findTargetsAndBlock(targets, block)
         result = blockingCallFromThread(self.smultiengine.get_result, i, targets=targets, block=block)
         if block:
             result = ResultList(result)
         else:
             result = PendingResult(self, result)
             result.add_callback(wrapResultList)
         return result
     
     def reset(self, targets=None, block=None):
         """
         Reset an engine.
         
         This method clears out the namespace of an engine.
 
         :Parameters:
             targets : id or list of ids
                 The engine to use for the execution
             block : boolean
                 If False, this method will return the actual result.  If False,
                 a `PendingResult` is returned which can be used to get the result
                 at a later time.
         """
         targets, block = self._findTargetsAndBlock(targets, block)
         return self._blockFromThread(self.smultiengine.reset, targets=targets, block=block)
     
     def keys(self, targets=None, block=None):
         """
         Get a list of all the variables in an engine's namespace.
 
         :Parameters:
             targets : id or list of ids
                 The engine to use for the execution
             block : boolean
                 If False, this method will return the actual result.  If False,
                 a `PendingResult` is returned which can be used to get the result
                 at a later time.
         """
         targets, block = self._findTargetsAndBlock(targets, block)
         return self._blockFromThread(self.smultiengine.keys, targets=targets, block=block)
     
     def kill(self, controller=False, targets=None, block=None):
         """
         Kill the engines and controller.
         
         This method is used to stop the engine and controller by calling
         `reactor.stop`.
         
         :Parameters:
             controller : boolean
                 If True, kill the engines and controller.  If False, just the 
                 engines
             targets : id or list of ids
                 The engine to use for the execution
             block : boolean
                 If False, this method will return the actual result.  If False,
                 a `PendingResult` is returned which can be used to get the result
                 at a later time.
         """
         targets, block = self._findTargetsAndBlock(targets, block)
         return self._blockFromThread(self.smultiengine.kill, controller, targets=targets, block=block)
     
     def clear_queue(self, targets=None, block=None):
         """
         Clear out the controller's queue for an engine.
         
         The controller maintains a queue for each engine.  This clear it out.
         
         :Parameters:
             targets : id or list of ids
                 The engine to use for the execution
             block : boolean
                 If False, this method will return the actual result.  If False,
                 a `PendingResult` is returned which can be used to get the result
                 at a later time.
         """
         targets, block = self._findTargetsAndBlock(targets, block)
         return self._blockFromThread(self.smultiengine.clear_queue, targets=targets, block=block)
     
     def queue_status(self, targets=None, block=None):
         """
         Get the status of an engines queue.
         
         :Parameters:
             targets : id or list of ids
                 The engine to use for the execution
             block : boolean
                 If False, this method will return the actual result.  If False,
                 a `PendingResult` is returned which can be used to get the result
                 at a later time.
         """
         targets, block = self._findTargetsAndBlock(targets, block)
         return self._blockFromThread(self.smultiengine.queue_status, targets=targets, block=block)
     
     def set_properties(self, properties, targets=None, block=None):
         targets, block = self._findTargetsAndBlock(targets, block)
         return self._blockFromThread(self.smultiengine.set_properties, properties, targets=targets, block=block)
     
     def get_properties(self, keys=None, targets=None, block=None):
         targets, block = self._findTargetsAndBlock(targets, block)
         return self._blockFromThread(self.smultiengine.get_properties, keys, targets=targets, block=block)
     
     def has_properties(self, keys, targets=None, block=None):
         targets, block = self._findTargetsAndBlock(targets, block)
         return self._blockFromThread(self.smultiengine.has_properties, keys, targets=targets, block=block)
     
     def del_properties(self, keys, targets=None, block=None):
         targets, block = self._findTargetsAndBlock(targets, block)
         return self._blockFromThread(self.smultiengine.del_properties, keys, targets=targets, block=block)
     
     def clear_properties(self, targets=None, block=None):
         targets, block = self._findTargetsAndBlock(targets, block)
         return self._blockFromThread(self.smultiengine.clear_properties, targets=targets, block=block)
     
     #---------------------------------------------------------------------------
     # IMultiEngine related methods
     #---------------------------------------------------------------------------
     
     def get_ids(self):
         """
         Returns the ids of currently registered engines.
         """
         result = blockingCallFromThread(self.smultiengine.get_ids)
         return result
         
     #---------------------------------------------------------------------------
     # IMultiEngineCoordinator
     #---------------------------------------------------------------------------
              
-    def scatter(self, key, seq, style='basic', flatten=False, targets=None, block=None):
+    def scatter(self, key, seq, dist='b', flatten=False, targets=None, block=None):
         """
         Partition a Python sequence and send the partitions to a set of engines.
         """
         targets, block = self._findTargetsAndBlock(targets, block)
         return self._blockFromThread(self.smultiengine.scatter, key, seq, 
-            style, flatten, targets=targets, block=block)
+            dist, flatten, targets=targets, block=block)
     
-    def gather(self, key, style='basic', targets=None, block=None):
+    def gather(self, key, dist='b', targets=None, block=None):
         """
         Gather a partitioned sequence on a set of engines as a single local seq.
         """
         targets, block = self._findTargetsAndBlock(targets, block)
-        return self._blockFromThread(self.smultiengine.gather, key, style, 
+        return self._blockFromThread(self.smultiengine.gather, key, dist, 
             targets=targets, block=block)
     
-    def map(self, func, seq, style='basic', targets=None, block=None):
+    def raw_map(self, func, seq, dist='b', targets=None, block=None):
+        """
+        A parallelized version of Python's builtin map.
+        
+        This has a slightly different syntax than the builtin `map`.
+        This is needed because we need to have keyword arguments and thus
+        can't use *args to capture all the sequences.  Instead, they must
+        be passed in a list or tuple.
+        
+        raw_map(func, seqs) -> map(func, seqs[0], seqs[1], ...)
+        
+        Most users will want to use parallel functions or the `mapper`
+        and `map` methods for an API that follows that of the builtin
+        `map`.
+        """
+        targets, block = self._findTargetsAndBlock(targets, block)
+        return self._blockFromThread(self.smultiengine.raw_map, func, seq, 
+            dist, targets=targets, block=block)
+    
+    def map(self, func, *sequences):
+        """
+        A parallel version of Python's builtin `map` function.
+        
+        This method applies a function to sequences of arguments.  It 
+        follows the same syntax as the builtin `map`.
+        
+        This method creates a mapper objects by calling `self.mapper` with
+        no arguments and then uses that mapper to do the mapping.  See
+        the documentation of `mapper` for more details.
+        """
+        return self.mapper().map(func, *sequences)
+    
+    def mapper(self, dist='b', targets='all', block=None):
+        """
+        Create a mapper object that has a `map` method.
+        
+        This method returns an object that implements the `IMapper` 
+        interface.  This method is a factory that is used to control how 
+        the map happens.
+        
+        :Parameters:
+            dist : str
+                What decomposition to use, 'b' is the only one supported
+                currently
+            targets : str, int, sequence of ints
+                Which engines to use for the map
+            block : boolean
+                Should calls to `map` block or not
+        """
+        return MultiEngineMapper(self, dist, targets, block)
+    
+    def parallel(self, dist='b', targets=None, block=None):
         """
-        A parallelized version of Python's builtin map
+        A decorator that turns a function into a parallel function.
+        
+        This can be used as:
+        
+        @parallel()
+        def f(x, y)
+            ...
+        
+        f(range(10), range(10))
+        
+        This causes f(0,0), f(1,1), ... to be called in parallel.
+        
+        :Parameters:
+            dist : str
+                What decomposition to use, 'b' is the only one supported
+                currently
+            targets : str, int, sequence of ints
+                Which engines to use for the map
+            block : boolean
+                Should calls to `map` block or not
         """
         targets, block = self._findTargetsAndBlock(targets, block)
-        return self._blockFromThread(self.smultiengine.map, func, seq, 
-            style, targets=targets, block=block)
+        mapper = self.mapper(dist, targets, block)
+        pf = ParallelFunction(mapper)
+        return pf
     
     #---------------------------------------------------------------------------
     # IMultiEngineExtras
     #---------------------------------------------------------------------------
     
     def zip_pull(self, keys, targets=None, block=None):
         targets, block = self._findTargetsAndBlock(targets, block)
         return self._blockFromThread(self.smultiengine.zip_pull, keys, 
             targets=targets, block=block)
     
     def run(self, filename, targets=None, block=None):
         """
         Run a Python code in a file on the engines.
         
         :Parameters:
             filename : str
                 The name of the local file to run
             targets : id or list of ids
                 The engine to use for the execution
             block : boolean
                 If False, this method will return the actual result.  If False,
                 a `PendingResult` is returned which can be used to get the result
                 at a later time.
         """
         targets, block = self._findTargetsAndBlock(targets, block)
         return self._blockFromThread(self.smultiengine.run, filename,
             targets=targets, block=block)
 
 
 
 components.registerAdapter(FullBlockingMultiEngineClient,
             IFullSynchronousMultiEngine, IFullBlockingMultiEngineClient)
 
 
 
 

IPython/kernel/multienginefc.py

@@ -1,668 +1,757 b''
 # encoding: utf-8
 
 """
 Expose the multiengine controller over the Foolscap network protocol.
 """
 
 __docformat__ = "restructuredtext en"
 
 #-------------------------------------------------------------------------------
 #  Copyright (C) 2008  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-------------------------------------------------------------------------------
 
 #-------------------------------------------------------------------------------
 # Imports
 #-------------------------------------------------------------------------------
 
 import cPickle as pickle
 from types import FunctionType
 
 from zope.interface import Interface, implements
 from twisted.internet import defer
 from twisted.python import components, failure, log
 
 from foolscap import Referenceable
 
 from IPython.kernel import error 
 from IPython.kernel.util import printer
 from IPython.kernel import map as Map
+from IPython.kernel.parallelfunction import ParallelFunction
+from IPython.kernel.mapper import (
+    MultiEngineMapper, 
+    IMultiEngineMapperFactory,
+    IMapper
+)
 from IPython.kernel.twistedutil import gatherBoth
 from IPython.kernel.multiengine import (MultiEngine,
     IMultiEngine,
     IFullSynchronousMultiEngine,
     ISynchronousMultiEngine)
 from IPython.kernel.multiengineclient import wrapResultList
 from IPython.kernel.pendingdeferred import PendingDeferredManager
 from IPython.kernel.pickleutil import (can, canDict,
     canSequence, uncan, uncanDict, uncanSequence)
 
 from IPython.kernel.clientinterfaces import (
     IFCClientInterfaceProvider, 
     IBlockingClientAdaptor
 )
 
 # Needed to access the true globals from __main__.__dict__ 
 import __main__
 
 #-------------------------------------------------------------------------------
 # The Controller side of things
 #-------------------------------------------------------------------------------
 
 def packageResult(wrappedMethod):
     
     def wrappedPackageResult(self, *args, **kwargs):
         d = wrappedMethod(self, *args, **kwargs)
         d.addCallback(self.packageSuccess)
         d.addErrback(self.packageFailure)
         return d
     return wrappedPackageResult
 
 
 class IFCSynchronousMultiEngine(Interface):
     """Foolscap interface to `ISynchronousMultiEngine`.  
     
     The methods in this interface are similar to those of 
     `ISynchronousMultiEngine`, but their arguments and return values are pickled
     if they are not already simple Python types that can be send over XML-RPC.
     
     See the documentation of `ISynchronousMultiEngine` and `IMultiEngine` for 
     documentation about the methods.
     
     Most methods in this interface act like the `ISynchronousMultiEngine`
     versions and can be called in blocking or non-blocking mode.
     """
     pass
 
 
 class FCSynchronousMultiEngineFromMultiEngine(Referenceable):
     """Adapt `IMultiEngine` -> `ISynchronousMultiEngine` -> `IFCSynchronousMultiEngine`.
     """
     
     implements(IFCSynchronousMultiEngine, IFCClientInterfaceProvider)
     
     addSlash = True
     
     def __init__(self, multiengine):
         # Adapt the raw multiengine to `ISynchronousMultiEngine` before saving
         # it.  This allow this class to do two adaptation steps.
         self.smultiengine = ISynchronousMultiEngine(multiengine)
         self._deferredIDCallbacks = {}
     
     #---------------------------------------------------------------------------
     # Non interface methods
     #---------------------------------------------------------------------------
     
     def packageFailure(self, f):
         f.cleanFailure()
         return self.packageSuccess(f)
     
     def packageSuccess(self, obj):
         serial = pickle.dumps(obj, 2)
         return serial
     
     #---------------------------------------------------------------------------
     # Things related to PendingDeferredManager
     #---------------------------------------------------------------------------
     
     @packageResult
     def remote_get_pending_deferred(self, deferredID, block):
         d = self.smultiengine.get_pending_deferred(deferredID, block)
         try:
             callback = self._deferredIDCallbacks.pop(deferredID)
         except KeyError:
             callback = None
         if callback is not None:
             d.addCallback(callback[0], *callback[1], **callback[2])
         return d
        
     @packageResult
     def remote_clear_pending_deferreds(self):
         return defer.maybeDeferred(self.smultiengine.clear_pending_deferreds)
     
     def _addDeferredIDCallback(self, did, callback, *args, **kwargs):
         self._deferredIDCallbacks[did] = (callback, args, kwargs)
         return did
         
     #---------------------------------------------------------------------------
     # IEngineMultiplexer related methods
     #---------------------------------------------------------------------------
     
     @packageResult
     def remote_execute(self, lines, targets, block):
         return self.smultiengine.execute(lines, targets=targets, block=block)
     
     @packageResult    
     def remote_push(self, binaryNS, targets, block):
         try:
             namespace = pickle.loads(binaryNS)
         except:
             d = defer.fail(failure.Failure())
         else:
             d = self.smultiengine.push(namespace, targets=targets, block=block)
         return d
     
     @packageResult
     def remote_pull(self, keys, targets, block):
         d = self.smultiengine.pull(keys, targets=targets, block=block)
         return d
     
     @packageResult    
     def remote_push_function(self, binaryNS, targets, block):
         try:
             namespace = pickle.loads(binaryNS)
         except:
             d = defer.fail(failure.Failure())
         else:
             namespace = uncanDict(namespace)
             d = self.smultiengine.push_function(namespace, targets=targets, block=block)
         return d
     
     def _canMultipleKeys(self, result):
         return [canSequence(r) for r in result]
     
     @packageResult
     def remote_pull_function(self, keys, targets, block):
         def can_functions(r, keys):
             if len(keys)==1 or isinstance(keys, str):
                 result = canSequence(r)
             elif len(keys)>1:
                 result = [canSequence(s) for s in r]
             return result
         d = self.smultiengine.pull_function(keys, targets=targets, block=block)
         if block:
             d.addCallback(can_functions, keys)
         else:
             d.addCallback(lambda did: self._addDeferredIDCallback(did, can_functions, keys))
         return d
     
     @packageResult    
     def remote_push_serialized(self, binaryNS, targets, block):
         try:
             namespace = pickle.loads(binaryNS)
         except:
             d = defer.fail(failure.Failure())
         else:
             d = self.smultiengine.push_serialized(namespace, targets=targets, block=block)
         return d
     
     @packageResult
     def remote_pull_serialized(self, keys, targets, block):
         d = self.smultiengine.pull_serialized(keys, targets=targets, block=block)
         return d
     
     @packageResult
     def remote_get_result(self, i, targets, block):
         if i == 'None':
             i = None
         return self.smultiengine.get_result(i, targets=targets, block=block)
     
     @packageResult
     def remote_reset(self, targets, block):
         return self.smultiengine.reset(targets=targets, block=block)
     
     @packageResult
     def remote_keys(self, targets, block):
         return self.smultiengine.keys(targets=targets, block=block)
     
     @packageResult
     def remote_kill(self, controller, targets, block):
         return self.smultiengine.kill(controller, targets=targets, block=block)
     
     @packageResult
     def remote_clear_queue(self, targets, block):
         return self.smultiengine.clear_queue(targets=targets, block=block)
     
     @packageResult
     def remote_queue_status(self, targets, block):
         return self.smultiengine.queue_status(targets=targets, block=block)
     
     @packageResult
     def remote_set_properties(self, binaryNS, targets, block):
         try:
             ns = pickle.loads(binaryNS)
         except:
             d = defer.fail(failure.Failure())
         else:
             d = self.smultiengine.set_properties(ns, targets=targets, block=block)
         return d
     
     @packageResult
     def remote_get_properties(self, keys, targets, block):
         if keys=='None':
             keys=None
         return self.smultiengine.get_properties(keys, targets=targets, block=block)
     
     @packageResult
     def remote_has_properties(self, keys, targets, block):
         return self.smultiengine.has_properties(keys, targets=targets, block=block)
     
     @packageResult
     def remote_del_properties(self, keys, targets, block):
         return self.smultiengine.del_properties(keys, targets=targets, block=block)
     
     @packageResult
     def remote_clear_properties(self, targets, block):
         return self.smultiengine.clear_properties(targets=targets, block=block)
     
     #---------------------------------------------------------------------------
     # IMultiEngine related methods
     #---------------------------------------------------------------------------
     
     def remote_get_ids(self):
         """Get the ids of the registered engines.
         
         This method always blocks.
         """
         return self.smultiengine.get_ids()
     
     #---------------------------------------------------------------------------
     # IFCClientInterfaceProvider related methods
     #---------------------------------------------------------------------------
     
     def remote_get_client_name(self):
         return 'IPython.kernel.multienginefc.FCFullSynchronousMultiEngineClient'
 
 
 # The __init__ method of `FCMultiEngineFromMultiEngine` first adapts the
 # `IMultiEngine` to `ISynchronousMultiEngine` so this is actually doing a
 # two phase adaptation.
 components.registerAdapter(FCSynchronousMultiEngineFromMultiEngine,
             IMultiEngine, IFCSynchronousMultiEngine)
 
 
 #-------------------------------------------------------------------------------
 # The Client side of things
 #-------------------------------------------------------------------------------
 
 
 class FCFullSynchronousMultiEngineClient(object):
     
-    implements(IFullSynchronousMultiEngine, IBlockingClientAdaptor)
+    implements(
+        IFullSynchronousMultiEngine, 
+        IBlockingClientAdaptor,
+        IMultiEngineMapperFactory,
+        IMapper
+    )
     
     def __init__(self, remote_reference):
         self.remote_reference = remote_reference
         self._deferredIDCallbacks = {}
         # This class manages some pending deferreds through this instance.  This
         # is required for methods like gather/scatter as it enables us to
         # create our own pending deferreds for composite operations.
         self.pdm = PendingDeferredManager()
     
     #---------------------------------------------------------------------------
     # Non interface methods
     #---------------------------------------------------------------------------
                  
     def unpackage(self, r):
         return pickle.loads(r)
     
     #---------------------------------------------------------------------------
     # Things related to PendingDeferredManager
     #---------------------------------------------------------------------------
     
     def get_pending_deferred(self, deferredID, block=True):
         
         # Because we are managing some pending deferreds locally (through
         # self.pdm) and some remotely (on the controller), we first try the 
         # local one and then the remote one.
         if self.pdm.quick_has_id(deferredID):
             d = self.pdm.get_pending_deferred(deferredID, block)
             return d
         else:
             d = self.remote_reference.callRemote('get_pending_deferred', deferredID, block)
             d.addCallback(self.unpackage)
             try:
                 callback = self._deferredIDCallbacks.pop(deferredID)
             except KeyError:
                 callback = None
             if callback is not None:
                 d.addCallback(callback[0], *callback[1], **callback[2])
             return d
     
     def clear_pending_deferreds(self):
         
         # This clear both the local (self.pdm) and remote pending deferreds
         self.pdm.clear_pending_deferreds()
         d2 = self.remote_reference.callRemote('clear_pending_deferreds')
         d2.addCallback(self.unpackage)
         return d2
     
     def _addDeferredIDCallback(self, did, callback, *args, **kwargs):
         self._deferredIDCallbacks[did] = (callback, args, kwargs)
         return did
        
     #---------------------------------------------------------------------------
     # IEngineMultiplexer related methods
     #---------------------------------------------------------------------------
         
     def execute(self, lines, targets='all', block=True):
         d = self.remote_reference.callRemote('execute', lines, targets, block)
         d.addCallback(self.unpackage)
         return d
     
     def push(self, namespace, targets='all', block=True):
         serial = pickle.dumps(namespace, 2)
         d =  self.remote_reference.callRemote('push', serial, targets, block)
         d.addCallback(self.unpackage)
         return d
     
     def pull(self, keys, targets='all', block=True):
         d = self.remote_reference.callRemote('pull', keys, targets, block)
         d.addCallback(self.unpackage)
         return d
     
     def push_function(self, namespace, targets='all', block=True):
         cannedNamespace = canDict(namespace)
         serial = pickle.dumps(cannedNamespace, 2)
         d = self.remote_reference.callRemote('push_function', serial, targets, block)
         d.addCallback(self.unpackage)
         return d
     
     def pull_function(self, keys, targets='all', block=True):
         def uncan_functions(r, keys):
             if len(keys)==1 or isinstance(keys, str):
                 return uncanSequence(r)
             elif len(keys)>1:
                 return [uncanSequence(s) for s in r]
         d = self.remote_reference.callRemote('pull_function', keys, targets, block)
         if block:
             d.addCallback(self.unpackage)
             d.addCallback(uncan_functions, keys)
         else:
             d.addCallback(self.unpackage)
             d.addCallback(lambda did: self._addDeferredIDCallback(did, uncan_functions, keys))
         return d
     
     def push_serialized(self, namespace, targets='all', block=True):
         cannedNamespace = canDict(namespace)
         serial = pickle.dumps(cannedNamespace, 2)
         d =  self.remote_reference.callRemote('push_serialized', serial, targets, block)
         d.addCallback(self.unpackage)
         return d
     
     def pull_serialized(self, keys, targets='all', block=True):
         d = self.remote_reference.callRemote('pull_serialized', keys, targets, block)
         d.addCallback(self.unpackage)
         return d
         
     def get_result(self, i=None, targets='all', block=True):
         if i is None: # This is because None cannot be marshalled by xml-rpc
             i = 'None'
         d = self.remote_reference.callRemote('get_result', i, targets, block)
         d.addCallback(self.unpackage)
         return d
     
     def reset(self, targets='all', block=True):
         d = self.remote_reference.callRemote('reset', targets, block)
         d.addCallback(self.unpackage)
         return d        
     
     def keys(self, targets='all', block=True):
         d = self.remote_reference.callRemote('keys', targets, block)
         d.addCallback(self.unpackage)
         return d
     
     def kill(self, controller=False, targets='all', block=True):
         d = self.remote_reference.callRemote('kill', controller, targets, block)
         d.addCallback(self.unpackage)
         return d
     
     def clear_queue(self, targets='all', block=True):
         d = self.remote_reference.callRemote('clear_queue', targets, block)
         d.addCallback(self.unpackage)
         return d
     
     def queue_status(self, targets='all', block=True):
         d = self.remote_reference.callRemote('queue_status', targets, block)
         d.addCallback(self.unpackage)
         return d
     
     def set_properties(self, properties, targets='all', block=True):
         serial = pickle.dumps(properties, 2)
         d = self.remote_reference.callRemote('set_properties', serial, targets, block)
         d.addCallback(self.unpackage)
         return d
     
     def get_properties(self, keys=None, targets='all', block=True):
         if keys==None:
             keys='None'
         d = self.remote_reference.callRemote('get_properties', keys, targets, block)
         d.addCallback(self.unpackage)
         return d
     
     def has_properties(self, keys, targets='all', block=True):
         d = self.remote_reference.callRemote('has_properties', keys, targets, block)
         d.addCallback(self.unpackage)
         return d
     
     def del_properties(self, keys, targets='all', block=True):
         d = self.remote_reference.callRemote('del_properties', keys, targets, block)
         d.addCallback(self.unpackage)
         return d
     
     def clear_properties(self, targets='all', block=True):
         d = self.remote_reference.callRemote('clear_properties', targets, block)
         d.addCallback(self.unpackage)
         return d
     
     #---------------------------------------------------------------------------
     # IMultiEngine related methods
     #---------------------------------------------------------------------------
     
     def get_ids(self):
         d = self.remote_reference.callRemote('get_ids')
         return d
     
     #---------------------------------------------------------------------------
     # ISynchronousMultiEngineCoordinator related methods
     #---------------------------------------------------------------------------
 
     def _process_targets(self, targets):
         def create_targets(ids):
             if isinstance(targets, int):
                 engines = [targets]
             elif targets=='all':
                 engines = ids
             elif isinstance(targets, (list, tuple)):
                 engines = targets
             for t in engines:
                 if not t in ids:
                     raise error.InvalidEngineID("engine with id %r does not exist"%t)
             return engines
         
         d = self.get_ids()
         d.addCallback(create_targets)
         return d
     
-    def scatter(self, key, seq, style='basic', flatten=False, targets='all', block=True):
+    def scatter(self, key, seq, dist='b', flatten=False, targets='all', block=True):
         
         # Note: scatter and gather handle pending deferreds locally through self.pdm.
         # This enables us to collect a bunch fo deferred ids and make a secondary 
         # deferred id that corresponds to the entire group.  This logic is extremely
         # difficult to get right though.
         def do_scatter(engines):
             nEngines = len(engines)
-            mapClass = Map.styles[style]
+            mapClass = Map.dists[dist]
             mapObject = mapClass()
             d_list = []
             # Loop through and push to each engine in non-blocking mode.
             # This returns a set of deferreds to deferred_ids
             for index, engineid in enumerate(engines):
                 partition = mapObject.getPartition(seq, index, nEngines)
                 if flatten and len(partition) == 1:
                     d = self.push({key: partition[0]}, targets=engineid, block=False)
                 else:
                     d = self.push({key: partition}, targets=engineid, block=False)
                 d_list.append(d)
             # Collect the deferred to deferred_ids
             d = gatherBoth(d_list,
                            fireOnOneErrback=0,
                            consumeErrors=1,
                            logErrors=0)
             # Now d has a list of deferred_ids or Failures coming
             d.addCallback(error.collect_exceptions, 'scatter')
             def process_did_list(did_list):
                 """Turn a list of deferred_ids into a final result or failure."""
                 new_d_list = [self.get_pending_deferred(did, True) for did in did_list]
                 final_d = gatherBoth(new_d_list,
                                      fireOnOneErrback=0,
                                      consumeErrors=1,
                                      logErrors=0)
                 final_d.addCallback(error.collect_exceptions, 'scatter')
                 final_d.addCallback(lambda lop: [i[0] for i in lop])
                 return final_d
             # Now, depending on block, we need to handle the list deferred_ids
             # coming down the pipe diferently.
             if block:
                 # If we are blocking register a callback that will transform the
                 # list of deferred_ids into the final result.
                 d.addCallback(process_did_list)
                 return d
             else:
                 # Here we are going to use a _local_ PendingDeferredManager.
                 deferred_id = self.pdm.get_deferred_id()
                 # This is the deferred we will return to the user that will fire
                 # with the local deferred_id AFTER we have received the list of 
                 # primary deferred_ids
                 d_to_return = defer.Deferred()
                 def do_it(did_list):
                     """Produce a deferred to the final result, but first fire the
                     deferred we will return to the user that has the local
                     deferred id."""
                     d_to_return.callback(deferred_id)
                     return process_did_list(did_list)
                 d.addCallback(do_it)
                 # Now save the deferred to the final result
                 self.pdm.save_pending_deferred(d, deferred_id)
                 return d_to_return
 
         d = self._process_targets(targets)
         d.addCallback(do_scatter)
         return d
 
-    def gather(self, key, style='basic', targets='all', block=True):
+    def gather(self, key, dist='b', targets='all', block=True):
         
         # Note: scatter and gather handle pending deferreds locally through self.pdm.
         # This enables us to collect a bunch fo deferred ids and make a secondary 
         # deferred id that corresponds to the entire group.  This logic is extremely
         # difficult to get right though.
         def do_gather(engines):
             nEngines = len(engines)
-            mapClass = Map.styles[style]
+            mapClass = Map.dists[dist]
             mapObject = mapClass()
             d_list = []
             # Loop through and push to each engine in non-blocking mode.
             # This returns a set of deferreds to deferred_ids
             for index, engineid in enumerate(engines):
                 d = self.pull(key, targets=engineid, block=False)
                 d_list.append(d)
             # Collect the deferred to deferred_ids
             d = gatherBoth(d_list,
                            fireOnOneErrback=0,
                            consumeErrors=1,
                            logErrors=0)
             # Now d has a list of deferred_ids or Failures coming
             d.addCallback(error.collect_exceptions, 'scatter')
             def process_did_list(did_list):
                 """Turn a list of deferred_ids into a final result or failure."""
                 new_d_list = [self.get_pending_deferred(did, True) for did in did_list]
                 final_d = gatherBoth(new_d_list,
                                      fireOnOneErrback=0,
                                      consumeErrors=1,
                                      logErrors=0)
                 final_d.addCallback(error.collect_exceptions, 'gather')
                 final_d.addCallback(lambda lop: [i[0] for i in lop])
                 final_d.addCallback(mapObject.joinPartitions)
                 return final_d
             # Now, depending on block, we need to handle the list deferred_ids
             # coming down the pipe diferently.
             if block:
                 # If we are blocking register a callback that will transform the
                 # list of deferred_ids into the final result.
                 d.addCallback(process_did_list)
                 return d
             else:
                 # Here we are going to use a _local_ PendingDeferredManager.
                 deferred_id = self.pdm.get_deferred_id()
                 # This is the deferred we will return to the user that will fire
                 # with the local deferred_id AFTER we have received the list of 
                 # primary deferred_ids
                 d_to_return = defer.Deferred()
                 def do_it(did_list):
                     """Produce a deferred to the final result, but first fire the
                     deferred we will return to the user that has the local
                     deferred id."""
                     d_to_return.callback(deferred_id)
                     return process_did_list(did_list)
                 d.addCallback(do_it)
                 # Now save the deferred to the final result
                 self.pdm.save_pending_deferred(d, deferred_id)
                 return d_to_return
 
         d = self._process_targets(targets)
         d.addCallback(do_gather)
         return d
 
-    def map(self, func, seq, style='basic', targets='all', block=True):
-        d_list = []
+    def raw_map(self, func, sequences, dist='b', targets='all', block=True):
+        """
+        A parallelized version of Python's builtin map.
+        
+        This has a slightly different syntax than the builtin `map`.
+        This is needed because we need to have keyword arguments and thus
+        can't use *args to capture all the sequences.  Instead, they must
+        be passed in a list or tuple.
+        
+        raw_map(func, seqs) -> map(func, seqs[0], seqs[1], ...)
+        
+        Most users will want to use parallel functions or the `mapper`
+        and `map` methods for an API that follows that of the builtin
+        `map`.
+        """
+        if not isinstance(sequences, (list, tuple)):
+            raise TypeError('sequences must be a list or tuple')
+        max_len = max(len(s) for s in sequences)
+        for s in sequences:
+            if len(s)!=max_len:
+                raise ValueError('all sequences must have equal length')
         if isinstance(func, FunctionType):
             d = self.push_function(dict(_ipython_map_func=func), targets=targets, block=False)
             d.addCallback(lambda did: self.get_pending_deferred(did, True))
-            sourceToRun = '_ipython_map_seq_result = map(_ipython_map_func, _ipython_map_seq)'
+            sourceToRun = '_ipython_map_seq_result = map(_ipython_map_func, *zip(*_ipython_map_seq))'
         elif isinstance(func, str):
             d = defer.succeed(None)
             sourceToRun = \
-                '_ipython_map_seq_result = map(%s, _ipython_map_seq)' % func
+                '_ipython_map_seq_result = map(%s, *zip(*_ipython_map_seq))' % func
         else:
             raise TypeError("func must be a function or str")
         
-        d.addCallback(lambda _: self.scatter('_ipython_map_seq', seq, style, targets=targets))
+        d.addCallback(lambda _: self.scatter('_ipython_map_seq', zip(*sequences), dist, targets=targets))
         d.addCallback(lambda _: self.execute(sourceToRun, targets=targets, block=False))
         d.addCallback(lambda did: self.get_pending_deferred(did, True))
-        d.addCallback(lambda _: self.gather('_ipython_map_seq_result', style, targets=targets, block=block))
+        d.addCallback(lambda _: self.gather('_ipython_map_seq_result', dist, targets=targets, block=block))
         return d
 
+    def map(self, func, *sequences):
+        """
+        A parallel version of Python's builtin `map` function.
+        
+        This method applies a function to sequences of arguments.  It 
+        follows the same syntax as the builtin `map`.
+        
+        This method creates a mapper objects by calling `self.mapper` with
+        no arguments and then uses that mapper to do the mapping.  See
+        the documentation of `mapper` for more details.
+        """
+        return self.mapper().map(func, *sequences)
+    
+    def mapper(self, dist='b', targets='all', block=True):
+        """
+        Create a mapper object that has a `map` method.
+        
+        This method returns an object that implements the `IMapper` 
+        interface.  This method is a factory that is used to control how 
+        the map happens.
+        
+        :Parameters:
+            dist : str
+                What decomposition to use, 'b' is the only one supported
+                currently
+            targets : str, int, sequence of ints
+                Which engines to use for the map
+            block : boolean
+                Should calls to `map` block or not
+        """
+        return MultiEngineMapper(self, dist, targets, block)
+
+    def parallel(self, dist='b', targets='all', block=True):
+        """
+        A decorator that turns a function into a parallel function.
+        
+        This can be used as:
+        
+        @parallel()
+        def f(x, y)
+            ...
+        
+        f(range(10), range(10))
+        
+        This causes f(0,0), f(1,1), ... to be called in parallel.
+        
+        :Parameters:
+            dist : str
+                What decomposition to use, 'b' is the only one supported
+                currently
+            targets : str, int, sequence of ints
+                Which engines to use for the map
+            block : boolean
+                Should calls to `map` block or not
+        """
+        mapper = self.mapper(dist, targets, block)
+        pf = ParallelFunction(mapper)
+        return pf
+    
     #---------------------------------------------------------------------------
     # ISynchronousMultiEngineExtras related methods
     #---------------------------------------------------------------------------
     
     def _transformPullResult(self, pushResult, multitargets, lenKeys):
         if not multitargets:
             result = pushResult[0]
         elif lenKeys > 1:
             result = zip(*pushResult)
         elif lenKeys is 1:
             result = list(pushResult)
         return result
     
     def zip_pull(self, keys, targets='all', block=True):
         multitargets = not isinstance(targets, int) and len(targets) > 1
         lenKeys = len(keys)
         d = self.pull(keys, targets=targets, block=block)
         if block:
             d.addCallback(self._transformPullResult, multitargets, lenKeys)
         else:
             d.addCallback(lambda did: self._addDeferredIDCallback(did, self._transformPullResult, multitargets, lenKeys))
         return d
     
     def run(self, fname, targets='all', block=True):
         fileobj = open(fname,'r')
         source = fileobj.read()
         fileobj.close()
         # if the compilation blows, we get a local error right away
         try:
             code = compile(source,fname,'exec')
         except:
             return defer.fail(failure.Failure()) 
         # Now run the code
         d = self.execute(source, targets=targets, block=block)
         return d
     
     #---------------------------------------------------------------------------
     # IBlockingClientAdaptor related methods
     #---------------------------------------------------------------------------
     
     def adapt_to_blocking_client(self):
         from IPython.kernel.multiengineclient import IFullBlockingMultiEngineClient
         return IFullBlockingMultiEngineClient(self)

IPython/kernel/parallelfunction.py

@@ -1,32 +1,107 b''
 # encoding: utf-8
 
 """A parallelized function that does scatter/execute/gather."""
 
 __docformat__ = "restructuredtext en"
 
 #-------------------------------------------------------------------------------
 #  Copyright (C) 2008  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-------------------------------------------------------------------------------
 
 #-------------------------------------------------------------------------------
 # Imports
 #-------------------------------------------------------------------------------
 
 from types import FunctionType
+from zope.interface import Interface, implements
 
-class ParallelFunction:
-    """A function that operates in parallel on sequences."""
-    def __init__(self, func, multiengine, targets, block):
-        """Create a `ParallelFunction`.
+
+class IMultiEngineParallelDecorator(Interface):
+    """A decorator that creates a parallel function."""
+    
+    def parallel(dist='b', targets=None, block=None):
+        """
+        A decorator that turns a function into a parallel function.
+        
+        This can be used as:
+        
+        @parallel()
+        def f(x, y)
+            ...
+        
+        f(range(10), range(10))
+        
+        This causes f(0,0), f(1,1), ... to be called in parallel.
+        
+        :Parameters:
+            dist : str
+                What decomposition to use, 'b' is the only one supported
+                currently
+            targets : str, int, sequence of ints
+                Which engines to use for the map
+            block : boolean
+                Should calls to `map` block or not
+        """
+
+class ITaskParallelDecorator(Interface):
+    """A decorator that creates a parallel function."""
+    
+    def parallel(clear_before=False, clear_after=False, retries=0, 
+        recovery_task=None, depend=None, block=True):
+        """
+        A decorator that turns a function into a parallel function.
+        
+        This can be used as:
+        
+        @parallel()
+        def f(x, y)
+            ...
+        
+        f(range(10), range(10))
+        
+        This causes f(0,0), f(1,1), ... to be called in parallel.
+        
+        See the documentation for `IPython.kernel.task.BaseTask` for 
+        documentation on the arguments to this method.
+        """
+
+class IParallelFunction(Interface):
+    pass
+
+class ParallelFunction(object):
+    """
+    The implementation of a parallel function.
+    
+    A parallel function is similar to Python's map function: 
+    
+    map(func, *sequences) -> pfunc(*sequences)
+    
+    Parallel functions should be created by using the @parallel decorator.
+    """
+    
+    implements(IParallelFunction)
+    
+    def __init__(self, mapper):
+        """
+        Create a parallel function from an `IMapper`.
+        
+        :Parameters:
+            mapper : an `IMapper` implementer.
+                The mapper to use for the parallel function
+        """
+        self.mapper = mapper
+        
+    def __call__(self, func):
+        """
+        Decorate a function to make it run in parallel.
         """
         assert isinstance(func, (str, FunctionType)), "func must be a fuction or str"
         self.func = func
-        self.multiengine = multiengine
-        self.targets = targets
-        self.block = block
+        def call_function(*sequences):
+            return self.mapper.map(self.func, *sequences)
+        return call_function
+
         
  No newline at end of file
-    def __call__(self, sequence):
-        return self.multiengine.map(self.func, sequence, targets=self.targets, block=self.block)
  No newline at end of file

IPython/kernel/scripts/ipcluster.py

@@ -1,323 +1,324 b''
 #!/usr/bin/env python
 # encoding: utf-8
 
 """Start an IPython cluster conveniently, either locally or remotely.
 
 Basic usage
 -----------
 
 For local operation, the simplest mode of usage is:
 
   %prog -n N
 
 where N is the number of engines you want started.
 
 For remote operation, you must call it with a cluster description file:
 
   %prog -f clusterfile.py
 
 The cluster file is a normal Python script which gets run via execfile().  You
 can have arbitrary logic in it, but all that matters is that at the end of the
 execution, it declares the variables 'controller', 'engines', and optionally
 'sshx'.  See the accompanying examples for details on what these variables must
 contain.
 
 
 Notes
 -----
 
 WARNING: this code is still UNFINISHED and EXPERIMENTAL!  It is incomplete,
 some listed options are not really implemented, and all of its interfaces are
 subject to change.
 
 When operating over SSH for a remote cluster, this program relies on the
 existence of a particular script called 'sshx'.  This script must live in the
 target systems where you'll be running your controller and engines, and is
 needed to configure your PATH and PYTHONPATH variables for further execution of
 python code at the other end of an SSH connection.  The script can be as simple
 as:
 
 #!/bin/sh
 . $HOME/.bashrc
 "$@"
 
 which is the default one provided by IPython.  You can modify this or provide
 your own.  Since it's quite likely that for different clusters you may need
 this script to configure things differently or that it may live in different
 locations, its full path can be set in the same file where you define the
 cluster setup.  IPython's order of evaluation for this variable is the
 following:
 
   a) Internal default: 'sshx'.  This only works if it is in the default system
   path which SSH sets up in non-interactive mode.
 
   b) Environment variable: if $IPYTHON_SSHX is defined, this overrides the
   internal default.
 
   c) Variable 'sshx' in the cluster configuration file: finally, this will
   override the previous two values.
  
 This code is Unix-only, with precious little hope of any of this ever working
 under Windows, since we need SSH from the ground up, we background processes,
 etc.  Ports of this functionality to Windows are welcome.
 
 
 Call summary
 ------------
 
     %prog [options]
 """
 
 __docformat__ = "restructuredtext en"
 
 #-------------------------------------------------------------------------------
 #  Copyright (C) 2008  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-------------------------------------------------------------------------------
 
 #-------------------------------------------------------------------------------
 # Stdlib imports
 #-------------------------------------------------------------------------------
 
 import os
 import signal
 import sys
 import time
 
 from optparse import OptionParser
 from subprocess import Popen,call
 
 #---------------------------------------------------------------------------
 # IPython imports
 #---------------------------------------------------------------------------
 from IPython.tools import utils
 from IPython.config import cutils
 
 #---------------------------------------------------------------------------
 # Normal code begins
 #---------------------------------------------------------------------------
 
 def parse_args():
     """Parse command line and return opts,args."""
 
     parser = OptionParser(usage=__doc__)
     newopt = parser.add_option  # shorthand
 
     newopt("--controller-port", type="int", dest="controllerport",
            help="the TCP port the controller is listening on")
 
     newopt("--controller-ip", type="string", dest="controllerip",
            help="the TCP ip address of the controller")
 
     newopt("-n", "--num", type="int", dest="n",default=2,
            help="the number of engines to start")
 
     newopt("--engine-port", type="int", dest="engineport",
            help="the TCP port the controller will listen on for engine "
            "connections")
     
     newopt("--engine-ip", type="string", dest="engineip",
            help="the TCP ip address the controller will listen on "
            "for engine connections")
 
     newopt("--mpi", type="string", dest="mpi",
            help="use mpi with package:  for instance --mpi=mpi4py")
     
     newopt("-l", "--logfile", type="string", dest="logfile",
            help="log file name")
 
     newopt('-f','--cluster-file',dest='clusterfile',
            help='file describing a remote cluster')
 
     return parser.parse_args()
 
 def numAlive(controller,engines):
     """Return the number of processes still alive."""
     retcodes = [controller.poll()] + \
                [e.poll() for e in engines]
     return retcodes.count(None)
 
 stop = lambda pid: os.kill(pid,signal.SIGINT)
 kill = lambda pid: os.kill(pid,signal.SIGTERM)
 
 def cleanup(clean,controller,engines):
     """Stop the controller and engines with the given cleanup method."""
     
     for e in engines:
         if e.poll() is None:
             print 'Stopping engine, pid',e.pid
             clean(e.pid)
     if controller.poll() is None:
         print 'Stopping controller, pid',controller.pid
         clean(controller.pid)
 
 
 def ensureDir(path):
     """Ensure a directory exists or raise an exception."""
     if not os.path.isdir(path):
         os.makedirs(path)
 
 
 def startMsg(control_host,control_port=10105):
     """Print a startup message"""
     print
     print 'Your cluster is up and running.'
     print
     print 'For interactive use, you can make a MultiEngineClient with:'
     print
     print 'from IPython.kernel import client'
-    print "mec = client.MultiEngineClient((%r,%s))" % \
-          (control_host,control_port)
+    print "mec = client.MultiEngineClient()"
     print
     print 'You can then cleanly stop the cluster from IPython using:'
     print
     print 'mec.kill(controller=True)'
     print
 
     
 def clusterLocal(opt,arg):
     """Start a cluster on the local machine."""
     
     # Store all logs inside the ipython directory
     ipdir = cutils.get_ipython_dir()
     pjoin = os.path.join
 
     logfile = opt.logfile
     if logfile is None:
         logdir_base = pjoin(ipdir,'log')
         ensureDir(logdir_base)
         logfile = pjoin(logdir_base,'ipcluster-')
 
     print 'Starting controller:',
-    controller = Popen(['ipcontroller','--logfile',logfile])
+    controller = Popen(['ipcontroller','--logfile',logfile,'-x','-y'])
     print 'Controller PID:',controller.pid
 
     print 'Starting engines:   ',
-    time.sleep(3)
+    time.sleep(5)
 
     englogfile = '%s%s-' % (logfile,controller.pid)
     mpi = opt.mpi
     if mpi: # start with mpi - killing the engines with sigterm will not work if you do this
-        engines = [Popen(['mpirun', '-np', str(opt.n), 'ipengine', '--mpi', mpi, '--logfile',englogfile])]
+        engines = [Popen(['mpirun', '-np', str(opt.n), 'ipengine', '--mpi', 
+            mpi, '--logfile',englogfile])]
+        # engines = [Popen(['mpirun', '-np', str(opt.n), 'ipengine', '--mpi', mpi])]
     else: # do what we would normally do
         engines = [ Popen(['ipengine','--logfile',englogfile])
                     for i in range(opt.n) ]
     eids = [e.pid for e in engines]
     print 'Engines PIDs:  ',eids
     print 'Log files: %s*' % englogfile
     
     proc_ids = eids + [controller.pid]
     procs = engines + [controller]
 
     grpid = os.getpgrp()
     try:
         startMsg('127.0.0.1')
         print 'You can also hit Ctrl-C to stop it, or use from the cmd line:'
         print
         print 'kill -INT',grpid
         print
         try:
             while True:
                 time.sleep(5)
         except:
             pass
     finally:
         print 'Stopping cluster.  Cleaning up...'
         cleanup(stop,controller,engines)
         for i in range(4):
             time.sleep(i+2)
             nZombies = numAlive(controller,engines)
             if  nZombies== 0:
                 print 'OK: All processes cleaned up.'
                 break
             print 'Trying again, %d processes did not stop...' % nZombies
             cleanup(kill,controller,engines)
             if numAlive(controller,engines) == 0:
                 print 'OK: All processes cleaned up.'
                 break
         else:
             print '*'*75
             print 'ERROR: could not kill some processes, try to do it',
             print 'manually.'
             zombies = []
             if controller.returncode is None:
                 print 'Controller is alive: pid =',controller.pid
                 zombies.append(controller.pid)
             liveEngines = [ e for e in engines if e.returncode is None ]
             for e in liveEngines:
                 print 'Engine is alive:     pid =',e.pid
                 zombies.append(e.pid)
             print
             print 'Zombie summary:',' '.join(map(str,zombies))
 
 def clusterRemote(opt,arg):
     """Start a remote cluster over SSH"""
 
     # Load the remote cluster configuration
     clConfig = {}
     execfile(opt.clusterfile,clConfig)
     contConfig = clConfig['controller']
     engConfig = clConfig['engines']
     # Determine where to find sshx:
     sshx = clConfig.get('sshx',os.environ.get('IPYTHON_SSHX','sshx'))
     
     # Store all logs inside the ipython directory
     ipdir = cutils.get_ipython_dir()
     pjoin = os.path.join
 
     logfile = opt.logfile
     if logfile is None:
         logdir_base = pjoin(ipdir,'log')
         ensureDir(logdir_base)
         logfile = pjoin(logdir_base,'ipcluster')
 
     # Append this script's PID to the logfile name always
     logfile = '%s-%s' % (logfile,os.getpid())
     
     print 'Starting controller:'
     # Controller data:
     xsys = os.system
 
     contHost = contConfig['host']
     contLog = '%s-con-%s-' % (logfile,contHost)
     cmd = "ssh %s '%s' 'ipcontroller --logfile %s' &" % \
           (contHost,sshx,contLog)
     #print 'cmd:<%s>' % cmd  # dbg
     xsys(cmd)
     time.sleep(2)
 
     print 'Starting engines:   '
     for engineHost,engineData in engConfig.iteritems():
         if isinstance(engineData,int):
             numEngines = engineData
         else:
             raise NotImplementedError('port configuration not finished for engines')
 
         print 'Sarting %d engines on %s' % (numEngines,engineHost)
         engLog = '%s-eng-%s-' % (logfile,engineHost)
         for i in range(numEngines):
             cmd = "ssh %s '%s' 'ipengine --controller-ip %s --logfile %s' &" % \
                       (engineHost,sshx,contHost,engLog)
             #print 'cmd:<%s>' % cmd  # dbg
             xsys(cmd)
         # Wait after each host a little bit
         time.sleep(1)
             
     startMsg(contConfig['host'])
         
 def main():
     """Main driver for the two big options: local or remote cluster."""
     
     opt,arg = parse_args()
 
     clusterfile = opt.clusterfile
     if clusterfile:
         clusterRemote(opt,arg)
     else:
         clusterLocal(opt,arg)
         
             
 if __name__=='__main__':
     main()

IPython/kernel/scripts/ipengine.py

@@ -1,169 +1,171 b''
 #!/usr/bin/env python
 # encoding: utf-8
 
 """Start the IPython Engine."""
 
 __docformat__ = "restructuredtext en"
 
 #-------------------------------------------------------------------------------
 #  Copyright (C) 2008  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-------------------------------------------------------------------------------
 
 #-------------------------------------------------------------------------------
 # Imports
 #-------------------------------------------------------------------------------
 
 # Python looks for an empty string at the beginning of sys.path to enable
 # importing from the cwd.
 import sys
 sys.path.insert(0, '')
 
 import sys, os
 from optparse import OptionParser
 
 from twisted.application import service
 from twisted.internet import reactor
 from twisted.python import log
 
 from IPython.kernel.fcutil import Tub, UnauthenticatedTub
 
 from IPython.kernel.core.config import config_manager as core_config_manager
 from IPython.config.cutils import import_item
 from IPython.kernel.engineservice import EngineService
 from IPython.kernel.config import config_manager as kernel_config_manager
 from IPython.kernel.engineconnector import EngineConnector
 
 
 #-------------------------------------------------------------------------------
 # Code
 #-------------------------------------------------------------------------------
 
 def start_engine():
     """
     Start the engine, by creating it and starting the Twisted reactor.
     
     This method does:
     
         * If it exists, runs the `mpi_import_statement` to call `MPI_Init`
         * Starts the engine logging
         * Creates an IPython shell and wraps it in an `EngineService`
         * Creates a `foolscap.Tub` to use in connecting to a controller.
         * Uses the tub and the `EngineService` along with a Foolscap URL
           (or FURL) to connect to the controller and register the engine
           with the controller
     """
     kernel_config = kernel_config_manager.get_config_obj()
     core_config = core_config_manager.get_config_obj()
     
+
     # Execute the mpi import statement that needs to call MPI_Init
+    global mpi
     mpikey = kernel_config['mpi']['default']
     mpi_import_statement = kernel_config['mpi'].get(mpikey, None)
     if mpi_import_statement is not None:
         try:
-            exec mpi_import_statement in locals(), globals()
+            exec mpi_import_statement in globals()
         except:
             mpi = None
     else:
         mpi = None
     
     # Start logging
     logfile = kernel_config['engine']['logfile']
     if logfile:
         logfile = logfile + str(os.getpid()) + '.log'
         try:
             openLogFile = open(logfile, 'w')
         except:
             openLogFile = sys.stdout
     else:
         openLogFile = sys.stdout
     log.startLogging(openLogFile)
     
     # Create the underlying shell class and EngineService
     shell_class = import_item(core_config['shell']['shell_class'])
     engine_service = EngineService(shell_class, mpi=mpi)
     shell_import_statement = core_config['shell']['import_statement']
     if shell_import_statement:
         try:
             engine_service.execute(shell_import_statement)
         except:
             log.msg("Error running import_statement: %s" % sis)
     
     # Create the service hierarchy
     main_service = service.MultiService()
     engine_service.setServiceParent(main_service)
     tub_service = Tub()
     tub_service.setServiceParent(main_service)
     # This needs to be called before the connection is initiated
     main_service.startService()
     
     # This initiates the connection to the controller and calls
     # register_engine to tell the controller we are ready to do work
     engine_connector = EngineConnector(tub_service)
     furl_file = kernel_config['engine']['furl_file']
     d = engine_connector.connect_to_controller(engine_service, furl_file)
     d.addErrback(lambda _: reactor.stop())
     
     reactor.run()
 
 
 def init_config():
     """
     Initialize the configuration using default and command line options.
     """
     
     parser = OptionParser()
     
     parser.add_option(
         "--furl-file",
         type="string", 
         dest="furl_file",
         help="The filename containing the FURL of the controller"
     )
     parser.add_option(
         "--mpi",
         type="string",
         dest="mpi",
         help="How to enable MPI (mpi4py, pytrilinos, or empty string to disable)"
     )
     parser.add_option(
         "-l",
         "--logfile",
         type="string",
         dest="logfile",
         help="log file name (default is stdout)"
     )
     parser.add_option(
         "--ipythondir",
         type="string",
         dest="ipythondir",
         help="look for config files and profiles in this directory"
     )
     
     (options, args) = parser.parse_args()
     
     kernel_config_manager.update_config_obj_from_default_file(options.ipythondir)
     core_config_manager.update_config_obj_from_default_file(options.ipythondir)
     
     kernel_config = kernel_config_manager.get_config_obj()
     # Now override with command line options
     if options.furl_file is not None:
         kernel_config['engine']['furl_file'] = options.furl_file
     if options.logfile is not None:
         kernel_config['engine']['logfile'] = options.logfile
     if options.mpi is not None:
         kernel_config['mpi']['default'] = options.mpi
 
 
 def main():
     """
     After creating the configuration information, start the engine.
     """
     init_config()
     start_engine()
 
 
 if __name__ == "__main__":
     main()
  No newline at end of file

IPython/kernel/task.py

@@ -1,799 +1,1113 b''
 # encoding: utf-8
 # -*- test-case-name: IPython.kernel.tests.test_task -*-
 
 """Task farming representation of the ControllerService."""
 
 __docformat__ = "restructuredtext en"
 
-#-------------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
 #  Copyright (C) 2008  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
-#-------------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
 
-#-------------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
 # Imports
-#-------------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
 
 import copy, time
-from types import FunctionType as function
+from types import FunctionType
 
 import zope.interface as zi, string
 from twisted.internet import defer, reactor
 from twisted.python import components, log, failure
 
-# from IPython.genutils import time
-
+from IPython.kernel.util import printer
 from IPython.kernel import engineservice as es, error
 from IPython.kernel import controllerservice as cs
 from IPython.kernel.twistedutil import gatherBoth, DeferredList
 
 from IPython.kernel.pickleutil import can,uncan, CannedFunction
 
-def canTask(task):
-    t = copy.copy(task)
-    t.depend = can(t.depend)
-    if t.recovery_task:
-        t.recovery_task = canTask(t.recovery_task)
-    return t
-
-def uncanTask(task):
-    t = copy.copy(task)
-    t.depend = uncan(t.depend)
-    if t.recovery_task and t.recovery_task is not task:
-        t.recovery_task = uncanTask(t.recovery_task)
-    return t
+#-----------------------------------------------------------------------------
+# Definition of the Task objects
+#-----------------------------------------------------------------------------
 
 time_format = '%Y/%m/%d %H:%M:%S'
 
-class Task(object):
-    """Our representation of a task for the `TaskController` interface.
+class ITask(zi.Interface):
+    """
+    This interface provides a generic definition of what constitutes a task.
+    
+    There are two sides to a task.  First a task needs to take input from
+    a user to determine what work is performed by the task.  Second, the
+    task needs to have the logic that knows how to turn that information
+    info specific calls to a worker, through the `IQueuedEngine` interface.
+    
+    Many method in this class get two things passed to them: a Deferred
+    and an IQueuedEngine implementer.  Such methods should register callbacks
+    on the Deferred that use the IQueuedEngine to accomplish something.  See
+    the existing task objects for examples.
+    """
+    
+    zi.Attribute('retries','How many times to retry the task')
+    zi.Attribute('recovery_task','A task to try if the initial one fails')
+    zi.Attribute('taskid','the id of the task')
+    
+    def start_time(result):
+        """
+        Do anything needed to start the timing of the task.
+        
+        Must simply return the result after starting the timers.
+        """
+    
+    def stop_time(result):
+        """
+        Do anything needed to stop the timing of the task.
+        
+        Must simply return the result after stopping the timers.  This 
+        method will usually set attributes that are used by `process_result`
+        in building result of the task.
+        """
+    
+    def pre_task(d, queued_engine):
+        """Do something with the queued_engine before the task is run.
+        
+        This method should simply add callbacks to the input Deferred
+        that do something with the `queued_engine` before the task is run.
+        
+        :Parameters:
+            d : Deferred
+                The deferred that actions should be attached to
+            queued_engine : IQueuedEngine implementer
+                The worker that has been allocated to perform the task
+        """
+    
+    def post_task(d, queued_engine):
+        """Do something with the queued_engine after the task is run.
+        
+        This method should simply add callbacks to the input Deferred
+        that do something with the `queued_engine` before the task is run.
+        
+        :Parameters:
+            d : Deferred
+                The deferred that actions should be attached to
+            queued_engine : IQueuedEngine implementer
+                The worker that has been allocated to perform the task
+        """
+    
+    def submit_task(d, queued_engine):
+        """Submit a task using the `queued_engine` we have been allocated.
+        
+        When a task is ready to run, this method is called.  This method
+        must take the internal information of the task and make suitable
+        calls on the queued_engine to have the actual work done.
+        
+        This method should simply add callbacks to the input Deferred
+        that do something with the `queued_engine` before the task is run.
+        
+        :Parameters:
+            d : Deferred
+                The deferred that actions should be attached to
+            queued_engine : IQueuedEngine implementer
+                The worker that has been allocated to perform the task
+        """
+
+    def process_result(d, result, engine_id):
+        """Take a raw task result.
+        
+        Objects that implement `ITask` can choose how the result of running
+        the task is presented.  This method takes the raw result and 
+        does this logic.  Two example are the `MapTask` which simply returns
+        the raw result or a `Failure` object and the `StringTask` which
+        returns a `TaskResult` object.
+        
+        :Parameters:
+            d : Deferred
+                The deferred that actions should be attached to
+            result : object
+                The raw task result that needs to be wrapped
+            engine_id : int
+                The id of the engine that did the task
+        
+        :Returns:
+            The result, as a tuple of the form: (success, result).
+            Here, success is a boolean indicating if the task
+            succeeded or failed and result is the result.
+        """
 
-    The user should create instances of this class to represent a task that 
-    needs to be done.
+    def check_depend(properties):
+        """Check properties to see if the task should be run.
+        
+        :Parameters:
+            properties : dict
+                A dictionary of properties that an engine has set
+        
+        :Returns:
+            True if the task should be run, False otherwise
+        """
+    
+    def can_task(self):
+        """Serialize (can) any functions in the task for pickling.
+        
+        Subclasses must override this method and make sure that all 
+        functions in the task are canned by calling `can` on the 
+        function.
+        """
+    
+    def uncan_task(self):
+        """Unserialize (uncan) any canned function in the task."""
+
+class BaseTask(object):
+    """
+    Common fuctionality for all objects implementing `ITask`.
+    """
+    
+    zi.implements(ITask)
+    
+    def __init__(self, clear_before=False, clear_after=False, retries=0,
+            recovery_task=None, depend=None):
+        """
+        Make a generic task.
     
     :Parameters:
-        expression : str
-            A str that is valid python code that is the task.
-        pull : str or list of str 
-            The names of objects to be pulled as results.  If not specified, 
-            will return {'result', None}
-        push : dict
-            A dict of objects to be pushed into the engines namespace before
-            execution of the expression.
         clear_before : boolean
-            Should the engine's namespace be cleared before the task is run.
-            Default=False.
+                Should the engines namespace be cleared before the task
+                is run
         clear_after : boolean 
-            Should the engine's namespace be cleared after the task is run.
-            Default=False.
+                Should the engines namespace be clear after the task is run
         retries : int
-            The number of times to resumbit the task if it fails.  Default=0.
-        recovery_task : Task
-            This is the Task to be run when the task has exhausted its retries
-            Default=None.
-        depend : bool function(properties)
-            This is the dependency function for the Task, which determines
-            whether a task can be run on a Worker.  `depend` is called with
-            one argument, the worker's properties dict, and should return
-            True if the worker meets the dependencies or False if it does
-            not.
-            Default=None - run on any worker
-        options : dict
-            Any other keyword options for more elaborate uses of tasks
-    
-    Examples
-    --------
+                The number of times a task should be retries upon failure
+            recovery_task : any task object
+                If a task fails and it has a recovery_task, that is run
+                upon a retry
+            depend : FunctionType
+                A function that is called to test for properties.  This function
+                must take one argument, the properties dict and return a boolean
+        """
+        self.clear_before = clear_before
+        self.clear_after = clear_after
+        self.retries = retries
+        self.recovery_task = recovery_task
+        self.depend = depend
+        self.taskid = None
+    
+    def start_time(self, result):
+        """
+        Start the basic timers.
+        """
+        self.start = time.time()
+        self.start_struct = time.localtime()
+        return result
+    
+    def stop_time(self, result):
+        """
+        Stop the basic timers.
+        """
+        self.stop = time.time()
+        self.stop_struct = time.localtime()
+        self.duration = self.stop - self.start 
+        self.submitted = time.strftime(time_format, self.start_struct)
+        self.completed = time.strftime(time_format)
+        return result
+    
+    def pre_task(self, d, queued_engine):
+        """
+        Clear the engine before running the task if clear_before is set.
+        """
+        if self.clear_before:
+            d.addCallback(lambda r: queued_engine.reset())
+    
+    def post_task(self, d, queued_engine):
+        """
+        Clear the engine after running the task if clear_after is set.
+        """
+        def reseter(result):
+            queued_engine.reset()
+            return result
+        if self.clear_after:
+            d.addBoth(reseter)
+    
+    def submit_task(self, d, queued_engine):
+        raise NotImplementedError('submit_task must be implemented in a subclass')
+    
+    def process_result(self, result, engine_id):
+        """
+        Process a task result.
+        
+        This is the default `process_result` that just returns the raw
+        result or a `Failure`.
+        """
+        if isinstance(result, failure.Failure):
+            return (False, result)
+        else:
+            return (True, result)
+    
+    def check_depend(self, properties):
+        """
+        Calls self.depend(properties) to see if a task should be run.
+        """
+        if self.depend is not None:
+            return self.depend(properties)
+        else:
+            return True
+
+    def can_task(self):
+        self.depend = can(self.depend)
+        if isinstance(self.recovery_task, BaseTask):
+            self.recovery_task.can_task()
+            
+    def uncan_task(self):
+        self.depend = uncan(self.depend)
+        if isinstance(self.recovery_task, BaseTask):
+            self.recovery_task.uncan_task()
+
+class MapTask(BaseTask):
+    """
+    A task that consists of a function and arguments.
+    """
     
-    >>> t = Task('dostuff(args)')
-    >>> t = Task('a=5', pull='a')
-    >>> t = Task('a=5\nb=4', pull=['a','b'])
-    >>> t = Task('os.kill(os.getpid(),9)', retries=100) # this is a bad idea
-        # A dependency case:
-    >>> def hasMPI(props):
-    ...     return props.get('mpi') is not None
-    >>> t = Task('mpi.send(blah,blah)', depend = hasMPI)
+    zi.implements(ITask)
+    
+    def __init__(self, function, args=None, kwargs=None, clear_before=False, 
+            clear_after=False, retries=0, recovery_task=None, depend=None):
+        """
+        Create a task based on a function, args and kwargs.
+        
+        This is a simple type of task that consists of calling:
+        function(*args, **kwargs) and wrapping the result in a `TaskResult`.
+        
+        The return value of the function, or a `Failure` wrapping an 
+        exception is the task result for this type of task.
+        """
+        BaseTask.__init__(self, clear_before, clear_after, retries, 
+            recovery_task, depend)
+        if not isinstance(function, FunctionType):
+            raise TypeError('a task function must be a FunctionType')
+        self.function = function
+        if args is None:
+            self.args = ()
+        else:
+            self.args = args
+        if not isinstance(self.args, (list, tuple)):
+            raise TypeError('a task args must be a list or tuple')
+        if kwargs is None:
+            self.kwargs = {}
+        else:
+            self.kwargs = kwargs
+        if not isinstance(self.kwargs, dict):
+            raise TypeError('a task kwargs must be a dict')
+    
+    def submit_task(self, d, queued_engine):
+        d.addCallback(lambda r: queued_engine.push_function(
+            dict(_ipython_task_function=self.function))
+        )
+        d.addCallback(lambda r: queued_engine.push(
+            dict(_ipython_task_args=self.args,_ipython_task_kwargs=self.kwargs))
+        )
+        d.addCallback(lambda r: queued_engine.execute(
+            '_ipython_task_result = _ipython_task_function(*_ipython_task_args,**_ipython_task_kwargs)')
+        )
+        d.addCallback(lambda r: queued_engine.pull('_ipython_task_result'))
+    
+    def can_task(self):
+        self.function = can(self.function)
+        BaseTask.can_task(self)
+    
+    def uncan_task(self):
+        self.function = uncan(self.function)
+        BaseTask.uncan_task(self)
+
+
+class StringTask(BaseTask):
+    """
+    A task that consists of a string of Python code to run.
     """
     
     def __init__(self, expression, pull=None, push=None,
             clear_before=False, clear_after=False, retries=0, 
-            recovery_task=None, depend=None, **options):
+            recovery_task=None, depend=None):
+        """
+        Create a task based on a Python expression and variables
+        
+        This type of task lets you push a set of variables to the engines
+        namespace, run a Python string in that namespace and then bring back
+        a different set of Python variables as the result.
+        
+        Because this type of task can return many results (through the
+        `pull` keyword argument) it returns a special `TaskResult` object
+        that wraps the pulled variables, statistics about the run and
+        any exceptions raised.
+        """
+        if not isinstance(expression, str):
+            raise TypeError('a task expression must be a string')
         self.expression = expression
-        if isinstance(pull, str):
-            self.pull = [pull]
-        else:
+        
+        if pull==None:
+            self.pull = ()
+        elif isinstance(pull, str):
+            self.pull = (pull,)
+        elif isinstance(pull, (list, tuple)):
             self.pull = pull
+        else:
+            raise TypeError('pull must be str or a sequence of strs')
+        
+        if push==None:
+            self.push = {}
+        elif isinstance(push, dict):
         self.push = push
-        self.clear_before = clear_before
-        self.clear_after = clear_after
-        self.retries=retries
-        self.recovery_task = recovery_task
-        self.depend = depend
-        self.options = options
-        self.taskid = None
+        else:
+            raise TypeError('push must be a dict')
+        
+        BaseTask.__init__(self, clear_before, clear_after, retries, 
+            recovery_task, depend)
 
-class ResultNS:
-    """The result namespace object for use in TaskResult objects as tr.ns.
+    def submit_task(self, d, queued_engine):
+        if self.push is not None:
+            d.addCallback(lambda r: queued_engine.push(self.push))
+        
+        d.addCallback(lambda r: queued_engine.execute(self.expression))
+        
+        if self.pull is not None:
+            d.addCallback(lambda r: queued_engine.pull(self.pull))
+        else:
+            d.addCallback(lambda r: None)    
+    
+    def process_result(self, result, engine_id):
+        if isinstance(result, failure.Failure):
+            tr = TaskResult(result, engine_id)
+        else:
+            if self.pull is None:
+                resultDict = {}
+            elif len(self.pull) == 1:
+                resultDict = {self.pull[0]:result}
+            else:
+                resultDict = dict(zip(self.pull, result))
+            tr = TaskResult(resultDict, engine_id)
+        # Assign task attributes
+        tr.submitted = self.submitted
+        tr.completed = self.completed
+        tr.duration = self.duration
+        if hasattr(self,'taskid'):
+            tr.taskid = self.taskid
+        else:
+            tr.taskid = None
+        if isinstance(result, failure.Failure):
+            return (False, tr)
+        else:
+            return (True, tr)
+
+class ResultNS(object):
+    """
+    A dict like object for holding the results of a task.
+    
+    The result namespace object for use in `TaskResult` objects as tr.ns.
     It builds an object from a dictionary, such that it has attributes
     according to the key,value pairs of the dictionary.
     
     This works by calling setattr on ALL key,value pairs in the dict.  If a user
     chooses to overwrite the `__repr__` or `__getattr__` attributes, they can.
     This can be a bad idea, as it may corrupt standard behavior of the
     ns object.
     
     Example
     --------
     
     >>> ns = ResultNS({'a':17,'foo':range(3)})
     >>> print ns
         NS{'a':17,'foo':range(3)}
     >>> ns.a
         17
     >>> ns['foo']
         [0,1,2]
     """
     def __init__(self, dikt):
         for k,v in dikt.iteritems():
             setattr(self,k,v)
     
     def __repr__(self):
         l = dir(self)
         d = {}
         for k in l:
             # do not print private objects
             if k[:2] != '__' and k[-2:] != '__':
                 d[k] = getattr(self, k)
         return "NS"+repr(d)
     
     def __getitem__(self, key):
         return getattr(self, key)
 
 class TaskResult(object):
     """
-    An object for returning task results.
+    An object for returning task results for certain types of tasks.
         
     This object encapsulates the results of a task.  On task
     success it will have a keys attribute that will have a list
     of the variables that have been pulled back.  These variables
     are accessible as attributes of this class as well.  On 
     success the failure attribute will be None.
         
     In task failure, keys will be empty, but failure will contain
     the failure object that encapsulates the remote exception.
-    One can also simply call the raiseException() method of 
+    One can also simply call the `raise_exception` method of 
     this class to re-raise any remote exception in the local
     session.
         
-    The TaskResult has a .ns member, which is a property for access
+    The `TaskResult` has a `.ns` member, which is a property for access
     to the results.  If the Task had pull=['a', 'b'], then the 
-    Task Result will have attributes tr.ns.a, tr.ns.b for those values.
-    Accessing tr.ns will raise the remote failure if the task failed.
+    Task Result will have attributes `tr.ns.a`, `tr.ns.b` for those values.
+    Accessing `tr.ns` will raise the remote failure if the task failed.
         
-    The engineid attribute should have the engineid of the engine
-    that ran the task.  But, because engines can come and go in
-    the ipython task system, the engineid may not continue to be
+    The `engineid` attribute should have the `engineid` of the engine
+    that ran the task.  But, because engines can come and go, 
+    the `engineid` may not continue to be
     valid or accurate.
         
-    The taskid attribute simply gives the taskid that the task
+    The `taskid` attribute simply gives the `taskid` that the task
     is tracked under.
     """
     taskid = None
     
     def _getNS(self):
         if isinstance(self.failure, failure.Failure):
             return self.failure.raiseException()
         else:
             return self._ns
     
     def _setNS(self, v):
-        raise Exception("I am protected!")
+        raise Exception("the ns attribute cannot be changed")
     
     ns = property(_getNS, _setNS)
     
     def __init__(self, results, engineid):
         self.engineid = engineid
         if isinstance(results, failure.Failure):
             self.failure = results
             self.results = {}
         else:
             self.results = results
             self.failure = None
         
         self._ns = ResultNS(self.results)
         
         self.keys = self.results.keys()
     
     def __repr__(self):
         if self.failure is not None:
             contents = self.failure
         else:
             contents = self.results
         return "TaskResult[ID:%r]:%r"%(self.taskid, contents)
     
     def __getitem__(self, key):
         if self.failure is not None:
-            self.raiseException()
+            self.raise_exception()
         return self.results[key]
     
-    def raiseException(self):
+    def raise_exception(self):
         """Re-raise any remote exceptions in the local python session."""
         if self.failure is not None:
             self.failure.raiseException()
 
 
+#-----------------------------------------------------------------------------
+# The controller side of things
+#-----------------------------------------------------------------------------
+
 class IWorker(zi.Interface):
     """The Basic Worker Interface. 
     
     A worked is a representation of an Engine that is ready to run tasks.
     """
     
     zi.Attribute("workerid", "the id of the worker")
     
     def run(task):
         """Run task in worker's namespace.
         
         :Parameters:
             task : a `Task` object
         
-        :Returns: `Deferred` to a `TaskResult` object.
+        :Returns: `Deferred` to a tuple of (success, result) where
+            success if a boolean that signifies success or failure
+            and result is the task result.
         """
 
 
 class WorkerFromQueuedEngine(object):
     """Adapt an `IQueuedEngine` to an `IWorker` object"""
+    
     zi.implements(IWorker)
     
     def __init__(self, qe):
         self.queuedEngine = qe
         self.workerid = None
     
     def _get_properties(self):
         return self.queuedEngine.properties
     
     properties = property(_get_properties, lambda self, _:None)
     
     def run(self, task):
         """Run task in worker's namespace.
         
+        This takes a task and calls methods on the task that actually
+        cause `self.queuedEngine` to do the task.  See the methods of
+        `ITask` for more information about how these methods are called.
+        
         :Parameters:
             task : a `Task` object
         
-        :Returns: `Deferred` to a `TaskResult` object.
+        :Returns: `Deferred` to a tuple of (success, result) where
+            success if a boolean that signifies success or failure
+            and result is the task result.
         """
-        if task.clear_before:
-            d = self.queuedEngine.reset()
-        else:
             d = defer.succeed(None)
-            
-        if task.push is not None:
-            d.addCallback(lambda r: self.queuedEngine.push(task.push))
-        
-        d.addCallback(lambda r: self.queuedEngine.execute(task.expression))
-        
-        if task.pull is not None:
-            d.addCallback(lambda r: self.queuedEngine.pull(task.pull))
-        else:
-            d.addCallback(lambda r: None)
-        
-        def reseter(result):
-            self.queuedEngine.reset()
-            return result
-            
-        if task.clear_after:
-            d.addBoth(reseter)
-        
-        return d.addBoth(self._zipResults, task.pull, time.time(), time.localtime())
-    
-    def _zipResults(self, result, names, start, start_struct):
-        """Callback for construting the TaskResult object."""
-        if isinstance(result, failure.Failure):
-            tr = TaskResult(result, self.queuedEngine.id)
-        else:
-            if names is None:
-                resultDict = {} 
-            elif len(names) == 1:
-                resultDict = {names[0]:result}
-            else:
-                resultDict = dict(zip(names, result))
-            tr = TaskResult(resultDict, self.queuedEngine.id)
-        # the time info
-        tr.submitted = time.strftime(time_format, start_struct)
-        tr.completed = time.strftime(time_format)
-        tr.duration = time.time()-start
-        return tr
+        d.addCallback(task.start_time)
+        task.pre_task(d, self.queuedEngine)
+        task.submit_task(d, self.queuedEngine)
+        task.post_task(d, self.queuedEngine)
+        d.addBoth(task.stop_time)
+        d.addBoth(task.process_result, self.queuedEngine.id)
+        # At this point, there will be (success, result) coming down the line
+        return d
     
 
 components.registerAdapter(WorkerFromQueuedEngine, es.IEngineQueued, IWorker)
 
 class IScheduler(zi.Interface):
     """The interface for a Scheduler.
     """
     zi.Attribute("nworkers", "the number of unassigned workers")
     zi.Attribute("ntasks", "the number of unscheduled tasks")
     zi.Attribute("workerids", "a list of the worker ids")
     zi.Attribute("taskids", "a list of the task ids")
     
     def add_task(task, **flags):
         """Add a task to the queue of the Scheduler.
         
         :Parameters:
-            task : a `Task` object
+            task : an `ITask` implementer
                 The task to be queued.
             flags : dict
                 General keywords for more sophisticated scheduling
         """
     
     def pop_task(id=None):
-        """Pops a Task object.
+        """Pops a task object from the queue.
         
         This gets the next task to be run.  If no `id` is requested, the highest priority
         task is returned.
         
         :Parameters:
             id
                 The id of the task to be popped.  The default (None) is to return 
                 the highest priority task.
                 
-        :Returns: a `Task` object
+        :Returns: an `ITask` implementer
         
         :Exceptions:
             IndexError : raised if no taskid in queue
         """
     
     def add_worker(worker, **flags):
         """Add a worker to the worker queue.
         
         :Parameters:
-            worker : an IWorker implementing object
-            flags : General keywords for more sophisticated scheduling
+            worker : an `IWorker` implementer
+            flags : dict
+                General keywords for more sophisticated scheduling
         """
     
     def pop_worker(id=None):
         """Pops an IWorker object that is ready to do work.
         
         This gets the next IWorker that is ready to do work. 
         
         :Parameters:
             id : if specified, will pop worker with workerid=id, else pops
                  highest priority worker.  Defaults to None.
         
         :Returns:
             an IWorker object
         
         :Exceptions:
             IndexError : raised if no workerid in queue
         """
     
     def ready():
         """Returns True if there is something to do, False otherwise"""
     
     def schedule():
-        """Returns a tuple of the worker and task pair for the next
-        task to be run.
-        """
+        """Returns (worker,task) pair for the next task to be run."""
     
 
 class FIFOScheduler(object):
-    """A basic First-In-First-Out (Queue) Scheduler.
-    This is the default Scheduler for the TaskController.
-    See the docstrings for IScheduler for interface details.
+    """
+    A basic First-In-First-Out (Queue) Scheduler.
+    
+    This is the default Scheduler for the `TaskController`.
+    See the docstrings for `IScheduler` for interface details.
     """
     
     zi.implements(IScheduler)
     
     def __init__(self):
         self.tasks = []
         self.workers = []
     
     def _ntasks(self):
         return len(self.tasks)
     
     def _nworkers(self):
         return len(self.workers)
     
     ntasks = property(_ntasks, lambda self, _:None)
     nworkers = property(_nworkers, lambda self, _:None)
     
     def _taskids(self):
         return [t.taskid for t in self.tasks]
     
     def _workerids(self):
         return [w.workerid for w in self.workers]
     
     taskids = property(_taskids, lambda self,_:None)
     workerids = property(_workerids, lambda self,_:None)
     
     def add_task(self, task, **flags):    
         self.tasks.append(task)
     
     def pop_task(self, id=None):
         if id is None:
             return self.tasks.pop(0)
         else:
             for i in range(len(self.tasks)):
                 taskid = self.tasks[i].taskid
                 if id == taskid:
                     return self.tasks.pop(i)
             raise IndexError("No task #%i"%id)
     
     def add_worker(self, worker, **flags):
         self.workers.append(worker)
     
     def pop_worker(self, id=None):
         if id is None:
             return self.workers.pop(0)
         else:
             for i in range(len(self.workers)):
                 workerid = self.workers[i].workerid
                 if id == workerid:
                     return self.workers.pop(i)
             raise IndexError("No worker #%i"%id)
     
     def schedule(self):
         for t in self.tasks:
             for w in self.workers:
                 try:# do not allow exceptions to break this
-                    cando = t.depend is None or t.depend(w.properties)
+                    # Allow the task to check itself using its
+                    # check_depend method.
+                    cando = t.check_depend(w.properties)
                 except:
                     cando = False
                 if cando:
                     return self.pop_worker(w.workerid), self.pop_task(t.taskid)
         return None, None
     
 
 
 class LIFOScheduler(FIFOScheduler):
-    """A Last-In-First-Out (Stack) Scheduler.  This scheduler should naively
-    reward fast engines by giving them more jobs.  This risks starvation, but
-    only in cases with low load, where starvation does not really matter.
+    """
+    A Last-In-First-Out (Stack) Scheduler.
+    
+    This scheduler should naively reward fast engines by giving 
+    them more jobs.  This risks starvation, but only in cases with 
+    low load, where starvation does not really matter.
     """
     
     def add_task(self, task, **flags):
         # self.tasks.reverse()
         self.tasks.insert(0, task)
         # self.tasks.reverse()
     
     def add_worker(self, worker, **flags):
         # self.workers.reverse()
         self.workers.insert(0, worker)
         # self.workers.reverse()
     
 
 class ITaskController(cs.IControllerBase):
-    """The Task based interface to a `ControllerService` object
+    """
+    The Task based interface to a `ControllerService` object
     
     This adapts a `ControllerService` to the ITaskController interface.
     """
     
     def run(task):
-        """Run a task.
+        """
+        Run a task.
         
         :Parameters:
             task : an IPython `Task` object
         
         :Returns: the integer ID of the task
         """
     
     def get_task_result(taskid, block=False):
-        """Get the result of a task by its ID.
+        """
+        Get the result of a task by its ID.
         
         :Parameters:
             taskid : int
                 the id of the task whose result is requested
         
-        :Returns: `Deferred` to (taskid, actualResult) if the task is done, and None
+        :Returns: `Deferred` to the task result if the task is done, and None
             if not.
         
         :Exceptions:
             actualResult will be an `IndexError` if no such task has been submitted
         """
     
     def abort(taskid):
         """Remove task from queue if task is has not been submitted.
         
         If the task has already been submitted, wait for it to finish and discard 
         results and prevent resubmission.
         
         :Parameters:
             taskid : the id of the task to be aborted
         
         :Returns:
             `Deferred` to abort attempt completion.  Will be None on success.
         
         :Exceptions:
             deferred will fail with `IndexError` if no such task has been submitted
             or the task has already completed.
         """
     
     def barrier(taskids):
-        """Block until the list of taskids are completed.
+        """
+        Block until the list of taskids are completed.
         
         Returns None on success.
         """
     
     def spin():
-        """touch the scheduler, to resume scheduling without submitting
-        a task.
+        """
+        Touch the scheduler, to resume scheduling without submitting a task.
         """
     
-    def queue_status(self, verbose=False):
-        """Get a dictionary with the current state of the task queue.
+    def queue_status(verbose=False):
+        """
+        Get a dictionary with the current state of the task queue.
         
         If verbose is True, then return lists of taskids, otherwise, 
         return the number of tasks with each status.
         """
     
+    def clear():
+        """
+        Clear all previously run tasks from the task controller.
+        
+        This is needed because the task controller keep all task results
+        in memory.  This can be a problem is there are many completed
+        tasks.  Users should call this periodically to clean out these
+        cached task results.
+        """
+    
 
 class TaskController(cs.ControllerAdapterBase):
     """The Task based interface to a Controller object.
     
     If you want to use a different scheduler, just subclass this and set
     the `SchedulerClass` member to the *class* of your chosen scheduler.
     """
     
     zi.implements(ITaskController)
     SchedulerClass = FIFOScheduler
     
     timeout = 30
     
     def __init__(self, controller):
         self.controller = controller
         self.controller.on_register_engine_do(self.registerWorker, True)
         self.controller.on_unregister_engine_do(self.unregisterWorker, True)
         self.taskid = 0
         self.failurePenalty = 1 # the time in seconds to penalize
                                 # a worker for failing a task
         self.pendingTasks = {} # dict of {workerid:(taskid, task)}
         self.deferredResults = {} # dict of {taskid:deferred}
         self.finishedResults = {} # dict of {taskid:actualResult}
         self.workers = {} # dict of {workerid:worker}
         self.abortPending = [] # dict of {taskid:abortDeferred}
         self.idleLater = None # delayed call object for timeout
         self.scheduler = self.SchedulerClass()
         
         for id in self.controller.engines.keys():
                 self.workers[id] = IWorker(self.controller.engines[id])
                 self.workers[id].workerid = id
                 self.schedule.add_worker(self.workers[id])
     
     def registerWorker(self, id):
         """Called by controller.register_engine."""
         if self.workers.get(id):
-            raise "We already have one!  This should not happen."
+            raise ValueError("worker with id %s already exists.  This should not happen." % id)
         self.workers[id] = IWorker(self.controller.engines[id])
         self.workers[id].workerid = id
         if not self.pendingTasks.has_key(id):# if not working
             self.scheduler.add_worker(self.workers[id])
         self.distributeTasks()
     
     def unregisterWorker(self, id):
         """Called by controller.unregister_engine"""
         
         if self.workers.has_key(id):
             try:
                 self.scheduler.pop_worker(id)
             except IndexError:
                 pass
             self.workers.pop(id)
     
     def _pendingTaskIDs(self):
         return [t.taskid for t in self.pendingTasks.values()]
     
     #---------------------------------------------------------------------------
     # Interface methods
     #---------------------------------------------------------------------------
     
     def run(self, task):
-        """Run a task and return `Deferred` to its taskid."""
+        """
+        Run a task and return `Deferred` to its taskid.
+        """
         task.taskid = self.taskid
         task.start = time.localtime()
         self.taskid += 1
         d = defer.Deferred()
         self.scheduler.add_task(task)
-        # log.msg('Queuing task: %i' % task.taskid)
+        log.msg('Queuing task: %i' % task.taskid)
         
         self.deferredResults[task.taskid] = []
         self.distributeTasks()
         return defer.succeed(task.taskid)
     
     def get_task_result(self, taskid, block=False):
-        """Returns a `Deferred` to a TaskResult tuple or None."""
-        # log.msg("Getting task result: %i" % taskid)
+        """
+        Returns a `Deferred` to the task result, or None.
+        """
+        log.msg("Getting task result: %i" % taskid)
         if self.finishedResults.has_key(taskid):
             tr = self.finishedResults[taskid]
             return defer.succeed(tr)
         elif self.deferredResults.has_key(taskid):
             if block:
                 d = defer.Deferred()
                 self.deferredResults[taskid].append(d)
                 return d
             else:
                 return defer.succeed(None)
         else:
             return defer.fail(IndexError("task ID not registered: %r" % taskid))
     
     def abort(self, taskid):
-        """Remove a task from the queue if it has not been run already."""
+        """
+        Remove a task from the queue if it has not been run already.
+        """
         if not isinstance(taskid, int):
             return defer.fail(failure.Failure(TypeError("an integer task id expected: %r" % taskid)))
         try:
             self.scheduler.pop_task(taskid)
         except IndexError, e:
             if taskid in self.finishedResults.keys():
                 d = defer.fail(IndexError("Task Already Completed"))
             elif taskid in self.abortPending:
                 d = defer.fail(IndexError("Task Already Aborted"))
             elif taskid in self._pendingTaskIDs():# task is pending
                 self.abortPending.append(taskid)
                 d = defer.succeed(None)
             else:
                 d = defer.fail(e)
         else:
             d = defer.execute(self._doAbort, taskid)
         
         return d
     
     def barrier(self, taskids):
         dList = []
         if isinstance(taskids, int):
             taskids = [taskids]
         for id in taskids:
             d = self.get_task_result(id, block=True)
             dList.append(d)
         d = DeferredList(dList, consumeErrors=1)
         d.addCallbacks(lambda r: None)
         return d
     
     def spin(self):
         return defer.succeed(self.distributeTasks())
     
     def queue_status(self, verbose=False):
         pending = self._pendingTaskIDs()
         failed = []
         succeeded = []
         for k,v in self.finishedResults.iteritems():
             if not isinstance(v, failure.Failure):
                 if hasattr(v,'failure'):
                     if v.failure is None:
                         succeeded.append(k)
                     else:
                         failed.append(k)
         scheduled = self.scheduler.taskids
         if verbose:
             result = dict(pending=pending, failed=failed, 
                 succeeded=succeeded, scheduled=scheduled)
         else:
             result = dict(pending=len(pending),failed=len(failed),
                 succeeded=len(succeeded),scheduled=len(scheduled))
         return defer.succeed(result)
     
     #---------------------------------------------------------------------------
     # Queue methods
     #---------------------------------------------------------------------------
     
     def _doAbort(self, taskid):
-        """Helper function for aborting a pending task."""
-        # log.msg("Task aborted: %i" % taskid)
+        """
+        Helper function for aborting a pending task.
+        """
+        log.msg("Task aborted: %i" % taskid)
         result = failure.Failure(error.TaskAborted())
         self._finishTask(taskid, result)
         if taskid in self.abortPending:
             self.abortPending.remove(taskid)
     
     def _finishTask(self, taskid, result):
         dlist = self.deferredResults.pop(taskid)
-        result.taskid = taskid   # The TaskResult should save the taskid
+        # result.taskid = taskid   # The TaskResult should save the taskid
         self.finishedResults[taskid] = result
         for d in dlist:
             d.callback(result)
     
     def distributeTasks(self):
-        """Distribute tasks while self.scheduler has things to do."""
-        # log.msg("distributing Tasks")
+        """
+        Distribute tasks while self.scheduler has things to do.
+        """
+        log.msg("distributing Tasks")
         worker, task = self.scheduler.schedule()
         if not worker and not task:
             if self.idleLater and self.idleLater.called:# we are inside failIdle
                 self.idleLater = None
             else:
                 self.checkIdle()
             return False
         # else something to do:
         while worker and task:
             # get worker and task
             # add to pending
             self.pendingTasks[worker.workerid] = task
             # run/link callbacks
             d = worker.run(task)
-            # log.msg("Running task %i on worker %i" %(task.taskid, worker.workerid))
+            log.msg("Running task %i on worker %i" %(task.taskid, worker.workerid))
             d.addBoth(self.taskCompleted, task.taskid, worker.workerid)
             worker, task = self.scheduler.schedule()
         # check for idle timeout:
         self.checkIdle()
         return True
     
     def checkIdle(self):
         if self.idleLater and not self.idleLater.called:
             self.idleLater.cancel()
         if self.scheduler.ntasks and self.workers and \
                     self.scheduler.nworkers == len(self.workers):
             self.idleLater = reactor.callLater(self.timeout, self.failIdle)
         else:
             self.idleLater = None
     
     def failIdle(self):
         if not self.distributeTasks():
             while self.scheduler.ntasks:
                 t = self.scheduler.pop_task()
                 msg = "task %i failed to execute due to unmet dependencies"%t.taskid
                 msg += " for %i seconds"%self.timeout
-                # log.msg("Task aborted by timeout: %i" % t.taskid)
+                log.msg("Task aborted by timeout: %i" % t.taskid)
                 f = failure.Failure(error.TaskTimeout(msg))
                 self._finishTask(t.taskid, f)
         self.idleLater = None
                 
     
-    def taskCompleted(self, result, taskid, workerid):
+    def taskCompleted(self, success_and_result, taskid, workerid):
         """This is the err/callback for a completed task."""
+        success, result = success_and_result
         try:
             task = self.pendingTasks.pop(workerid)
         except:
             # this should not happen
             log.msg("Tried to pop bad pending task %i from worker %i"%(taskid, workerid))
             log.msg("Result: %r"%result)
             log.msg("Pending tasks: %s"%self.pendingTasks)
             return
         
         # Check if aborted while pending
         aborted = False
         if taskid in self.abortPending:
             self._doAbort(taskid)
             aborted = True
         
         if not aborted:
-            if result.failure is not None and isinstance(result.failure, failure.Failure): # we failed
+            if not success:
                 log.msg("Task %i failed on worker %i"% (taskid, workerid))
                 if task.retries > 0: # resubmit
                     task.retries -= 1
                     self.scheduler.add_task(task)
                     s = "Resubmitting task %i, %i retries remaining" %(taskid, task.retries)
                     log.msg(s)
                     self.distributeTasks()
-                elif isinstance(task.recovery_task, Task) and \
+                elif isinstance(task.recovery_task, BaseTask) and \
                                     task.recovery_task.retries > -1:
                     # retries = -1 is to prevent infinite recovery_task loop
                     task.retries = -1 
                     task.recovery_task.taskid = taskid
                     task = task.recovery_task
                     self.scheduler.add_task(task)
                     s = "Recovering task %i, %i retries remaining" %(taskid, task.retries)
                     log.msg(s)
                     self.distributeTasks()
                 else: # done trying
                     self._finishTask(taskid, result)
                 # wait a second before readmitting a worker that failed
                 # it may have died, and not yet been unregistered
                 reactor.callLater(self.failurePenalty, self.readmitWorker, workerid)
             else: # we succeeded
-                # log.msg("Task completed: %i"% taskid)
+                log.msg("Task completed: %i"% taskid)
                 self._finishTask(taskid, result)
                 self.readmitWorker(workerid)
         else:# we aborted the task
-            if result.failure is not None and isinstance(result.failure, failure.Failure): # it failed, penalize worker
+            if not success:
                 reactor.callLater(self.failurePenalty, self.readmitWorker, workerid)
             else:
                 self.readmitWorker(workerid)
     
     def readmitWorker(self, workerid):
-        """Readmit a worker to the scheduler.  
+        """
+        Readmit a worker to the scheduler.  
         
         This is outside `taskCompleted` because of the `failurePenalty` being 
         implemented through `reactor.callLater`.
         """
         
         if workerid in self.workers.keys() and workerid not in self.pendingTasks.keys():
             self.scheduler.add_worker(self.workers[workerid])
             self.distributeTasks()
         
+    def clear(self):
+        """
+        Clear all previously run tasks from the task controller.
+        
+        This is needed because the task controller keep all task results
+        in memory.  This can be a problem is there are many completed
+        tasks.  Users should call this periodically to clean out these
+        cached task results.
+        """
+        self.finishedResults = {}
+        return defer.succeed(None)
+        
     
 components.registerAdapter(TaskController, cs.IControllerBase, ITaskController)

IPython/kernel/taskclient.py

@@ -1,161 +1,180 b''
 # encoding: utf-8
 # -*- test-case-name: IPython.kernel.tests.test_taskcontrollerxmlrpc -*-
 
-"""The Generic Task Client object.  
-
-This must be subclassed based on your connection method.
+"""
+A blocking version of the task client.
 """
 
 __docformat__ = "restructuredtext en"
 
 #-------------------------------------------------------------------------------
 #  Copyright (C) 2008  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-------------------------------------------------------------------------------
 
 #-------------------------------------------------------------------------------
 # Imports
 #-------------------------------------------------------------------------------
 
 from zope.interface import Interface, implements
 from twisted.python import components, log
 
 from IPython.kernel.twistedutil import blockingCallFromThread
 from IPython.kernel import task, error
+from IPython.kernel.mapper import (
+    SynchronousTaskMapper,
+    ITaskMapperFactory,
+    IMapper
+)
+from IPython.kernel.parallelfunction import (
+    ParallelFunction, 
+    ITaskParallelDecorator
+)
 
 #-------------------------------------------------------------------------------
-# Connecting Task Client
+# The task client
 #-------------------------------------------------------------------------------
 
-class InteractiveTaskClient(object):
-    
-    def irun(self, *args, **kwargs):
-        """Run a task on the `TaskController`.
-        
-        This method is a shorthand for run(task) and its arguments are simply
-        passed onto a `Task` object:
-        
-        irun(*args, **kwargs) -> run(Task(*args, **kwargs))
-
-        :Parameters:
-            expression : str
-                A str that is valid python code that is the task.
-            pull : str or list of str 
-                The names of objects to be pulled as results.
-            push : dict
-                A dict of objects to be pushed into the engines namespace before
-                execution of the expression.
-            clear_before : boolean
-                Should the engine's namespace be cleared before the task is run.
-                Default=False.
-            clear_after : boolean 
-                Should the engine's namespace be cleared after the task is run.
-                Default=False.
-            retries : int
-                The number of times to resumbit the task if it fails.  Default=0.
-            options : dict
-                Any other keyword options for more elaborate uses of tasks
-            
-        :Returns: A `TaskResult` object.      
-        """
-        block = kwargs.pop('block', False)
-        if len(args) == 1 and isinstance(args[0], task.Task):
-            t = args[0]
-        else:
-            t = task.Task(*args, **kwargs)
-        taskid = self.run(t)
-        print "TaskID = %i"%taskid
-        if block:
-            return self.get_task_result(taskid, block)
-        else:
-            return taskid
-
 class IBlockingTaskClient(Interface):
     """
-    An interface for blocking task clients.
+    A vague interface of the blocking task client
     """
     pass
 
-
-class BlockingTaskClient(InteractiveTaskClient):
+class BlockingTaskClient(object):
     """
-    This class provides a blocking task client.
+    A blocking task client that adapts a non-blocking one.
     """
     
-    implements(IBlockingTaskClient)
+    implements(
+        IBlockingTaskClient, 
+        ITaskMapperFactory,
+        IMapper,
+        ITaskParallelDecorator
+    )
     
     def __init__(self, task_controller):
         self.task_controller = task_controller
         self.block = True
         
-    def run(self, task):
-        """
-        Run a task and return a task id that can be used to get the task result.
+    def run(self, task, block=False):
+        """Run a task on the `TaskController`.
+        
+        See the documentation of the `MapTask` and `StringTask` classes for 
+        details on how to build a task of different types.
         
         :Parameters:
-            task : `Task`
-                The `Task` object to run
+            task : an `ITask` implementer
+        
+        :Returns: The int taskid of the submitted task.  Pass this to 
+            `get_task_result` to get the `TaskResult` object.
         """
-        return blockingCallFromThread(self.task_controller.run, task)
+        tid = blockingCallFromThread(self.task_controller.run, task)
+        if block:
+            return self.get_task_result(tid, block=True)
+        else:
+            return tid
     
     def get_task_result(self, taskid, block=False):
         """
-        Get or poll for a task result.
+        Get a task result by taskid.
         
         :Parameters:
             taskid : int
-                The id of the task whose result to get
+                The taskid of the task to be retrieved.
             block : boolean
-                If True, wait until the task is done and then result the
-                `TaskResult` object.  If False, just poll for the result and
-                return None if the task is not done.
+                Should I block until the task is done?
+        
+        :Returns: A `TaskResult` object that encapsulates the task result.
         """
         return blockingCallFromThread(self.task_controller.get_task_result,
             taskid, block)
     
     def abort(self, taskid):
         """
-        Abort a task by task id if it has not been started.
+        Abort a task by taskid.
+        
+        :Parameters:
+            taskid : int
+                The taskid of the task to be aborted.
         """
         return blockingCallFromThread(self.task_controller.abort, taskid)
     
     def barrier(self, taskids):
-        """
-        Wait for a set of tasks to finish.
+        """Block until a set of tasks are completed.
         
         :Parameters:
-            taskids : list of ints
-                A list of task ids to wait for.
+            taskids : list, tuple
+                A sequence of taskids to block on.
         """
         return blockingCallFromThread(self.task_controller.barrier, taskids)
     
     def spin(self):
         """
-        Cause the scheduler to schedule tasks.
+        Touch the scheduler, to resume scheduling without submitting a task.
         
         This method only needs to be called in unusual situations where the
         scheduler is idle for some reason.
         """
         return blockingCallFromThread(self.task_controller.spin)
     
     def queue_status(self, verbose=False):
         """
         Get a dictionary with the current state of the task queue.
         
         :Parameters:
             verbose : boolean
                 If True, return a list of taskids.  If False, simply give
                 the number of tasks with each status.
         
         :Returns:
             A dict with the queue status.
         """
         return blockingCallFromThread(self.task_controller.queue_status, verbose)
 
+    def clear(self):
+        """
+        Clear all previously run tasks from the task controller.
+        
+        This is needed because the task controller keep all task results
+        in memory.  This can be a problem is there are many completed
+        tasks.  Users should call this periodically to clean out these
+        cached task results.
+        """
+        return blockingCallFromThread(self.task_controller.clear)
+    
+    def map(self, func, *sequences):
+        """
+        Apply func to *sequences elementwise.  Like Python's builtin map.
+        
+        This version is load balanced.
+        """
+        return self.mapper().map(func, *sequences)
+
+    def mapper(self, clear_before=False, clear_after=False, retries=0, 
+                recovery_task=None, depend=None, block=True):
+        """
+        Create an `IMapper` implementer with a given set of arguments.
+        
+        The `IMapper` created using a task controller is load balanced.
+        
+        See the documentation for `IPython.kernel.task.BaseTask` for 
+        documentation on the arguments to this method.
+        """
+        return SynchronousTaskMapper(self, clear_before=clear_before, 
+            clear_after=clear_after, retries=retries, 
+            recovery_task=recovery_task, depend=depend, block=block)
+    
+    def parallel(self, clear_before=False, clear_after=False, retries=0, 
+        recovery_task=None, depend=None, block=True):
+        mapper = self.mapper(clear_before, clear_after, retries,
+            recovery_task, depend, block)
+        pf = ParallelFunction(mapper)
+        return pf
 
 components.registerAdapter(BlockingTaskClient,
             task.ITaskController, IBlockingTaskClient)
 
 

IPython/kernel/taskfc.py

@@ -1,267 +1,329 b''
 # encoding: utf-8
 # -*- test-case-name: IPython.kernel.tests.test_taskxmlrpc -*-
 """A Foolscap interface to a TaskController.
 
 This class lets Foolscap clients talk to a TaskController.
 """
 
 __docformat__ = "restructuredtext en"
 
 #-------------------------------------------------------------------------------
 #  Copyright (C) 2008  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-------------------------------------------------------------------------------
 
 #-------------------------------------------------------------------------------
 # Imports
 #-------------------------------------------------------------------------------
 
 import cPickle as pickle
 import xmlrpclib, copy
 
 from zope.interface import Interface, implements
 from twisted.internet import defer
 from twisted.python import components, failure
 
 from foolscap import Referenceable
 
 from IPython.kernel.twistedutil import blockingCallFromThread
 from IPython.kernel import error, task as taskmodule, taskclient
 from IPython.kernel.pickleutil import can, uncan
 from IPython.kernel.clientinterfaces import (
     IFCClientInterfaceProvider, 
     IBlockingClientAdaptor
 )
+from IPython.kernel.mapper import (
+    TaskMapper,
+    ITaskMapperFactory,
+    IMapper
+)
+from IPython.kernel.parallelfunction import (
+    ParallelFunction, 
+    ITaskParallelDecorator
+)
 
 #-------------------------------------------------------------------------------
 # The Controller side of things
 #-------------------------------------------------------------------------------
 
 
 class IFCTaskController(Interface):
     """Foolscap interface to task controller.
         
-    See the documentation of ITaskController for documentation about the methods.
+    See the documentation of `ITaskController` for more information.
     """
-    def remote_run(request, binTask):
+    def remote_run(binTask):
         """"""
     
-    def remote_abort(request, taskid):
+    def remote_abort(taskid):
         """"""
         
-    def remote_get_task_result(request, taskid, block=False):
+    def remote_get_task_result(taskid, block=False):
         """"""
         
-    def remote_barrier(request, taskids):
+    def remote_barrier(taskids):
         """"""
     
-    def remote_spin(request):
+    def remote_spin():
         """"""
     
-    def remote_queue_status(request, verbose):
+    def remote_queue_status(verbose):
+        """"""
+    
+    def remote_clear():
         """"""
 
 
 class FCTaskControllerFromTaskController(Referenceable):
-    """XML-RPC attachmeot for controller.
+    """
+    Adapt a `TaskController` to an `IFCTaskController`
         
-    See IXMLRPCTaskController and ITaskController (and its children) for documentation. 
+    This class is used to expose a `TaskController` over the wire using
+    the Foolscap network protocol.
     """
+    
     implements(IFCTaskController, IFCClientInterfaceProvider)
     
     def __init__(self, taskController):
         self.taskController = taskController
     
     #---------------------------------------------------------------------------
     # Non interface methods
     #---------------------------------------------------------------------------
     
     def packageFailure(self, f):
         f.cleanFailure()
         return self.packageSuccess(f)
     
     def packageSuccess(self, obj):
         serial = pickle.dumps(obj, 2)
         return serial
     
     #---------------------------------------------------------------------------
     # ITaskController related methods
     #---------------------------------------------------------------------------
     
     def remote_run(self, ptask):
         try:
-            ctask = pickle.loads(ptask)
-            task = taskmodule.uncanTask(ctask)
+            task = pickle.loads(ptask)
+            task.uncan_task()
         except:
             d = defer.fail(pickle.UnpickleableError("Could not unmarshal task"))
         else:
             d = self.taskController.run(task)
         d.addCallback(self.packageSuccess)
         d.addErrback(self.packageFailure)
         return d
     
     def remote_abort(self, taskid):
         d = self.taskController.abort(taskid)
         d.addCallback(self.packageSuccess)
         d.addErrback(self.packageFailure)
         return d
     
     def remote_get_task_result(self, taskid, block=False):
         d = self.taskController.get_task_result(taskid, block)
         d.addCallback(self.packageSuccess)
         d.addErrback(self.packageFailure)
         return d
     
     def remote_barrier(self, taskids):
         d = self.taskController.barrier(taskids)
         d.addCallback(self.packageSuccess)
         d.addErrback(self.packageFailure)
         return d        
     
     def remote_spin(self):
         d = self.taskController.spin()
         d.addCallback(self.packageSuccess)
         d.addErrback(self.packageFailure)
         return d        
     
     def remote_queue_status(self, verbose):
         d = self.taskController.queue_status(verbose)
         d.addCallback(self.packageSuccess)
         d.addErrback(self.packageFailure)
         return d
     
+    def remote_clear(self):
+        return self.taskController.clear()
+    
     def remote_get_client_name(self):
         return 'IPython.kernel.taskfc.FCTaskClient'
     
 components.registerAdapter(FCTaskControllerFromTaskController,
             taskmodule.ITaskController, IFCTaskController)
 
 
 #-------------------------------------------------------------------------------
 # The Client side of things
 #-------------------------------------------------------------------------------
 
 class FCTaskClient(object):
-    """XML-RPC based TaskController client that implements ITaskController.
+    """
+    Client class for Foolscap exposed `TaskController`.
         
-    :Parameters:
-        addr : (ip, port)
-            The ip (str) and port (int) tuple of the `TaskController`.  
+    This class is an adapter that makes a `RemoteReference` to a 
+    `TaskController` look like an actual `ITaskController` on the client side.
+
+    This class also implements `IBlockingClientAdaptor` so that clients can 
+    automatically get a blocking version of this class.
     """
-    implements(taskmodule.ITaskController, IBlockingClientAdaptor)
+    
+    implements(
+        taskmodule.ITaskController, 
+        IBlockingClientAdaptor,
+        ITaskMapperFactory,
+        IMapper,
+        ITaskParallelDecorator
+    )
     
     def __init__(self, remote_reference):
         self.remote_reference = remote_reference
     
     #---------------------------------------------------------------------------
     # Non interface methods
     #---------------------------------------------------------------------------
     
     def unpackage(self, r):
         return pickle.loads(r)
     
     #---------------------------------------------------------------------------
     # ITaskController related methods
     #---------------------------------------------------------------------------
     def run(self, task):
         """Run a task on the `TaskController`.
         
+        See the documentation of the `MapTask` and `StringTask` classes for 
+        details on how to build a task of different types.
+        
         :Parameters:
-            task : a `Task` object
-        
-        The Task object is created using the following signature:
-        
-        Task(expression, pull=None, push={}, clear_before=False, 
-            clear_after=False, retries=0, **options):)
-        
-        The meaning of the arguments is as follows:
-        
-        :Task Parameters:
-            expression : str
-                A str that is valid python code that is the task.
-            pull : str or list of str 
-                The names of objects to be pulled as results.
-            push : dict
-                A dict of objects to be pushed into the engines namespace before
-                execution of the expression.
-            clear_before : boolean
-                Should the engine's namespace be cleared before the task is run.
-                Default=False.
-            clear_after : boolean 
-                Should the engine's namespace be cleared after the task is run.
-                Default=False.
-            retries : int
-                The number of times to resumbit the task if it fails.  Default=0.
-            options : dict
-                Any other keyword options for more elaborate uses of tasks
+            task : an `ITask` implementer
             
         :Returns: The int taskid of the submitted task.  Pass this to 
             `get_task_result` to get the `TaskResult` object.
         """
-        assert isinstance(task, taskmodule.Task), "task must be a Task object!"
-        ctask = taskmodule.canTask(task) # handles arbitrary function in .depend
-                                # as well as arbitrary recovery_task chains
-        ptask = pickle.dumps(ctask, 2)
+        assert isinstance(task, taskmodule.BaseTask), "task must be a Task object!"
+        task.can_task()
+        ptask = pickle.dumps(task, 2)
+        task.uncan_task()
         d = self.remote_reference.callRemote('run', ptask)
         d.addCallback(self.unpackage)
         return d
     
     def get_task_result(self, taskid, block=False):
-        """The task result by taskid.
+        """
+        Get a task result by taskid.
         
         :Parameters:
             taskid : int
                 The taskid of the task to be retrieved.
             block : boolean
                 Should I block until the task is done?
         
         :Returns: A `TaskResult` object that encapsulates the task result.
         """
         d = self.remote_reference.callRemote('get_task_result', taskid, block)
         d.addCallback(self.unpackage)
         return d 
     
     def abort(self, taskid):
-        """Abort a task by taskid.
+        """
+        Abort a task by taskid.
         
         :Parameters:
             taskid : int
                 The taskid of the task to be aborted.
-            block : boolean
-                Should I block until the task is aborted.        
         """
         d = self.remote_reference.callRemote('abort', taskid)
         d.addCallback(self.unpackage)
         return d 
         
     def barrier(self, taskids):
-        """Block until all tasks are completed.
+        """Block until a set of tasks are completed.
         
         :Parameters:
             taskids : list, tuple
                 A sequence of taskids to block on.
         """
         d = self.remote_reference.callRemote('barrier', taskids)
         d.addCallback(self.unpackage)
         return d 
     
     def spin(self):
-        """touch the scheduler, to resume scheduling without submitting
-        a task.
+        """
+        Touch the scheduler, to resume scheduling without submitting a task.
+        
+        This method only needs to be called in unusual situations where the
+        scheduler is idle for some reason.
         """
         d = self.remote_reference.callRemote('spin')
         d.addCallback(self.unpackage)
         return d
     
     def queue_status(self, verbose=False):
-        """Return a dict with the status of the task queue."""
+        """
+        Get a dictionary with the current state of the task queue.
+        
+        :Parameters:
+            verbose : boolean
+                If True, return a list of taskids.  If False, simply give
+                the number of tasks with each status.
+        
+        :Returns:
+            A dict with the queue status.
+        """
         d = self.remote_reference.callRemote('queue_status', verbose)
         d.addCallback(self.unpackage)
         return d
     
+    def clear(self):
+        """
+        Clear all previously run tasks from the task controller.
+        
+        This is needed because the task controller keep all task results
+        in memory.  This can be a problem is there are many completed
+        tasks.  Users should call this periodically to clean out these
+        cached task results.
+        """
+        d = self.remote_reference.callRemote('clear')
+        return d
+    
     def adapt_to_blocking_client(self):
+        """
+        Wrap self in a blocking version that implements `IBlockingTaskClient.
+        """
         from IPython.kernel.taskclient import IBlockingTaskClient
         return IBlockingTaskClient(self)
 
+    def map(self, func, *sequences):
+        """
+        Apply func to *sequences elementwise.  Like Python's builtin map.
+        
+        This version is load balanced.
+        """
+        return self.mapper().map(func, *sequences)
+    
+    def mapper(self, clear_before=False, clear_after=False, retries=0, 
+                recovery_task=None, depend=None, block=True):
+        """
+        Create an `IMapper` implementer with a given set of arguments.
+        
+        The `IMapper` created using a task controller is load balanced.
+        
+        See the documentation for `IPython.kernel.task.BaseTask` for 
+        documentation on the arguments to this method.
+        """
+        return TaskMapper(self, clear_before=clear_before, 
+            clear_after=clear_after, retries=retries, 
+            recovery_task=recovery_task, depend=depend, block=block)
+    
+    def parallel(self, clear_before=False, clear_after=False, retries=0, 
+        recovery_task=None, depend=None, block=True):
+        mapper = self.mapper(clear_before, clear_after, retries,
+            recovery_task, depend, block)
+        pf = ParallelFunction(mapper)
+        return pf
+

IPython/kernel/tests/engineservicetest.py

@@ -1,373 +1,372 b''
 # encoding: utf-8
 
 """Test template for complete engine object"""
 
 __docformat__ = "restructuredtext en"
 
 #-------------------------------------------------------------------------------
 #  Copyright (C) 2008  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-------------------------------------------------------------------------------
 
 #-------------------------------------------------------------------------------
 # Imports
 #-------------------------------------------------------------------------------
 
 import cPickle as pickle
 
 from twisted.internet import defer, reactor
 from twisted.python import failure
 from twisted.application import service
 import zope.interface as zi
 
 from IPython.kernel import newserialized
 from IPython.kernel import error
 from IPython.kernel.pickleutil import can, uncan
 import IPython.kernel.engineservice as es
 from IPython.kernel.core.interpreter import Interpreter
 from IPython.testing.parametric import Parametric, parametric
 
 #-------------------------------------------------------------------------------
 # Tests
 #-------------------------------------------------------------------------------
 
     
 # A sequence of valid commands run through execute
 validCommands = ['a=5',
                  'b=10',
                  'a=5; b=10; c=a+b',
                  'import math; 2.0*math.pi',
                  """def f():
     result = 0.0
     for i in range(10):
         result += i
 """,
                  'if 1<2: a=5',
                  """import time
 time.sleep(0.1)""",
                 """from math import cos;
 x = 1.0*cos(0.5)""", # Semicolons lead to Discard ast nodes that should be discarded
                 """from sets import Set
 s = Set()
    """, # Trailing whitespace should be allowed.
                 """import math
 math.cos(1.0)""", # Test a method call with a discarded return value
                 """x=1.0234
 a=5; b=10""", # Test an embedded semicolon
                 """x=1.0234
 a=5; b=10;""" # Test both an embedded and trailing semicolon
                  ]
                  
 # A sequence of commands that raise various exceptions
 invalidCommands = [('a=1/0',ZeroDivisionError),
                    ('print v',NameError),
                    ('l=[];l[0]',IndexError),
                    ("d={};d['a']",KeyError),
                    ("assert 1==0",AssertionError),
                    ("import abababsdbfsbaljasdlja",ImportError),
                    ("raise Exception()",Exception)]
 
 def testf(x):
     return 2.0*x
 
 globala = 99
 
 def testg(x):
     return  globala*x
 
 class IEngineCoreTestCase(object):
     """Test an IEngineCore implementer."""
 
     def createShell(self):
         return Interpreter()
 
     def catchQueueCleared(self, f):
         try:
             f.raiseException()
         except error.QueueCleared:
             pass
         
     def testIEngineCoreInterface(self):
         """Does self.engine claim to implement IEngineCore?"""
         self.assert_(es.IEngineCore.providedBy(self.engine))
         
     def testIEngineCoreInterfaceMethods(self):
         """Does self.engine have the methods and attributes in IEngineCore."""
         for m in list(es.IEngineCore):
             self.assert_(hasattr(self.engine, m))
     
     def testIEngineCoreDeferreds(self):
         d = self.engine.execute('a=5')
         d.addCallback(lambda _: self.engine.pull('a'))
         d.addCallback(lambda _: self.engine.get_result())
         d.addCallback(lambda _: self.engine.keys())
         d.addCallback(lambda _: self.engine.push(dict(a=10)))
         return d
     
     def runTestExecute(self, cmd):
         self.shell = Interpreter()
         actual = self.shell.execute(cmd)
         def compare(computed):
             actual['id'] = computed['id']
             self.assertEquals(actual, computed)
         d = self.engine.execute(cmd)
         d.addCallback(compare)
         return d
     
     @parametric
     def testExecute(cls):
         return [(cls.runTestExecute, cmd) for cmd in validCommands]
 
     def runTestExecuteFailures(self, cmd, exc):
         def compare(f):
             self.assertRaises(exc, f.raiseException)
         d = self.engine.execute(cmd)
         d.addErrback(compare)
         return d
     
     @parametric
     def testExecuteFailures(cls):
         return [(cls.runTestExecuteFailures, cmd, exc) for cmd, exc in invalidCommands]
     
     def runTestPushPull(self, o):
         d = self.engine.push(dict(a=o))
         d.addCallback(lambda r: self.engine.pull('a'))
         d.addCallback(lambda r: self.assertEquals(o,r))
         return d
     
     @parametric
     def testPushPull(cls):
         objs = [10,"hi there",1.2342354,{"p":(1,2)},None]
         return [(cls.runTestPushPull, o) for o in objs]
         
     def testPullNameError(self):
         d = self.engine.push(dict(a=5))
         d.addCallback(lambda _:self.engine.reset())
         d.addCallback(lambda _: self.engine.pull("a"))
         d.addErrback(lambda f: self.assertRaises(NameError, f.raiseException))
         return d        
     
     def testPushPullFailures(self):
         d = self.engine.pull('a')
         d.addErrback(lambda f: self.assertRaises(NameError, f.raiseException))
         d.addCallback(lambda _: self.engine.execute('l = lambda x: x'))
         d.addCallback(lambda _: self.engine.pull('l'))
         d.addErrback(lambda f: self.assertRaises(pickle.PicklingError, f.raiseException))
         d.addCallback(lambda _: self.engine.push(dict(l=lambda x: x)))
         d.addErrback(lambda f: self.assertRaises(pickle.PicklingError, f.raiseException))
         return d
         
     def testPushPullArray(self):
         try:
             import numpy
         except:
-            print 'no numpy, ',
             return
         a = numpy.random.random(1000)
         d = self.engine.push(dict(a=a))
         d.addCallback(lambda _: self.engine.pull('a'))
         d.addCallback(lambda b: b==a)
         d.addCallback(lambda c: c.all())
         return self.assertDeferredEquals(d, True)
         
     def testPushFunction(self):
                     
         d = self.engine.push_function(dict(f=testf))
         d.addCallback(lambda _: self.engine.execute('result = f(10)'))
         d.addCallback(lambda _: self.engine.pull('result'))
         d.addCallback(lambda r: self.assertEquals(r, testf(10)))
         return d
 
     def testPullFunction(self):
         d = self.engine.push_function(dict(f=testf, g=testg))
         d.addCallback(lambda _: self.engine.pull_function(('f','g')))
         d.addCallback(lambda r: self.assertEquals(r[0](10), testf(10)))
         return d
         
     def testPushFunctionGlobal(self):
         """Make sure that pushed functions pick up the user's namespace for globals."""
         d = self.engine.push(dict(globala=globala))
         d.addCallback(lambda _: self.engine.push_function(dict(g=testg)))
         d.addCallback(lambda _: self.engine.execute('result = g(10)'))
         d.addCallback(lambda _: self.engine.pull('result'))
         d.addCallback(lambda r: self.assertEquals(r, testg(10)))
         return d
         
     def testGetResultFailure(self):
         d = self.engine.get_result(None)
         d.addErrback(lambda f: self.assertRaises(IndexError, f.raiseException))
         d.addCallback(lambda _: self.engine.get_result(10))
         d.addErrback(lambda f: self.assertRaises(IndexError, f.raiseException))
         return d
 
     def runTestGetResult(self, cmd):
         self.shell = Interpreter()
         actual = self.shell.execute(cmd)
         def compare(computed):
             actual['id'] = computed['id']
             self.assertEquals(actual, computed)
         d = self.engine.execute(cmd)
         d.addCallback(lambda r: self.engine.get_result(r['number']))
         d.addCallback(compare)
         return d
     
     @parametric
     def testGetResult(cls):
         return [(cls.runTestGetResult, cmd) for cmd in validCommands]
     
     def testGetResultDefault(self):
         cmd = 'a=5'
         shell = self.createShell()
         shellResult = shell.execute(cmd)
         def popit(dikt, key):
             dikt.pop(key)
             return dikt
         d = self.engine.execute(cmd)
         d.addCallback(lambda _: self.engine.get_result())
         d.addCallback(lambda r: self.assertEquals(shellResult, popit(r,'id')))
         return d
 
     def testKeys(self):
         d = self.engine.keys()
         d.addCallback(lambda s: isinstance(s, list))
         d.addCallback(lambda r: self.assertEquals(r, True))
         return d
             
 Parametric(IEngineCoreTestCase)
             
 class IEngineSerializedTestCase(object):
     """Test an IEngineCore implementer."""
         
     def testIEngineSerializedInterface(self):
         """Does self.engine claim to implement IEngineCore?"""
         self.assert_(es.IEngineSerialized.providedBy(self.engine))
         
     def testIEngineSerializedInterfaceMethods(self):
         """Does self.engine have the methods and attributes in IEngireCore."""
         for m in list(es.IEngineSerialized):
             self.assert_(hasattr(self.engine, m))
        
     def testIEngineSerializedDeferreds(self):
         dList = []
         d = self.engine.push_serialized(dict(key=newserialized.serialize(12345)))
         self.assert_(isinstance(d, defer.Deferred))
         dList.append(d)
         d = self.engine.pull_serialized('key')
         self.assert_(isinstance(d, defer.Deferred))
         dList.append(d)
         D = defer.DeferredList(dList)
         return D
                 
     def testPushPullSerialized(self):
         objs = [10,"hi there",1.2342354,{"p":(1,2)}]
         d = defer.succeed(None)
         for o in objs:
             self.engine.push_serialized(dict(key=newserialized.serialize(o)))
             value = self.engine.pull_serialized('key')
             value.addCallback(lambda serial: newserialized.IUnSerialized(serial).getObject())
             d = self.assertDeferredEquals(value,o,d)
         return d
 
     def testPullSerializedFailures(self):
         d = self.engine.pull_serialized('a')
         d.addErrback(lambda f: self.assertRaises(NameError, f.raiseException))
         d.addCallback(lambda _: self.engine.execute('l = lambda x: x'))
         d.addCallback(lambda _: self.engine.pull_serialized('l'))
         d.addErrback(lambda f: self.assertRaises(pickle.PicklingError, f.raiseException))
         return d
 
 Parametric(IEngineSerializedTestCase)
 
 class IEngineQueuedTestCase(object):
     """Test an IEngineQueued implementer."""
         
     def testIEngineQueuedInterface(self):
         """Does self.engine claim to implement IEngineQueued?"""
         self.assert_(es.IEngineQueued.providedBy(self.engine))
         
     def testIEngineQueuedInterfaceMethods(self):
         """Does self.engine have the methods and attributes in IEngireQueued."""
         for m in list(es.IEngineQueued):
             self.assert_(hasattr(self.engine, m))
             
     def testIEngineQueuedDeferreds(self): 
         dList = []
         d = self.engine.clear_queue()
         self.assert_(isinstance(d, defer.Deferred))
         dList.append(d)
         d = self.engine.queue_status()
         self.assert_(isinstance(d, defer.Deferred))
         dList.append(d)
         D = defer.DeferredList(dList)
         return D
             
     def testClearQueue(self):
         result = self.engine.clear_queue()
         d1 = self.assertDeferredEquals(result, None)
         d1.addCallback(lambda _: self.engine.queue_status())
         d2 = self.assertDeferredEquals(d1, {'queue':[], 'pending':'None'})
         return d2
         
     def testQueueStatus(self):
         result = self.engine.queue_status()
         result.addCallback(lambda r: 'queue' in r and 'pending' in r)
         d = self.assertDeferredEquals(result, True)
         return d
 
 Parametric(IEngineQueuedTestCase)
 
 class IEnginePropertiesTestCase(object):
     """Test an IEngineProperties implementor."""
     
     def testIEnginePropertiesInterface(self):
         """Does self.engine claim to implement IEngineProperties?"""
         self.assert_(es.IEngineProperties.providedBy(self.engine))
     
     def testIEnginePropertiesInterfaceMethods(self):
         """Does self.engine have the methods and attributes in IEngireProperties."""
         for m in list(es.IEngineProperties):
             self.assert_(hasattr(self.engine, m))
     
     def testGetSetProperties(self):
         dikt = dict(a=5, b='asdf', c=True, d=None, e=range(5))
         d = self.engine.set_properties(dikt)
         d.addCallback(lambda r: self.engine.get_properties())
         d = self.assertDeferredEquals(d, dikt)
         d.addCallback(lambda r: self.engine.get_properties(('c',)))
         d = self.assertDeferredEquals(d, {'c': dikt['c']})
         d.addCallback(lambda r: self.engine.set_properties(dict(c=False)))
         d.addCallback(lambda r: self.engine.get_properties(('c', 'd')))
         d = self.assertDeferredEquals(d, dict(c=False, d=None))
         return d
     
     def testClearProperties(self):
         dikt = dict(a=5, b='asdf', c=True, d=None, e=range(5))
         d = self.engine.set_properties(dikt)
         d.addCallback(lambda r: self.engine.clear_properties())
         d.addCallback(lambda r: self.engine.get_properties())
         d = self.assertDeferredEquals(d, {})
         return d
     
     def testDelHasProperties(self):
         dikt = dict(a=5, b='asdf', c=True, d=None, e=range(5))
         d = self.engine.set_properties(dikt)
         d.addCallback(lambda r: self.engine.del_properties(('b','e')))
         d.addCallback(lambda r: self.engine.has_properties(('a','b','c','d','e')))
         d = self.assertDeferredEquals(d, [True, False, True, True, False])
         return d
     
     def testStrictDict(self):
         s = """from IPython.kernel.engineservice import get_engine
 p = get_engine(%s).properties"""%self.engine.id
         d = self.engine.execute(s)
         d.addCallback(lambda r: self.engine.execute("p['a'] = lambda _:None"))
         d = self.assertDeferredRaises(d, error.InvalidProperty)
         d.addCallback(lambda r: self.engine.execute("p['a'] = range(5)"))
         d.addCallback(lambda r: self.engine.execute("p['a'].append(5)"))
         d.addCallback(lambda r: self.engine.get_properties('a'))
         d = self.assertDeferredEquals(d, dict(a=range(5)))
         return d
         
 Parametric(IEnginePropertiesTestCase)

IPython/kernel/tests/multienginetest.py

@@ -1,838 +1,828 b''
 # encoding: utf-8
 
 """"""
 
 __docformat__ = "restructuredtext en"
 
 #-------------------------------------------------------------------------------
 #  Copyright (C) 2008  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-------------------------------------------------------------------------------
 
 #-------------------------------------------------------------------------------
 # Imports
 #-------------------------------------------------------------------------------
 
 from twisted.internet import defer
 
 from IPython.kernel import engineservice as es
 from IPython.kernel import multiengine as me
 from IPython.kernel import newserialized
 from IPython.kernel.error import NotDefined
 from IPython.testing import util
 from IPython.testing.parametric import parametric, Parametric
 from IPython.kernel import newserialized
 from IPython.kernel.util import printer
 from IPython.kernel.error import (InvalidEngineID,
     NoEnginesRegistered,
     CompositeError,
     InvalidDeferredID)
 from IPython.kernel.tests.engineservicetest import validCommands, invalidCommands
 from IPython.kernel.core.interpreter import Interpreter
 
 
 #-------------------------------------------------------------------------------
 # Base classes and utilities
 #-------------------------------------------------------------------------------
 
 class IMultiEngineBaseTestCase(object):
     """Basic utilities for working with multiengine tests.
 
     Some subclass should define:
 
     * self.multiengine
     * self.engines to keep track of engines for clean up"""
 
     def createShell(self):
         return Interpreter()
 
     def addEngine(self, n=1):
         for i in range(n):
             e = es.EngineService()
             e.startService()
             regDict = self.controller.register_engine(es.QueuedEngine(e), None)
             e.id = regDict['id']
             self.engines.append(e)
 
 
 def testf(x):
     return 2.0*x
 
 
 globala = 99
 
 
 def testg(x):
     return  globala*x
 
 
 def isdid(did):
     if not isinstance(did, str):
         return False
     if not len(did)==40:
         return False
     return True
 
 
 def _raise_it(f):
     try:
         f.raiseException()
     except CompositeError, e:
         e.raise_exception()
 
 #-------------------------------------------------------------------------------
 # IMultiEngineTestCase
 #-------------------------------------------------------------------------------
 
 class IMultiEngineTestCase(IMultiEngineBaseTestCase):
     """A test for any object that implements IEngineMultiplexer.
 
     self.multiengine must be defined and implement IEngineMultiplexer.
     """
 
     def testIMultiEngineInterface(self):
         """Does self.engine claim to implement IEngineCore?"""
         self.assert_(me.IEngineMultiplexer.providedBy(self.multiengine))
         self.assert_(me.IMultiEngine.providedBy(self.multiengine))
 
     def testIEngineMultiplexerInterfaceMethods(self):
         """Does self.engine have the methods and attributes in IEngineCore."""
         for m in list(me.IEngineMultiplexer):
             self.assert_(hasattr(self.multiengine, m))
 
     def testIEngineMultiplexerDeferreds(self):
         self.addEngine(1)
         d= self.multiengine.execute('a=5', targets=0)
         d.addCallback(lambda _: self.multiengine.push(dict(a=5),targets=0))
         d.addCallback(lambda _: self.multiengine.push(dict(a=5, b='asdf', c=[1,2,3]),targets=0))
         d.addCallback(lambda _: self.multiengine.pull(('a','b','c'),targets=0))
         d.addCallback(lambda _: self.multiengine.get_result(targets=0))
         d.addCallback(lambda _: self.multiengine.reset(targets=0))
         d.addCallback(lambda _: self.multiengine.keys(targets=0))
         d.addCallback(lambda _: self.multiengine.push_serialized(dict(a=newserialized.serialize(10)),targets=0))
         d.addCallback(lambda _: self.multiengine.pull_serialized('a',targets=0))
         d.addCallback(lambda _: self.multiengine.clear_queue(targets=0))
         d.addCallback(lambda _: self.multiengine.queue_status(targets=0))
         return d
 
     def testInvalidEngineID(self):
          self.addEngine(1)
          badID = 100
          d = self.multiengine.execute('a=5', targets=badID)
          d.addErrback(lambda f: self.assertRaises(InvalidEngineID, f.raiseException))
          d.addCallback(lambda _: self.multiengine.push(dict(a=5), targets=badID))
          d.addErrback(lambda f: self.assertRaises(InvalidEngineID, f.raiseException))
          d.addCallback(lambda _: self.multiengine.pull('a', targets=badID))
          d.addErrback(lambda f: self.assertRaises(InvalidEngineID, f.raiseException))
          d.addCallback(lambda _: self.multiengine.reset(targets=badID))
          d.addErrback(lambda f: self.assertRaises(InvalidEngineID, f.raiseException))
          d.addCallback(lambda _: self.multiengine.keys(targets=badID))
          d.addErrback(lambda f: self.assertRaises(InvalidEngineID, f.raiseException))
          d.addCallback(lambda _: self.multiengine.push_serialized(dict(a=newserialized.serialize(10)), targets=badID))
          d.addErrback(lambda f: self.assertRaises(InvalidEngineID, f.raiseException))
          d.addCallback(lambda _: self.multiengine.pull_serialized('a', targets=badID))
          d.addErrback(lambda f: self.assertRaises(InvalidEngineID, f.raiseException))
          d.addCallback(lambda _: self.multiengine.queue_status(targets=badID))
          d.addErrback(lambda f: self.assertRaises(InvalidEngineID, f.raiseException))
          return d
 
     def testNoEnginesRegistered(self):
         badID = 'all'
         d= self.multiengine.execute('a=5', targets=badID)
         d.addErrback(lambda f: self.assertRaises(NoEnginesRegistered, f.raiseException))
         d.addCallback(lambda _: self.multiengine.push(dict(a=5), targets=badID))
         d.addErrback(lambda f: self.assertRaises(NoEnginesRegistered, f.raiseException))
         d.addCallback(lambda _: self.multiengine.pull('a', targets=badID))
         d.addErrback(lambda f: self.assertRaises(NoEnginesRegistered, f.raiseException))
         d.addCallback(lambda _: self.multiengine.get_result(targets=badID))
         d.addErrback(lambda f: self.assertRaises(NoEnginesRegistered, f.raiseException))
         d.addCallback(lambda _: self.multiengine.reset(targets=badID))
         d.addErrback(lambda f: self.assertRaises(NoEnginesRegistered, f.raiseException))
         d.addCallback(lambda _: self.multiengine.keys(targets=badID))
         d.addErrback(lambda f: self.assertRaises(NoEnginesRegistered, f.raiseException))
         d.addCallback(lambda _: self.multiengine.push_serialized(dict(a=newserialized.serialize(10)), targets=badID))
         d.addErrback(lambda f: self.assertRaises(NoEnginesRegistered, f.raiseException))
         d.addCallback(lambda _: self.multiengine.pull_serialized('a', targets=badID))
         d.addErrback(lambda f: self.assertRaises(NoEnginesRegistered, f.raiseException))
         d.addCallback(lambda _: self.multiengine.queue_status(targets=badID))
         d.addErrback(lambda f: self.assertRaises(NoEnginesRegistered, f.raiseException))
         return d
 
     def runExecuteAll(self, d, cmd, shell):
         actual = shell.execute(cmd)
         d.addCallback(lambda _: self.multiengine.execute(cmd))
         def compare(result):
             for r in result:
                 actual['id'] = r['id']
                 self.assertEquals(r, actual)
         d.addCallback(compare)
 
     def testExecuteAll(self):
         self.addEngine(4)
         d= defer.Deferred()
         shell = Interpreter()
         for cmd in validCommands:
                 self.runExecuteAll(d, cmd, shell)
         d.callback(None)
         return d
 
     # The following two methods show how to do parametrized
     # tests.  This is really slick!  Same is used above.
     def runExecuteFailures(self, cmd, exc):
         self.addEngine(4)
         d= self.multiengine.execute(cmd)
         d.addErrback(lambda f: self.assertRaises(exc, _raise_it, f))
         return d
 
     @parametric
     def testExecuteFailures(cls):
         return [(cls.runExecuteFailures,cmd,exc) for
                 cmd,exc in invalidCommands]
 
     def testPushPull(self):
         self.addEngine(1)
         objs = [10,"hi there",1.2342354,{"p":(1,2)}]
         d= self.multiengine.push(dict(key=objs[0]), targets=0)
         d.addCallback(lambda _: self.multiengine.pull('key', targets=0))
         d.addCallback(lambda r: self.assertEquals(r, [objs[0]]))
         d.addCallback(lambda _: self.multiengine.push(dict(key=objs[1]), targets=0))
         d.addCallback(lambda _: self.multiengine.pull('key', targets=0))
         d.addCallback(lambda r: self.assertEquals(r, [objs[1]]))
         d.addCallback(lambda _: self.multiengine.push(dict(key=objs[2]), targets=0))
         d.addCallback(lambda _: self.multiengine.pull('key', targets=0))
         d.addCallback(lambda r: self.assertEquals(r, [objs[2]]))
         d.addCallback(lambda _: self.multiengine.push(dict(key=objs[3]), targets=0))
         d.addCallback(lambda _: self.multiengine.pull('key', targets=0))
         d.addCallback(lambda r: self.assertEquals(r, [objs[3]]))
         d.addCallback(lambda _: self.multiengine.reset(targets=0))
         d.addCallback(lambda _: self.multiengine.pull('a', targets=0))
         d.addErrback(lambda f: self.assertRaises(NameError, _raise_it, f))
         d.addCallback(lambda _: self.multiengine.push(dict(a=10,b=20)))
         d.addCallback(lambda _: self.multiengine.pull(('a','b')))
         d.addCallback(lambda r: self.assertEquals(r, [[10,20]]))
         return d
 
     def testPushPullAll(self):
         self.addEngine(4)
         d= self.multiengine.push(dict(a=10))
         d.addCallback(lambda _: self.multiengine.pull('a'))
         d.addCallback(lambda r: self.assert_(r==[10,10,10,10]))
         d.addCallback(lambda _: self.multiengine.push(dict(a=10, b=20)))
         d.addCallback(lambda _: self.multiengine.pull(('a','b')))
         d.addCallback(lambda r: self.assert_(r==4*[[10,20]]))
         d.addCallback(lambda _: self.multiengine.push(dict(a=10, b=20), targets=0))
         d.addCallback(lambda _: self.multiengine.pull(('a','b'), targets=0))
         d.addCallback(lambda r: self.assert_(r==[[10,20]]))
         d.addCallback(lambda _: self.multiengine.push(dict(a=None, b=None), targets=0))
         d.addCallback(lambda _: self.multiengine.pull(('a','b'), targets=0))
         d.addCallback(lambda r: self.assert_(r==[[None,None]]))
         return d
 
     def testPushPullSerialized(self):
         self.addEngine(1)
         objs = [10,"hi there",1.2342354,{"p":(1,2)}]
         d= self.multiengine.push_serialized(dict(key=newserialized.serialize(objs[0])), targets=0)
         d.addCallback(lambda _: self.multiengine.pull_serialized('key', targets=0))
         d.addCallback(lambda serial: newserialized.IUnSerialized(serial[0]).getObject())
         d.addCallback(lambda r: self.assertEquals(r, objs[0]))
         d.addCallback(lambda _: self.multiengine.push_serialized(dict(key=newserialized.serialize(objs[1])), targets=0))
         d.addCallback(lambda _: self.multiengine.pull_serialized('key', targets=0))
         d.addCallback(lambda serial: newserialized.IUnSerialized(serial[0]).getObject())
         d.addCallback(lambda r: self.assertEquals(r, objs[1]))
         d.addCallback(lambda _: self.multiengine.push_serialized(dict(key=newserialized.serialize(objs[2])), targets=0))
         d.addCallback(lambda _: self.multiengine.pull_serialized('key', targets=0))
         d.addCallback(lambda serial: newserialized.IUnSerialized(serial[0]).getObject())
         d.addCallback(lambda r: self.assertEquals(r, objs[2]))
         d.addCallback(lambda _: self.multiengine.push_serialized(dict(key=newserialized.serialize(objs[3])), targets=0))
         d.addCallback(lambda _: self.multiengine.pull_serialized('key', targets=0))
         d.addCallback(lambda serial: newserialized.IUnSerialized(serial[0]).getObject())
         d.addCallback(lambda r: self.assertEquals(r, objs[3]))
         d.addCallback(lambda _: self.multiengine.push(dict(a=10,b=range(5)), targets=0))
         d.addCallback(lambda _: self.multiengine.pull_serialized(('a','b'), targets=0))
         d.addCallback(lambda serial: [newserialized.IUnSerialized(s).getObject() for s in serial[0]])
         d.addCallback(lambda r: self.assertEquals(r, [10, range(5)]))
         d.addCallback(lambda _: self.multiengine.reset(targets=0))
         d.addCallback(lambda _: self.multiengine.pull_serialized('a', targets=0))
         d.addErrback(lambda f: self.assertRaises(NameError, _raise_it, f))
         return d
 
         objs = [10,"hi there",1.2342354,{"p":(1,2)}]
         d= defer.succeed(None)
         for o in objs:
             self.multiengine.push_serialized(0, key=newserialized.serialize(o))
             value = self.multiengine.pull_serialized(0, 'key')
             value.addCallback(lambda serial: newserialized.IUnSerialized(serial[0]).getObject())
             d = self.assertDeferredEquals(value,o,d)
         return d
 
     def runGetResultAll(self, d, cmd, shell):
         actual = shell.execute(cmd)
         d.addCallback(lambda _: self.multiengine.execute(cmd))
         d.addCallback(lambda _: self.multiengine.get_result())
         def compare(result):
             for r in result:
                 actual['id'] = r['id']
                 self.assertEquals(r, actual)
         d.addCallback(compare)
 
     def testGetResultAll(self):
         self.addEngine(4)
         d= defer.Deferred()
         shell = Interpreter()
         for cmd in validCommands:
                 self.runGetResultAll(d, cmd, shell)
         d.callback(None)
         return d
 
     def testGetResultDefault(self):
         self.addEngine(1)
         target = 0
         cmd = 'a=5'
         shell = self.createShell()
         shellResult = shell.execute(cmd)
         def popit(dikt, key):
             dikt.pop(key)
             return dikt
         d= self.multiengine.execute(cmd, targets=target)
         d.addCallback(lambda _: self.multiengine.get_result(targets=target))
         d.addCallback(lambda r: self.assertEquals(shellResult, popit(r[0],'id')))
         return d
 
     def testGetResultFailure(self):
         self.addEngine(1)
         d= self.multiengine.get_result(None, targets=0)
         d.addErrback(lambda f: self.assertRaises(IndexError, _raise_it, f))
         d.addCallback(lambda _: self.multiengine.get_result(10, targets=0))
         d.addErrback(lambda f: self.assertRaises(IndexError, _raise_it, f))
         return d
 
     def testPushFunction(self):
         self.addEngine(1)
         d= self.multiengine.push_function(dict(f=testf), targets=0)
         d.addCallback(lambda _: self.multiengine.execute('result = f(10)', targets=0))
         d.addCallback(lambda _: self.multiengine.pull('result', targets=0))
         d.addCallback(lambda r: self.assertEquals(r[0], testf(10)))
         d.addCallback(lambda _: self.multiengine.push(dict(globala=globala), targets=0))
         d.addCallback(lambda _: self.multiengine.push_function(dict(g=testg), targets=0))
         d.addCallback(lambda _: self.multiengine.execute('result = g(10)', targets=0))
         d.addCallback(lambda _: self.multiengine.pull('result', targets=0))
         d.addCallback(lambda r: self.assertEquals(r[0], testg(10)))
         return d
 
     def testPullFunction(self):
         self.addEngine(1)
         d= self.multiengine.push(dict(a=globala), targets=0)
         d.addCallback(lambda _: self.multiengine.push_function(dict(f=testf), targets=0))
         d.addCallback(lambda _: self.multiengine.pull_function('f', targets=0))
         d.addCallback(lambda r: self.assertEquals(r[0](10), testf(10)))
         d.addCallback(lambda _: self.multiengine.execute("def g(x): return x*x", targets=0))
         d.addCallback(lambda _: self.multiengine.pull_function(('f','g'),targets=0))
         d.addCallback(lambda r: self.assertEquals((r[0][0](10),r[0][1](10)), (testf(10), 100)))
         return d
 
     def testPushFunctionAll(self):
         self.addEngine(4)
         d= self.multiengine.push_function(dict(f=testf))
         d.addCallback(lambda _: self.multiengine.execute('result = f(10)'))
         d.addCallback(lambda _: self.multiengine.pull('result'))
         d.addCallback(lambda r: self.assertEquals(r, 4*[testf(10)]))
         d.addCallback(lambda _: self.multiengine.push(dict(globala=globala)))
         d.addCallback(lambda _: self.multiengine.push_function(dict(testg=testg)))
         d.addCallback(lambda _: self.multiengine.execute('result = testg(10)'))
         d.addCallback(lambda _: self.multiengine.pull('result'))
         d.addCallback(lambda r: self.assertEquals(r, 4*[testg(10)]))
         return d
 
     def testPullFunctionAll(self):
         self.addEngine(4)
         d= self.multiengine.push_function(dict(f=testf))
         d.addCallback(lambda _: self.multiengine.pull_function('f'))
         d.addCallback(lambda r: self.assertEquals([func(10) for func in r], 4*[testf(10)]))
         return d
 
     def testGetIDs(self):
         self.addEngine(1)
         d= self.multiengine.get_ids()
         d.addCallback(lambda r: self.assertEquals(r, [0]))
         d.addCallback(lambda _: self.addEngine(3))
         d.addCallback(lambda _: self.multiengine.get_ids())
         d.addCallback(lambda r: self.assertEquals(r, [0,1,2,3]))
         return d
 
     def testClearQueue(self):
         self.addEngine(4)
         d= self.multiengine.clear_queue()
         d.addCallback(lambda r: self.assertEquals(r,4*[None]))
         return d
 
     def testQueueStatus(self):
         self.addEngine(4)
         d= self.multiengine.queue_status(targets=0)
         d.addCallback(lambda r: self.assert_(isinstance(r[0],tuple)))
         return d
 
     def testGetSetProperties(self):
         self.addEngine(4)
         dikt = dict(a=5, b='asdf', c=True, d=None, e=range(5))
         d= self.multiengine.set_properties(dikt)
         d.addCallback(lambda r: self.multiengine.get_properties())
         d.addCallback(lambda r: self.assertEquals(r, 4*[dikt]))
         d.addCallback(lambda r: self.multiengine.get_properties(('c',)))
         d.addCallback(lambda r: self.assertEquals(r, 4*[{'c': dikt['c']}]))
         d.addCallback(lambda r: self.multiengine.set_properties(dict(c=False)))
         d.addCallback(lambda r: self.multiengine.get_properties(('c', 'd')))
         d.addCallback(lambda r: self.assertEquals(r, 4*[dict(c=False, d=None)]))
         return d
 
     def testClearProperties(self):
         self.addEngine(4)
         dikt = dict(a=5, b='asdf', c=True, d=None, e=range(5))
         d= self.multiengine.set_properties(dikt)
         d.addCallback(lambda r: self.multiengine.clear_properties())
         d.addCallback(lambda r: self.multiengine.get_properties())
         d.addCallback(lambda r: self.assertEquals(r, 4*[{}]))
         return d
 
     def testDelHasProperties(self):
         self.addEngine(4)
         dikt = dict(a=5, b='asdf', c=True, d=None, e=range(5))
         d= self.multiengine.set_properties(dikt)
         d.addCallback(lambda r: self.multiengine.del_properties(('b','e')))
         d.addCallback(lambda r: self.multiengine.has_properties(('a','b','c','d','e')))
         d.addCallback(lambda r: self.assertEquals(r, 4*[[True, False, True, True, False]]))
         return d
 
 Parametric(IMultiEngineTestCase)
 
 #-------------------------------------------------------------------------------
 # ISynchronousMultiEngineTestCase
 #-------------------------------------------------------------------------------
 
 class ISynchronousMultiEngineTestCase(IMultiEngineBaseTestCase):
 
     def testISynchronousMultiEngineInterface(self):
         """Does self.engine claim to implement IEngineCore?"""
         self.assert_(me.ISynchronousEngineMultiplexer.providedBy(self.multiengine))
         self.assert_(me.ISynchronousMultiEngine.providedBy(self.multiengine))
 
     def testExecute(self):
         self.addEngine(4)
         execute = self.multiengine.execute
         d= execute('a=5', targets=0, block=True)
         d.addCallback(lambda r: self.assert_(len(r)==1))
         d.addCallback(lambda _: execute('b=10'))
         d.addCallback(lambda r: self.assert_(len(r)==4))
         d.addCallback(lambda _: execute('c=30', block=False))
         d.addCallback(lambda did: self.assert_(isdid(did)))
         d.addCallback(lambda _: execute('d=[0,1,2]', block=False))
         d.addCallback(lambda did: self.multiengine.get_pending_deferred(did, True))
         d.addCallback(lambda r: self.assert_(len(r)==4))
         return d
 
     def testPushPull(self):
         data = dict(a=10, b=1.05, c=range(10), d={'e':(1,2),'f':'hi'})
         self.addEngine(4)
         push = self.multiengine.push
         pull = self.multiengine.pull
         d= push({'data':data}, targets=0)
         d.addCallback(lambda r: pull('data', targets=0))
         d.addCallback(lambda r: self.assertEqual(r,[data]))
         d.addCallback(lambda _: push({'data':data}))
         d.addCallback(lambda r: pull('data'))
         d.addCallback(lambda r: self.assertEqual(r,4*[data]))
         d.addCallback(lambda _: push({'data':data}, block=False))
         d.addCallback(lambda did: self.multiengine.get_pending_deferred(did, True))
         d.addCallback(lambda _: pull('data', block=False))
         d.addCallback(lambda did: self.multiengine.get_pending_deferred(did, True))
         d.addCallback(lambda r: self.assertEqual(r,4*[data]))
         d.addCallback(lambda _: push(dict(a=10,b=20)))
         d.addCallback(lambda _: pull(('a','b')))
         d.addCallback(lambda r: self.assertEquals(r, 4*[[10,20]]))
         return d
 
     def testPushPullFunction(self):
         self.addEngine(4)
         pushf = self.multiengine.push_function
         pullf = self.multiengine.pull_function
         push = self.multiengine.push
         pull = self.multiengine.pull
         execute = self.multiengine.execute
         d= pushf({'testf':testf}, targets=0)
         d.addCallback(lambda r: pullf('testf', targets=0))
         d.addCallback(lambda r: self.assertEqual(r[0](1.0), testf(1.0)))
         d.addCallback(lambda _: execute('r = testf(10)', targets=0))
         d.addCallback(lambda _: pull('r', targets=0))
         d.addCallback(lambda r: self.assertEquals(r[0], testf(10)))
         d.addCallback(lambda _: pushf({'testf':testf}, block=False))
         d.addCallback(lambda did: self.multiengine.get_pending_deferred(did, True))
         d.addCallback(lambda _: pullf('testf', block=False))
         d.addCallback(lambda did: self.multiengine.get_pending_deferred(did, True))
         d.addCallback(lambda r: self.assertEqual(r[0](1.0), testf(1.0)))
         d.addCallback(lambda _: execute("def g(x): return x*x", targets=0))
         d.addCallback(lambda _: pullf(('testf','g'),targets=0))
         d.addCallback(lambda r: self.assertEquals((r[0][0](10),r[0][1](10)), (testf(10), 100)))
         return d
 
     def testGetResult(self):
         shell = Interpreter()
         result1 = shell.execute('a=10')
         result1['id'] = 0
         result2 = shell.execute('b=20')
         result2['id'] = 0
         execute= self.multiengine.execute
         get_result = self.multiengine.get_result
         self.addEngine(1)
         d= execute('a=10')
         d.addCallback(lambda _: get_result())
         d.addCallback(lambda r: self.assertEquals(r[0], result1))
         d.addCallback(lambda _: execute('b=20'))
         d.addCallback(lambda _: get_result(1))
         d.addCallback(lambda r: self.assertEquals(r[0], result1))
         d.addCallback(lambda _: get_result(2, block=False))
         d.addCallback(lambda did: self.multiengine.get_pending_deferred(did, True))
         d.addCallback(lambda r: self.assertEquals(r[0], result2))
         return d
 
     def testResetAndKeys(self):
         self.addEngine(1)
 
         #Blocking mode
         d= self.multiengine.push(dict(a=10, b=20, c=range(10)), targets=0)
         d.addCallback(lambda _: self.multiengine.keys(targets=0))
         def keys_found(keys):
             self.assert_('a' in keys[0])
             self.assert_('b' in keys[0])
             self.assert_('b' in keys[0])
         d.addCallback(keys_found)
         d.addCallback(lambda _: self.multiengine.reset(targets=0))
         d.addCallback(lambda _: self.multiengine.keys(targets=0))
         def keys_not_found(keys):
             self.assert_('a' not in keys[0])
             self.assert_('b' not in keys[0])
             self.assert_('b' not in keys[0])
         d.addCallback(keys_not_found)
 
         #Non-blocking mode
         d.addCallback(lambda _: self.multiengine.push(dict(a=10, b=20, c=range(10)), targets=0))
         d.addCallback(lambda _: self.multiengine.keys(targets=0, block=False))
         d.addCallback(lambda did: self.multiengine.get_pending_deferred(did, True))
         def keys_found(keys):
             self.assert_('a' in keys[0])
             self.assert_('b' in keys[0])
             self.assert_('b' in keys[0])
         d.addCallback(keys_found)
         d.addCallback(lambda _: self.multiengine.reset(targets=0, block=False))
         d.addCallback(lambda did: self.multiengine.get_pending_deferred(did, True))
         d.addCallback(lambda _: self.multiengine.keys(targets=0, block=False))
         d.addCallback(lambda did: self.multiengine.get_pending_deferred(did, True))
         def keys_not_found(keys):
             self.assert_('a' not in keys[0])
             self.assert_('b' not in keys[0])
             self.assert_('b' not in keys[0])
         d.addCallback(keys_not_found)
 
         return d
 
     def testPushPullSerialized(self):
         self.addEngine(1)
         dikt = dict(a=10,b='hi there',c=1.2345,d={'p':(1,2)})
         sdikt = {}
         for k,v in dikt.iteritems():
             sdikt[k] = newserialized.serialize(v)
         d= self.multiengine.push_serialized(dict(a=sdikt['a']), targets=0)
         d.addCallback(lambda _: self.multiengine.pull('a',targets=0))
         d.addCallback(lambda r: self.assertEquals(r[0], dikt['a']))
         d.addCallback(lambda _: self.multiengine.pull_serialized('a', targets=0))
         d.addCallback(lambda serial: newserialized.IUnSerialized(serial[0]).getObject())
         d.addCallback(lambda r: self.assertEquals(r, dikt['a']))
         d.addCallback(lambda _: self.multiengine.push_serialized(sdikt, targets=0))
         d.addCallback(lambda _: self.multiengine.pull_serialized(sdikt.keys(), targets=0))
         d.addCallback(lambda serial: [newserialized.IUnSerialized(s).getObject() for s in serial[0]])
         d.addCallback(lambda r: self.assertEquals(r, dikt.values()))
         d.addCallback(lambda _: self.multiengine.reset(targets=0))
         d.addCallback(lambda _: self.multiengine.pull_serialized('a', targets=0))
         d.addErrback(lambda f: self.assertRaises(NameError, _raise_it, f))
 
         #Non-blocking mode
         d.addCallback(lambda r: self.multiengine.push_serialized(dict(a=sdikt['a']), targets=0, block=False))
         d.addCallback(lambda did: self.multiengine.get_pending_deferred(did, True))
         d.addCallback(lambda _: self.multiengine.pull('a',targets=0))
         d.addCallback(lambda r: self.assertEquals(r[0], dikt['a']))
         d.addCallback(lambda _: self.multiengine.pull_serialized('a', targets=0, block=False))
         d.addCallback(lambda did: self.multiengine.get_pending_deferred(did, True))
         d.addCallback(lambda serial: newserialized.IUnSerialized(serial[0]).getObject())
         d.addCallback(lambda r: self.assertEquals(r, dikt['a']))
         d.addCallback(lambda _: self.multiengine.push_serialized(sdikt, targets=0, block=False))
         d.addCallback(lambda did: self.multiengine.get_pending_deferred(did, True))
         d.addCallback(lambda _: self.multiengine.pull_serialized(sdikt.keys(), targets=0, block=False))
         d.addCallback(lambda did: self.multiengine.get_pending_deferred(did, True))
         d.addCallback(lambda serial: [newserialized.IUnSerialized(s).getObject() for s in serial[0]])
         d.addCallback(lambda r: self.assertEquals(r, dikt.values()))
         d.addCallback(lambda _: self.multiengine.reset(targets=0))
         d.addCallback(lambda _: self.multiengine.pull_serialized('a', targets=0, block=False))
         d.addCallback(lambda did: self.multiengine.get_pending_deferred(did, True))
         d.addErrback(lambda f: self.assertRaises(NameError, _raise_it, f))
         return d
 
     def testClearQueue(self):
         self.addEngine(4)
         d= self.multiengine.clear_queue()
         d.addCallback(lambda r: self.multiengine.clear_queue(block=False))
         d.addCallback(lambda did: self.multiengine.get_pending_deferred(did, True))
         d.addCallback(lambda r: self.assertEquals(r,4*[None]))
         return d
 
     def testQueueStatus(self):
         self.addEngine(4)
         d= self.multiengine.queue_status(targets=0)
         d.addCallback(lambda r: self.assert_(isinstance(r[0],tuple)))
         d.addCallback(lambda r: self.multiengine.queue_status(targets=0, block=False))
         d.addCallback(lambda did: self.multiengine.get_pending_deferred(did, True))
         d.addCallback(lambda r: self.assert_(isinstance(r[0],tuple)))
         return d
 
     def testGetIDs(self):
         self.addEngine(1)
         d= self.multiengine.get_ids()
         d.addCallback(lambda r: self.assertEquals(r, [0]))
         d.addCallback(lambda _: self.addEngine(3))
         d.addCallback(lambda _: self.multiengine.get_ids())
         d.addCallback(lambda r: self.assertEquals(r, [0,1,2,3]))
         return d
 
     def testGetSetProperties(self):
         self.addEngine(4)
         dikt = dict(a=5, b='asdf', c=True, d=None, e=range(5))
         d= self.multiengine.set_properties(dikt)
         d.addCallback(lambda r: self.multiengine.get_properties())
         d.addCallback(lambda r: self.assertEquals(r, 4*[dikt]))
         d.addCallback(lambda r: self.multiengine.get_properties(('c',)))
         d.addCallback(lambda r: self.assertEquals(r, 4*[{'c': dikt['c']}]))
         d.addCallback(lambda r: self.multiengine.set_properties(dict(c=False)))
         d.addCallback(lambda r: self.multiengine.get_properties(('c', 'd')))
         d.addCallback(lambda r: self.assertEquals(r, 4*[dict(c=False, d=None)]))
 
         #Non-blocking
         d.addCallback(lambda r: self.multiengine.set_properties(dikt, block=False))
         d.addCallback(lambda did: self.multiengine.get_pending_deferred(did, True))
         d.addCallback(lambda r: self.multiengine.get_properties(block=False))
         d.addCallback(lambda did: self.multiengine.get_pending_deferred(did, True))
         d.addCallback(lambda r: self.assertEquals(r, 4*[dikt]))
         d.addCallback(lambda r: self.multiengine.get_properties(('c',), block=False))
         d.addCallback(lambda did: self.multiengine.get_pending_deferred(did, True))
         d.addCallback(lambda r: self.assertEquals(r, 4*[{'c': dikt['c']}]))
         d.addCallback(lambda r: self.multiengine.set_properties(dict(c=False), block=False))
         d.addCallback(lambda did: self.multiengine.get_pending_deferred(did, True))
         d.addCallback(lambda r: self.multiengine.get_properties(('c', 'd'), block=False))
         d.addCallback(lambda did: self.multiengine.get_pending_deferred(did, True))
         d.addCallback(lambda r: self.assertEquals(r, 4*[dict(c=False, d=None)]))
         return d
 
     def testClearProperties(self):
         self.addEngine(4)
         dikt = dict(a=5, b='asdf', c=True, d=None, e=range(5))
         d= self.multiengine.set_properties(dikt)
         d.addCallback(lambda r: self.multiengine.clear_properties())
         d.addCallback(lambda r: self.multiengine.get_properties())
         d.addCallback(lambda r: self.assertEquals(r, 4*[{}]))
 
         #Non-blocking
         d.addCallback(lambda r: self.multiengine.set_properties(dikt, block=False))
         d.addCallback(lambda did: self.multiengine.get_pending_deferred(did, True))
         d.addCallback(lambda r: self.multiengine.clear_properties(block=False))
         d.addCallback(lambda did: self.multiengine.get_pending_deferred(did, True))
         d.addCallback(lambda r: self.multiengine.get_properties(block=False))
         d.addCallback(lambda did: self.multiengine.get_pending_deferred(did, True))
         d.addCallback(lambda r: self.assertEquals(r, 4*[{}]))
         return d
 
     def testDelHasProperties(self):
         self.addEngine(4)
         dikt = dict(a=5, b='asdf', c=True, d=None, e=range(5))
         d= self.multiengine.set_properties(dikt)
         d.addCallback(lambda r: self.multiengine.del_properties(('b','e')))
         d.addCallback(lambda r: self.multiengine.has_properties(('a','b','c','d','e')))
         d.addCallback(lambda r: self.assertEquals(r, 4*[[True, False, True, True, False]]))
 
         #Non-blocking
         d.addCallback(lambda r: self.multiengine.set_properties(dikt, block=False))
         d.addCallback(lambda did: self.multiengine.get_pending_deferred(did, True))
         d.addCallback(lambda r: self.multiengine.del_properties(('b','e'), block=False))
         d.addCallback(lambda did: self.multiengine.get_pending_deferred(did, True))
         d.addCallback(lambda r: self.multiengine.has_properties(('a','b','c','d','e'), block=False))
         d.addCallback(lambda did: self.multiengine.get_pending_deferred(did, True))
         d.addCallback(lambda r: self.assertEquals(r, 4*[[True, False, True, True, False]]))
         return d
 
     def test_clear_pending_deferreds(self):
         self.addEngine(4)
         did_list = []
         d= self.multiengine.execute('a=10',block=False)
         d.addCallback(lambda did: did_list.append(did))
         d.addCallback(lambda _: self.multiengine.push(dict(b=10),block=False))
         d.addCallback(lambda did: did_list.append(did))
         d.addCallback(lambda _: self.multiengine.pull(('a','b'),block=False))
         d.addCallback(lambda did: did_list.append(did))
         d.addCallback(lambda _: self.multiengine.clear_pending_deferreds())
         d.addCallback(lambda _: self.multiengine.get_pending_deferred(did_list[0],True))
         d.addErrback(lambda f: self.assertRaises(InvalidDeferredID, f.raiseException))
         d.addCallback(lambda _: self.multiengine.get_pending_deferred(did_list[1],True))
         d.addErrback(lambda f: self.assertRaises(InvalidDeferredID, f.raiseException))
         d.addCallback(lambda _: self.multiengine.get_pending_deferred(did_list[2],True))
         d.addErrback(lambda f: self.assertRaises(InvalidDeferredID, f.raiseException))
         return d
 
 #-------------------------------------------------------------------------------
 # Coordinator test cases
 #-------------------------------------------------------------------------------
 
 class IMultiEngineCoordinatorTestCase(object):
 
     def testScatterGather(self):
         self.addEngine(4)
         d= self.multiengine.scatter('a', range(16))
         d.addCallback(lambda r: self.multiengine.gather('a'))
         d.addCallback(lambda r: self.assertEquals(r, range(16)))
         d.addCallback(lambda _: self.multiengine.gather('asdf'))
         d.addErrback(lambda f: self.assertRaises(NameError, _raise_it, f))
         return d
 
     def testScatterGatherNumpy(self):
         try:
             import numpy
             from numpy.testing.utils import assert_array_equal, assert_array_almost_equal
         except:
             return
         else:
             self.addEngine(4)
             a = numpy.arange(16)
             d = self.multiengine.scatter('a', a)
             d.addCallback(lambda r: self.multiengine.gather('a'))
             d.addCallback(lambda r: assert_array_equal(r, a))
             return d
 
     def testMap(self):
         self.addEngine(4)
         def f(x):
             return x**2
         data = range(16)
         d= self.multiengine.map(f, data)
         d.addCallback(lambda r: self.assertEquals(r,[f(x) for x in data]))
         return d
 
 
 class ISynchronousMultiEngineCoordinatorTestCase(IMultiEngineCoordinatorTestCase):
 
     def testScatterGatherNonblocking(self):
         self.addEngine(4)
         d= self.multiengine.scatter('a', range(16), block=False)
         d.addCallback(lambda did: self.multiengine.get_pending_deferred(did, True))
         d.addCallback(lambda r: self.multiengine.gather('a', block=False))
         d.addCallback(lambda did: self.multiengine.get_pending_deferred(did, True))
         d.addCallback(lambda r: self.assertEquals(r, range(16)))
         return d
 
     def testScatterGatherNumpyNonblocking(self):
         try:
             import numpy
             from numpy.testing.utils import assert_array_equal, assert_array_almost_equal
         except:
             return
         else:
             self.addEngine(4)
             a = numpy.arange(16)
             d = self.multiengine.scatter('a', a, block=False)
             d.addCallback(lambda did: self.multiengine.get_pending_deferred(did, True))
             d.addCallback(lambda r: self.multiengine.gather('a', block=False))
             d.addCallback(lambda did: self.multiengine.get_pending_deferred(did, True))
             d.addCallback(lambda r: assert_array_equal(r, a))
             return d
 
-    def testMapNonblocking(self):
-        self.addEngine(4)
-        def f(x):
-            return x**2
-        data = range(16)
-        d= self.multiengine.map(f, data, block=False)
-        d.addCallback(lambda did: self.multiengine.get_pending_deferred(did, True))
-        d.addCallback(lambda r: self.assertEquals(r,[f(x) for x in data]))
-        return d
-
     def test_clear_pending_deferreds(self):
         self.addEngine(4)
         did_list = []
         d= self.multiengine.scatter('a',range(16),block=False)
         d.addCallback(lambda did: did_list.append(did))
         d.addCallback(lambda _: self.multiengine.gather('a',block=False))
         d.addCallback(lambda did: did_list.append(did))
         d.addCallback(lambda _: self.multiengine.map(lambda x: x, range(16),block=False))
         d.addCallback(lambda did: did_list.append(did))
         d.addCallback(lambda _: self.multiengine.clear_pending_deferreds())
         d.addCallback(lambda _: self.multiengine.get_pending_deferred(did_list[0],True))
         d.addErrback(lambda f: self.assertRaises(InvalidDeferredID, f.raiseException))
         d.addCallback(lambda _: self.multiengine.get_pending_deferred(did_list[1],True))
         d.addErrback(lambda f: self.assertRaises(InvalidDeferredID, f.raiseException))
         d.addCallback(lambda _: self.multiengine.get_pending_deferred(did_list[2],True))
         d.addErrback(lambda f: self.assertRaises(InvalidDeferredID, f.raiseException))
         return d
 
 #-------------------------------------------------------------------------------
 # Extras test cases
 #-------------------------------------------------------------------------------
 
 class IMultiEngineExtrasTestCase(object):
 
     def testZipPull(self):
         self.addEngine(4)
         d= self.multiengine.push(dict(a=10,b=20))
         d.addCallback(lambda r: self.multiengine.zip_pull(('a','b')))
         d.addCallback(lambda r: self.assert_(r, [4*[10],4*[20]]))
         return d
 
     def testRun(self):
         self.addEngine(4)
         import tempfile
         fname = tempfile.mktemp('foo.py')
         f= open(fname, 'w')
         f.write('a = 10\nb=30')
         f.close()
         d= self.multiengine.run(fname)
         d.addCallback(lambda r: self.multiengine.pull(('a','b')))
         d.addCallback(lambda r: self.assertEquals(r, 4*[[10,30]]))
         return d
 
 
 class ISynchronousMultiEngineExtrasTestCase(IMultiEngineExtrasTestCase):
 
     def testZipPullNonblocking(self):
         self.addEngine(4)
         d= self.multiengine.push(dict(a=10,b=20))
         d.addCallback(lambda r: self.multiengine.zip_pull(('a','b'), block=False))
         d.addCallback(lambda did: self.multiengine.get_pending_deferred(did, True))
         d.addCallback(lambda r: self.assert_(r, [4*[10],4*[20]]))
         return d
 
     def testRunNonblocking(self):
         self.addEngine(4)
         import tempfile
         fname = tempfile.mktemp('foo.py')
         f= open(fname, 'w')
         f.write('a = 10\nb=30')
         f.close()
         d= self.multiengine.run(fname, block=False)
         d.addCallback(lambda did: self.multiengine.get_pending_deferred(did, True))
         d.addCallback(lambda r: self.multiengine.pull(('a','b')))
         d.addCallback(lambda r: self.assertEquals(r, 4*[[10,30]]))
         return d
 
 
 #-------------------------------------------------------------------------------
 # IFullSynchronousMultiEngineTestCase
 #-------------------------------------------------------------------------------
 
 class IFullSynchronousMultiEngineTestCase(ISynchronousMultiEngineTestCase,
     ISynchronousMultiEngineCoordinatorTestCase,
     ISynchronousMultiEngineExtrasTestCase):
     pass