Merging with upstream

IPython/frontend/asyncfrontendbase.py

@@ -1,76 +1,77 b''
 """
 Base front end class for all async frontends.
 """
 __docformat__ = "restructuredtext en"
 
 #-------------------------------------------------------------------------------
 #  Copyright (C) 2008  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-------------------------------------------------------------------------------
 
 
 #-------------------------------------------------------------------------------
 # Imports
 #-------------------------------------------------------------------------------
-from IPython.external import guid
 
+from IPython.external import guid
 
 from zope.interface import Interface, Attribute, implements, classProvides
 from twisted.python.failure import Failure
-from IPython.frontend.frontendbase import FrontEndBase, IFrontEnd, IFrontEndFactory
+from IPython.frontend.frontendbase import (
+    FrontEndBase, IFrontEnd, IFrontEndFactory)
 from IPython.kernel.core.history import FrontEndHistory
 from IPython.kernel.engineservice import IEngineCore
 
 
 class AsyncFrontEndBase(FrontEndBase):
     """
     Overrides FrontEndBase to wrap execute in a deferred result.
     All callbacks are made as callbacks on the deferred result.
     """
     
     implements(IFrontEnd)
     classProvides(IFrontEndFactory)
     
     def __init__(self, engine=None, history=None):
         assert(engine==None or IEngineCore.providedBy(engine))
         self.engine = IEngineCore(engine)
         if history is None:
                 self.history = FrontEndHistory(input_cache=[''])
         else:
             self.history = history
     
     
     def execute(self, block, blockID=None):
         """Execute the block and return the deferred result.
         
         Parameters:
             block : {str, AST}
             blockID : any
                 Caller may provide an ID to identify this block. 
                 result['blockID'] := blockID
         
         Result:
             Deferred result of self.interpreter.execute
         """
         
         if(not self.is_complete(block)):
             return Failure(Exception("Block is not compilable"))
         
         if(blockID == None):
             blockID = guid.generate() 
         
         d = self.engine.execute(block)
         d.addCallback(self._add_history, block=block)
         d.addCallbacks(self._add_block_id_for_result,
                 errback=self._add_block_id_for_failure,
                 callbackArgs=(blockID,),
                 errbackArgs=(blockID,))
         d.addBoth(self.update_cell_prompt, blockID=blockID)
         d.addCallbacks(self.render_result, 
             errback=self.render_error)
         
         return d
     
 

IPython/frontend/cocoa/tests/test_cocoa_frontend.py

@@ -1,94 +1,100 b''
 # encoding: utf-8
 """This file contains unittests for the 
 IPython.frontend.cocoa.cocoa_frontend module.
 """
 __docformat__ = "restructuredtext en"
                                                                             
 #---------------------------------------------------------------------------
 #       Copyright (C) 2005  The IPython Development Team    
 #                                                                           
 #  Distributed under the terms of the BSD License.  The full license is in  
 #  the file COPYING, distributed as part of this software.                  
 #---------------------------------------------------------------------------
                                                                             
 #---------------------------------------------------------------------------
 # Imports                                                                   
 #---------------------------------------------------------------------------
 
+# Tell nose to skip this module
+__test__ = {}
+
+from twisted.trial import unittest
+from twisted.internet.defer import succeed
+
+from IPython.kernel.core.interpreter import Interpreter
+import IPython.kernel.engineservice as es
+
 try:
-    from IPython.kernel.core.interpreter import Interpreter
-    import IPython.kernel.engineservice as es
-    from IPython.testing.util import DeferredTestCase
-    from twisted.internet.defer import succeed
-    from IPython.frontend.cocoa.cocoa_frontend import IPythonCocoaController                                                    
+    from IPython.frontend.cocoa.cocoa_frontend import IPythonCocoaController
     from Foundation import NSMakeRect
-    from AppKit import NSTextView, NSScrollView                                                
+    from AppKit import NSTextView, NSScrollView
 except ImportError:
-    import nose
-    raise nose.SkipTest("This test requires zope.interface, Twisted, Foolscap and PyObjC")
+    # This tells twisted.trial to skip this module if PyObjC is not found
+    skip = True
 
-class TestIPythonCocoaControler(DeferredTestCase):
+#---------------------------------------------------------------------------
+# Tests
+#---------------------------------------------------------------------------
+class TestIPythonCocoaControler(unittest.TestCase):
     """Tests for IPythonCocoaController"""
-   
+
     def setUp(self):
         self.controller = IPythonCocoaController.alloc().init()
         self.engine = es.EngineService()
         self.engine.startService()
-   
+
     def tearDown(self):
         self.controller = None
         self.engine.stopService()
-   
+
     def testControllerExecutesCode(self):
         code ="""5+5"""
         expected = Interpreter().execute(code)
         del expected['number']
         def removeNumberAndID(result):
             del result['number']
             del result['id']
             return result
-        self.assertDeferredEquals(
-            self.controller.execute(code).addCallback(removeNumberAndID), 
-            expected)
-   
+        d = self.controller.execute(code)
+        d.addCallback(removeNumberAndID)
+        d.addCallback(lambda r: self.assertEquals(r, expected))
+
     def testControllerMirrorsUserNSWithValuesAsStrings(self):
         code = """userns1=1;userns2=2"""
         def testControllerUserNS(result):
             self.assertEquals(self.controller.userNS['userns1'], 1)
             self.assertEquals(self.controller.userNS['userns2'], 2)
-       
         self.controller.execute(code).addCallback(testControllerUserNS)
-   
-   
+
     def testControllerInstantiatesIEngine(self):
         self.assert_(es.IEngineBase.providedBy(self.controller.engine))
-   
+
     def testControllerCompletesToken(self):
         code = """longNameVariable=10"""
         def testCompletes(result):
             self.assert_("longNameVariable" in result)
-       
+
         def testCompleteToken(result):
             self.controller.complete("longNa").addCallback(testCompletes)
-       
+
         self.controller.execute(code).addCallback(testCompletes)
-   
-   
+
+
     def testCurrentIndent(self):
         """test that current_indent_string returns current indent or None.
         Uses _indent_for_block for direct unit testing.
         """
-       
+        
         self.controller.tabUsesSpaces = True
         self.assert_(self.controller._indent_for_block("""a=3""") == None)
         self.assert_(self.controller._indent_for_block("") == None)
         block = """def test():\n    a=3"""
         self.assert_(self.controller._indent_for_block(block) == \
                     ' ' * self.controller.tabSpaces)
-       
+        
         block = """if(True):\n%sif(False):\n%spass""" % \
                     (' '*self.controller.tabSpaces,
                      2*' '*self.controller.tabSpaces)
         self.assert_(self.controller._indent_for_block(block) == \
                     2*(' '*self.controller.tabSpaces))
-   
+

IPython/frontend/tests/test_asyncfrontendbase.py

@@ -1,111 +1,109 b''
 # encoding: utf-8
 
 """This file contains unittests for the asyncfrontendbase module."""
 
 __docformat__ = "restructuredtext en"
 
 #---------------------------------------------------------------------------
 #  Copyright (C) 2008  The IPython Development Team                         
 #                                                                           
 #  Distributed under the terms of the BSD License.  The full license is in  
 #  the file COPYING, distributed as part of this software.                  
 #---------------------------------------------------------------------------
                                                                             
 #---------------------------------------------------------------------------
 # Imports                                                                   
 #---------------------------------------------------------------------------
 
+# Tell nose to skip this module
+__test__ = {}
 
-try:
-    from twisted.trial import unittest
-    from IPython.frontend.asyncfrontendbase import AsyncFrontEndBase
-    from IPython.frontend import frontendbase 
-    from IPython.kernel.engineservice import EngineService
-    from IPython.testing.parametric import Parametric, parametric
-except ImportError:
-    import nose
-    raise nose.SkipTest("This test requires zope.interface, Twisted and Foolscap")
+from twisted.trial import unittest
+from IPython.frontend.asyncfrontendbase import AsyncFrontEndBase
+from IPython.frontend import frontendbase 
+from IPython.kernel.engineservice import EngineService
+from IPython.testing.parametric import Parametric, parametric
 
 
 class FrontEndCallbackChecker(AsyncFrontEndBase):
     """FrontEndBase subclass for checking callbacks"""
     def __init__(self, engine=None, history=None):
         super(FrontEndCallbackChecker, self).__init__(engine=engine, 
                                                     history=history)
         self.updateCalled = False
         self.renderResultCalled = False
         self.renderErrorCalled = False
    
     def update_cell_prompt(self, result, blockID=None):
         self.updateCalled = True
         return result
    
     def render_result(self, result):
         self.renderResultCalled = True
         return result
     
     def render_error(self, failure):
         self.renderErrorCalled = True
         return failure
 
 
 class TestAsyncFrontendBase(unittest.TestCase):
     def setUp(self):
         """Setup the EngineService and FrontEndBase"""
         
         self.fb = FrontEndCallbackChecker(engine=EngineService())
     
     def test_implements_IFrontEnd(self):
         self.assert_(frontendbase.IFrontEnd.implementedBy(
                                     AsyncFrontEndBase))
     
     def test_is_complete_returns_False_for_incomplete_block(self):
         block = """def test(a):"""
         self.assert_(self.fb.is_complete(block) == False)
     
     def test_is_complete_returns_True_for_complete_block(self):
         block = """def test(a): pass"""
         self.assert_(self.fb.is_complete(block))
         block = """a=3"""
         self.assert_(self.fb.is_complete(block))
     
     def test_blockID_added_to_result(self):
         block = """3+3"""
         d = self.fb.execute(block, blockID='TEST_ID')
         d.addCallback(lambda r: self.assert_(r['blockID']=='TEST_ID'))
         return d
     
     def test_blockID_added_to_failure(self):
         block = "raise Exception()"
         d = self.fb.execute(block,blockID='TEST_ID')
         d.addErrback(lambda f: self.assert_(f.blockID=='TEST_ID'))
         return d
     
     def test_callbacks_added_to_execute(self):
         d = self.fb.execute("10+10")
         d.addCallback(lambda r: self.assert_(self.fb.updateCalled and self.fb.renderResultCalled))
         return d
     
     def test_error_callback_added_to_execute(self):
         """Test that render_error called on execution error."""
         
         d = self.fb.execute("raise Exception()")
         d.addErrback(lambda f: self.assert_(self.fb.renderErrorCalled))
         return d
     
     def test_history_returns_expected_block(self):
         """Make sure history browsing doesn't fail."""
         
         blocks = ["a=1","a=2","a=3"]
         d = self.fb.execute(blocks[0])
         d.addCallback(lambda _: self.fb.execute(blocks[1]))
         d.addCallback(lambda _: self.fb.execute(blocks[2]))
         d.addCallback(lambda _: self.assert_(self.fb.get_history_previous("")==blocks[-2]))
         d.addCallback(lambda _: self.assert_(self.fb.get_history_previous("")==blocks[-3]))
         d.addCallback(lambda _: self.assert_(self.fb.get_history_next()==blocks[-2]))
         return d
     
     def test_history_returns_none_at_startup(self):        
         self.assert_(self.fb.get_history_previous("")==None)
         self.assert_(self.fb.get_history_next()==None)
 

IPython/frontend/tests/test_process.py

@@ -1,67 +1,67 b''
 # encoding: utf-8
 """
 Test process execution and IO redirection.
 """
 
 __docformat__ = "restructuredtext en"
 
-#-------------------------------------------------------------------------------
-#  Copyright (C) 2008  The IPython Development Team
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2009  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is
 #  in the file COPYING, distributed as part of this software.
-#-------------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
 
 from cStringIO import StringIO
 from time import sleep
 import sys
 
 from IPython.frontend.process import PipedProcess
 from IPython.testing import decorators as testdec
 
 
 def test_capture_out():
     """ A simple test to see if we can execute a process and get the output.
     """
     s = StringIO()
     p = PipedProcess('echo 1', out_callback=s.write, )
     p.start()
     p.join()
     result = s.getvalue().rstrip()
     assert result == '1'
 
 
 def test_io():
     """ Checks that we can send characters on stdin to the process.
     """
     s = StringIO()
     p = PipedProcess(sys.executable + ' -c "a = raw_input(); print a"', 
                             out_callback=s.write, )
     p.start()
     test_string = '12345\n'
     while not hasattr(p, 'process'):
         sleep(0.1)
     p.process.stdin.write(test_string)
     p.join()
     result = s.getvalue()
     assert result == test_string
 
 
 def test_kill():
     """ Check that we can kill a process, and its subprocess.
     """
     s = StringIO()
     p = PipedProcess(sys.executable + ' -c "a = raw_input();"', 
                             out_callback=s.write, )
     p.start()
     while not hasattr(p, 'process'):
         sleep(0.1)
     p.process.kill() 
     assert p.process.poll() is not None
 
 
 if __name__ == '__main__':
     test_capture_out()
     test_io()
     test_kill()
 

IPython/kernel/core/tests/test_interpreter.py

@@ -1,61 +1,64 b''
 # encoding: utf-8
 
 """This file contains unittests for the interpreter.py module."""
 
 __docformat__ = "restructuredtext en"
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008-2009  The IPython Development Team                           
 #                                                                             
 #  Distributed under the terms of the BSD License.  The full license is in    
 #  the file COPYING, distributed as part of this software.                    
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports                                                                     
 #-----------------------------------------------------------------------------
 
 import unittest
 from IPython.kernel.core.interpreter import Interpreter
 
 #-----------------------------------------------------------------------------
 # Tests 
 #-----------------------------------------------------------------------------
 
+# Tell nose to skip this module
+__test__ = {}
+
 class TestInterpreter(unittest.TestCase):
 
     def test_unicode(self):
         """ Test unicode handling with the interpreter."""
         i = Interpreter()
         i.execute_python(u'print "ù"')
         i.execute_python('print "ù"')
 
     def test_ticket266993(self):
         """ Test for ticket 266993."""
         i = Interpreter()
         i.execute('str("""a\nb""")')
 
     def test_ticket364347(self):
         """Test for ticket 364347."""
         i = Interpreter()
         i.split_commands('str("a\\nb")')
 
     def test_split_commands(self):
         """ Test that commands are indeed individually split."""
         i = Interpreter()
         test_atoms = [('(1\n + 1)', ),
                       ('1', '1', ),
                       ]
         for atoms in test_atoms:
             atoms = [atom.rstrip() + '\n' for atom in atoms]
             self.assertEquals(i.split_commands(''.join(atoms)),atoms)
 
     def test_long_lines(self):
         """ Test for spurious syntax error created by the interpreter."""
         test_strings = [u'( 1 +\n 1\n )\n\n',
                         u'(1 \n + 1\n )\n\n',
                        ]
         i = Interpreter()
         for s in test_strings:
             i.execute(s)
 

IPython/kernel/core/tests/test_notification.py

@@ -1,171 +1,174 b''
 # encoding: utf-8
 
 """This file contains unittests for the notification.py module."""
 
 __docformat__ = "restructuredtext en"
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008  The IPython Development Team                           
 #                                                                             
 #  Distributed under the terms of the BSD License.  The full license is in    
 #  the file COPYING, distributed as part of this software.                    
 #-----------------------------------------------------------------------------
                                                                               
 #-----------------------------------------------------------------------------
 # Imports                                                                     
 #-----------------------------------------------------------------------------
 
+# Tell nose to skip this module
+__test__ = {}
+
 import unittest
 import IPython.kernel.core.notification as notification
 from nose.tools import timed
 
 #
 # Supporting test classes
 #
 
 class Observer(object):
     """docstring for Observer"""
     def __init__(self, expectedType, expectedSender, 
                     center=notification.sharedCenter, **kwargs):
         super(Observer, self).__init__()
         self.expectedType = expectedType
         self.expectedSender = expectedSender
         self.expectedKwArgs = kwargs
         self.recieved = False
         center.add_observer(self.callback, 
                             self.expectedType, 
                             self.expectedSender)
     
     
     def callback(self, theType, sender, args={}):
         """callback"""
         
         assert(theType == self.expectedType or
                 self.expectedType == None)
         assert(sender == self.expectedSender or
                 self.expectedSender == None)
         assert(args == self.expectedKwArgs)
         self.recieved = True
     
     
     def verify(self):
         """verify"""
         
         assert(self.recieved)
     
     def reset(self):
         """reset"""
         
         self.recieved = False
     
 
 
 class Notifier(object):
     """docstring for Notifier"""
     def __init__(self, theType, **kwargs):
         super(Notifier, self).__init__()
         self.theType = theType
         self.kwargs = kwargs
     
     def post(self, center=notification.sharedCenter):
         """fire"""
         
         center.post_notification(self.theType, self,
             **self.kwargs)
     
 
 #
 # Test Cases
 #
 
 class NotificationTests(unittest.TestCase):
     """docstring for NotificationTests"""
     
     def tearDown(self):
         notification.sharedCenter.remove_all_observers()
     
     def test_notification_delivered(self):
         """Test that notifications are delivered"""
         expectedType = 'EXPECTED_TYPE'
         sender = Notifier(expectedType)
         observer = Observer(expectedType, sender)
         
         sender.post()
         
         observer.verify()
     
     
     def test_type_specificity(self):
         """Test that observers are registered by type"""
         
         expectedType = 1
         unexpectedType = "UNEXPECTED_TYPE"
         sender = Notifier(expectedType)
         unexpectedSender = Notifier(unexpectedType)
         observer = Observer(expectedType, sender)
     
         sender.post()
         unexpectedSender.post()
     
         observer.verify()
     
     
     def test_sender_specificity(self):
         """Test that observers are registered by sender"""
         
         expectedType = "EXPECTED_TYPE"
         sender1 = Notifier(expectedType)
         sender2 = Notifier(expectedType)
         observer = Observer(expectedType, sender1)
         
         sender1.post()
         sender2.post()
         
         observer.verify()
     
     
     def test_remove_all_observers(self):
         """White-box test for remove_all_observers"""
         
         for i in xrange(10):
             Observer('TYPE', None, center=notification.sharedCenter)
         
         self.assert_(len(notification.sharedCenter.observers[('TYPE',None)]) >= 10, 
             "observers registered")
         
         notification.sharedCenter.remove_all_observers()
         
         self.assert_(len(notification.sharedCenter.observers) == 0, "observers removed")
     
     
     def test_any_sender(self):
         """test_any_sender"""
         
         expectedType = "EXPECTED_TYPE"
         sender1 = Notifier(expectedType)
         sender2 = Notifier(expectedType)
         observer = Observer(expectedType, None)
         
         
         sender1.post()
         observer.verify()
         
         observer.reset()
         sender2.post()
         observer.verify()
         
     
     @timed(.01)
     def test_post_performance(self):
         """Test that post_notification, even with many registered irrelevant
         observers is fast"""
         
         for i in xrange(10):
             Observer("UNRELATED_TYPE", None)
         
         o = Observer('EXPECTED_TYPE', None)
         
         notification.sharedCenter.post_notification('EXPECTED_TYPE', self)
         
         o.verify()
     

IPython/kernel/core/tests/test_redirectors.py

@@ -1,70 +1,70 b''
 # encoding: utf-8
 """
 Test the output capture at the OS level, using file descriptors.
 """
 
 __docformat__ = "restructuredtext en"
 
 #-------------------------------------------------------------------------------
 #  Copyright (C) 2008  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is
 #  in the file COPYING, distributed as part of this software.
 #-------------------------------------------------------------------------------
 
+# Tell nose to skip this module
+__test__ = {}
 
-# Stdlib imports
 import os
 from cStringIO import StringIO
 
-# Our own imports
 from IPython.testing import decorators as dec
 
 #-----------------------------------------------------------------------------
 # Test functions
 
 @dec.skip_win32
 def test_redirector():
     """ Checks that the redirector can be used to do synchronous capture.
     """
     from IPython.kernel.core.fd_redirector import FDRedirector
     r = FDRedirector()
     out = StringIO()
     try:
         r.start()
         for i in range(10):
             os.system('echo %ic' % i)
             print >>out, r.getvalue(),
             print >>out, i
     except:
         r.stop()
         raise
     r.stop()
     result1 = out.getvalue()
     result2 = "".join("%ic\n%i\n" %(i, i) for i in range(10))
     assert result1 == result2
 
 
 @dec.skip_win32
 def test_redirector_output_trap():
     """ This test check not only that the redirector_output_trap does
         trap the output, but also that it does it in a gready way, that
         is by calling the callback ASAP.
     """
     from IPython.kernel.core.redirector_output_trap import RedirectorOutputTrap
     out = StringIO()
     trap = RedirectorOutputTrap(out.write, out.write)
     try:
         trap.set()
         for i in range(10):
             os.system('echo %ic' % i)
             print "%ip" % i
             print >>out, i
     except:
         trap.unset()
         raise
     trap.unset()
     result1 = out.getvalue()
     result2 = "".join("%ic\n%ip\n%i\n" %(i, i, i) for i in range(10))
     assert result1 == result2
     

IPython/kernel/core/ultraTB.py

@@ -1,1064 +1,1066 b''
 # -*- coding: utf-8 -*-
 """
 ultraTB.py -- Spice up your tracebacks!
 
 * ColorTB
 I've always found it a bit hard to visually parse tracebacks in Python.  The
 ColorTB class is a solution to that problem.  It colors the different parts of a
 traceback in a manner similar to what you would expect from a syntax-highlighting
 text editor.
 
 Installation instructions for ColorTB:
     import sys,ultraTB
     sys.excepthook = ultraTB.ColorTB()
 
 * VerboseTB  
 I've also included a port of Ka-Ping Yee's "cgitb.py" that produces all kinds
 of useful info when a traceback occurs.  Ping originally had it spit out HTML
 and intended it for CGI programmers, but why should they have all the fun?  I
 altered it to spit out colored text to the terminal.  It's a bit overwhelming,
 but kind of neat, and maybe useful for long-running programs that you believe
 are bug-free.  If a crash *does* occur in that type of program you want details.
 Give it a shot--you'll love it or you'll hate it.
 
 Note:
 
   The Verbose mode prints the variables currently visible where the exception
   happened (shortening their strings if too long). This can potentially be
   very slow, if you happen to have a huge data structure whose string
   representation is complex to compute. Your computer may appear to freeze for
   a while with cpu usage at 100%. If this occurs, you can cancel the traceback
   with Ctrl-C (maybe hitting it more than once).
 
   If you encounter this kind of situation often, you may want to use the
   Verbose_novars mode instead of the regular Verbose, which avoids formatting
   variables (but otherwise includes the information and context given by
   Verbose).
   
 
 Installation instructions for ColorTB:
     import sys,ultraTB
     sys.excepthook = ultraTB.VerboseTB()
 
 Note:  Much of the code in this module was lifted verbatim from the standard
 library module 'traceback.py' and Ka-Ping Yee's 'cgitb.py'.
 
 * Color schemes
 The colors are defined in the class TBTools through the use of the
 ColorSchemeTable class. Currently the following exist:
 
   - NoColor: allows all of this module to be used in any terminal (the color
   escapes are just dummy blank strings).
 
   - Linux: is meant to look good in a terminal like the Linux console (black
   or very dark background).
 
   - LightBG: similar to Linux but swaps dark/light colors to be more readable
   in light background terminals.
 
 You can implement other color schemes easily, the syntax is fairly
 self-explanatory. Please send back new schemes you develop to the author for
 possible inclusion in future releases.
 """
 
 #*****************************************************************************
 #       Copyright (C) 2001 Nathaniel Gray <[e-mail]>
 #       Copyright (C) 2001-2004 Fernando Perez <[e-mail]>
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #*****************************************************************************
 
 # Required modules
 import inspect
 import keyword
 import linecache
 import os
 import pydoc
 import re
 import string
 import sys
 import time
 import tokenize
 import traceback
 import types
 
 # For purposes of monkeypatching inspect to fix a bug in it.
 from inspect import getsourcefile, getfile, getmodule,\
      ismodule,  isclass, ismethod, isfunction, istraceback, isframe, iscode
 
 
 # IPython's own modules
 # Modified pdb which doesn't damage IPython's readline handling
 from IPython import Debugger, PyColorize
 from IPython.ipstruct import Struct
 from IPython.excolors import exception_colors
 from IPython.genutils import Term,uniq_stable,error,info
 
 # Globals
 # amount of space to put line numbers before verbose tracebacks
 INDENT_SIZE = 8
 
 # Default color scheme.  This is used, for example, by the traceback
 # formatter.  When running in an actual IPython instance, the user's rc.colors
 # value is used, but havinga module global makes this functionality available
 # to users of ultraTB who are NOT running inside ipython.
 DEFAULT_SCHEME = 'NoColor'
 
 #---------------------------------------------------------------------------
 # Code begins
 
 # Utility functions
 def inspect_error():
     """Print a message about internal inspect errors.
 
     These are unfortunately quite common."""
     
     error('Internal Python error in the inspect module.\n'
           'Below is the traceback from this internal error.\n')
 
 
 def findsource(object):
     """Return the entire source file and starting line number for an object.
 
     The argument may be a module, class, method, function, traceback, frame,
     or code object.  The source code is returned as a list of all the lines
     in the file and the line number indexes a line in that list.  An IOError
     is raised if the source code cannot be retrieved.
 
     FIXED version with which we monkeypatch the stdlib to work around a bug."""
 
     file = getsourcefile(object) or getfile(object)
     # If the object is a frame, then trying to get the globals dict from its
     # module won't work. Instead, the frame object itself has the globals
     # dictionary.
     globals_dict = None
     if inspect.isframe(object):
         # XXX: can this ever be false?
         globals_dict = object.f_globals
     else:
         module = getmodule(object, file)
         if module:
             globals_dict = module.__dict__
     lines = linecache.getlines(file, globals_dict)
     if not lines:
         raise IOError('could not get source code')
 
     if ismodule(object):
         return lines, 0
 
     if isclass(object):
         name = object.__name__
         pat = re.compile(r'^(\s*)class\s*' + name + r'\b')
         # make some effort to find the best matching class definition:
         # use the one with the least indentation, which is the one
         # that's most probably not inside a function definition.
         candidates = []
         for i in range(len(lines)):
             match = pat.match(lines[i])
             if match:
                 # if it's at toplevel, it's already the best one
                 if lines[i][0] == 'c':
                     return lines, i
                 # else add whitespace to candidate list
                 candidates.append((match.group(1), i))
         if candidates:
             # this will sort by whitespace, and by line number,
             # less whitespace first
             candidates.sort()
             return lines, candidates[0][1]
         else:
             raise IOError('could not find class definition')
 
     if ismethod(object):
         object = object.im_func
     if isfunction(object):
         object = object.func_code
     if istraceback(object):
         object = object.tb_frame
     if isframe(object):
         object = object.f_code
     if iscode(object):
         if not hasattr(object, 'co_firstlineno'):
             raise IOError('could not find function definition')
         pat = re.compile(r'^(\s*def\s)|(.*(?<!\w)lambda(:|\s))|^(\s*@)')
         pmatch = pat.match
         # fperez - fix: sometimes, co_firstlineno can give a number larger than
         # the length of lines, which causes an error.  Safeguard against that.
         lnum = min(object.co_firstlineno,len(lines))-1
         while lnum > 0:
             if pmatch(lines[lnum]): break
             lnum -= 1
  
         return lines, lnum
     raise IOError('could not find code object')
 
 # Monkeypatch inspect to apply our bugfix.  This code only works with py25
 if sys.version_info[:2] >= (2,5):
     inspect.findsource = findsource
 
 def fix_frame_records_filenames(records):
     """Try to fix the filenames in each record from inspect.getinnerframes().
 
     Particularly, modules loaded from within zip files have useless filenames
     attached to their code object, and inspect.getinnerframes() just uses it.
     """
     fixed_records = []
     for frame, filename, line_no, func_name, lines, index in records:
         # Look inside the frame's globals dictionary for __file__, which should
         # be better.
         better_fn = frame.f_globals.get('__file__', None)
         if isinstance(better_fn, str):
             # Check the type just in case someone did something weird with
             # __file__. It might also be None if the error occurred during
             # import.
             filename = better_fn
         fixed_records.append((frame, filename, line_no, func_name, lines, index))
     return fixed_records
 
 
 def _fixed_getinnerframes(etb, context=1,tb_offset=0):
     import linecache
     LNUM_POS, LINES_POS, INDEX_POS =  2, 4, 5
 
     records  = fix_frame_records_filenames(inspect.getinnerframes(etb, context))
 
     # If the error is at the console, don't build any context, since it would
     # otherwise produce 5 blank lines printed out (there is no file at the
     # console)
     rec_check = records[tb_offset:]
     try:
         rname = rec_check[0][1]
         if rname == '<ipython console>' or rname.endswith('<string>'):
             return rec_check
     except IndexError:
         pass
 
     aux = traceback.extract_tb(etb)
     assert len(records) == len(aux)
     for i, (file, lnum, _, _) in zip(range(len(records)), aux):
         maybeStart = lnum-1 - context//2
         start =  max(maybeStart, 0)
         end   = start + context
         lines = linecache.getlines(file)[start:end]
         # pad with empty lines if necessary
         if maybeStart < 0:
             lines = (['\n'] * -maybeStart) + lines
         if len(lines) < context:
             lines += ['\n'] * (context - len(lines))
         buf = list(records[i])
         buf[LNUM_POS] = lnum
         buf[INDEX_POS] = lnum - 1 - start
         buf[LINES_POS] = lines
         records[i] = tuple(buf)
     return records[tb_offset:]
 
 # Helper function -- largely belongs to VerboseTB, but we need the same
 # functionality to produce a pseudo verbose TB for SyntaxErrors, so that they
 # can be recognized properly by ipython.el's py-traceback-line-re
 # (SyntaxErrors have to be treated specially because they have no traceback)
 
 _parser = PyColorize.Parser()
     
 def _formatTracebackLines(lnum, index, lines, Colors, lvals=None,scheme=None):
     numbers_width = INDENT_SIZE - 1
     res = []
     i = lnum - index
 
     # This lets us get fully syntax-highlighted tracebacks.
     if scheme is None:
         try:
+            # Again, reference to a global __IPYTHON__ that doesn't exist.
+            # XXX
             scheme = __IPYTHON__.rc.colors
         except:
             scheme = DEFAULT_SCHEME
     _line_format = _parser.format2
 
     for line in lines:
         new_line, err = _line_format(line,'str',scheme)
         if not err: line = new_line
         
         if i == lnum:
             # This is the line with the error
             pad = numbers_width - len(str(i))
             if pad >= 3:
                 marker = '-'*(pad-3) + '-> '
             elif pad == 2:
                 marker = '> '
             elif pad == 1:
                 marker = '>'
             else:
                 marker = ''
             num = marker + str(i)
             line = '%s%s%s %s%s' %(Colors.linenoEm, num, 
                                    Colors.line, line, Colors.Normal)
         else:
             num = '%*s' % (numbers_width,i)
             line = '%s%s%s %s' %(Colors.lineno, num, 
                                  Colors.Normal, line)
 
         res.append(line)
         if lvals and i == lnum:
             res.append(lvals + '\n')
         i = i + 1
     return res
 
 
 #---------------------------------------------------------------------------
 # Module classes
 class TBTools:
     """Basic tools used by all traceback printer classes."""
 
     def __init__(self,color_scheme = 'NoColor',call_pdb=False):
         # Whether to call the interactive pdb debugger after printing
         # tracebacks or not
         self.call_pdb = call_pdb
 
         # Create color table
         self.color_scheme_table = exception_colors()
 
         self.set_colors(color_scheme)
         self.old_scheme = color_scheme  # save initial value for toggles
 
         if call_pdb:
             self.pdb = Debugger.Pdb(self.color_scheme_table.active_scheme_name)
         else:
             self.pdb = None
 
     def set_colors(self,*args,**kw):
         """Shorthand access to the color table scheme selector method."""
 
         # Set own color table
         self.color_scheme_table.set_active_scheme(*args,**kw)
         # for convenience, set Colors to the active scheme
         self.Colors = self.color_scheme_table.active_colors
         # Also set colors of debugger
         if hasattr(self,'pdb') and self.pdb is not None:
             self.pdb.set_colors(*args,**kw)
 
     def color_toggle(self):
         """Toggle between the currently active color scheme and NoColor."""
         
         if self.color_scheme_table.active_scheme_name == 'NoColor':
             self.color_scheme_table.set_active_scheme(self.old_scheme)
             self.Colors = self.color_scheme_table.active_colors
         else:
             self.old_scheme = self.color_scheme_table.active_scheme_name
             self.color_scheme_table.set_active_scheme('NoColor')
             self.Colors = self.color_scheme_table.active_colors
 
 #---------------------------------------------------------------------------
 class ListTB(TBTools):
     """Print traceback information from a traceback list, with optional color.
         
     Calling: requires 3 arguments:
       (etype, evalue, elist)
     as would be obtained by:
       etype, evalue, tb = sys.exc_info()
       if tb:
         elist = traceback.extract_tb(tb)
       else:
         elist = None
 
     It can thus be used by programs which need to process the traceback before
     printing (such as console replacements based on the code module from the
     standard library).
 
     Because they are meant to be called without a full traceback (only a
     list), instances of this class can't call the interactive pdb debugger."""
 
     def __init__(self,color_scheme = 'NoColor'):
         TBTools.__init__(self,color_scheme = color_scheme,call_pdb=0)
         
     def __call__(self, etype, value, elist):
         Term.cout.flush()
         print >> Term.cerr, self.text(etype,value,elist)
         Term.cerr.flush()
 
     def text(self,etype, value, elist,context=5):
         """Return a color formatted string with the traceback info."""
 
         Colors = self.Colors
         out_string = ['%s%s%s\n' % (Colors.topline,'-'*60,Colors.Normal)]
         if elist:
             out_string.append('Traceback %s(most recent call last)%s:' % \
                                 (Colors.normalEm, Colors.Normal) + '\n')
             out_string.extend(self._format_list(elist))
         lines = self._format_exception_only(etype, value)
         for line in lines[:-1]:
             out_string.append(" "+line)
         out_string.append(lines[-1])
         return ''.join(out_string)
 
     def _format_list(self, extracted_list):
         """Format a list of traceback entry tuples for printing.
 
         Given a list of tuples as returned by extract_tb() or
         extract_stack(), return a list of strings ready for printing.
         Each string in the resulting list corresponds to the item with the
         same index in the argument list.  Each string ends in a newline;
         the strings may contain internal newlines as well, for those items
         whose source text line is not None.
         
         Lifted almost verbatim from traceback.py
         """
 
         Colors = self.Colors
         list = []
         for filename, lineno, name, line in extracted_list[:-1]:
             item = '  File %s"%s"%s, line %s%d%s, in %s%s%s\n' % \
                     (Colors.filename, filename, Colors.Normal, 
                      Colors.lineno, lineno, Colors.Normal,
                      Colors.name, name, Colors.Normal)
             if line:
                 item = item + '    %s\n' % line.strip()
             list.append(item)
         # Emphasize the last entry
         filename, lineno, name, line = extracted_list[-1]
         item = '%s  File %s"%s"%s, line %s%d%s, in %s%s%s%s\n' % \
                 (Colors.normalEm,
                  Colors.filenameEm, filename, Colors.normalEm,
                  Colors.linenoEm, lineno, Colors.normalEm,
                  Colors.nameEm, name, Colors.normalEm,
                  Colors.Normal)
         if line:
             item = item + '%s    %s%s\n' % (Colors.line, line.strip(),
                                             Colors.Normal)
         list.append(item)
         return list
         
     def _format_exception_only(self, etype, value):
         """Format the exception part of a traceback.
 
         The arguments are the exception type and value such as given by
         sys.exc_info()[:2]. The return value is a list of strings, each ending
         in a newline.  Normally, the list contains a single string; however,
         for SyntaxError exceptions, it contains several lines that (when
         printed) display detailed information about where the syntax error
         occurred.  The message indicating which exception occurred is the
         always last string in the list.
         
         Also lifted nearly verbatim from traceback.py
         """
 
         have_filedata = False
         Colors = self.Colors
         list = []
         try:
             stype = Colors.excName + etype.__name__ + Colors.Normal
         except AttributeError:
             stype = etype  # String exceptions don't get special coloring
         if value is None:
             list.append( str(stype) + '\n')
         else:
             if etype is SyntaxError:
                 try:
                     msg, (filename, lineno, offset, line) = value
                 except:
                     have_filedata = False
                 else:
                     have_filedata = True
                     #print 'filename is',filename  # dbg
                     if not filename: filename = "<string>"
                     list.append('%s  File %s"%s"%s, line %s%d%s\n' % \
                             (Colors.normalEm,
                              Colors.filenameEm, filename, Colors.normalEm,
                              Colors.linenoEm, lineno, Colors.Normal  ))
                     if line is not None:
                         i = 0
                         while i < len(line) and line[i].isspace():
                             i = i+1
                         list.append('%s    %s%s\n' % (Colors.line,
                                                       line.strip(), 
                                                       Colors.Normal))
                         if offset is not None:
                             s = '    '
                             for c in line[i:offset-1]:
                                 if c.isspace():
                                     s = s + c
                                 else:
                                     s = s + ' '
                             list.append('%s%s^%s\n' % (Colors.caret, s,
                                                        Colors.Normal) )
                         value = msg
             s = self._some_str(value)
             if s:
                 list.append('%s%s:%s %s\n' % (str(stype), Colors.excName,
                                               Colors.Normal, s))
             else:
                 list.append('%s\n' % str(stype))
 
         # This is being commented out for now as the __IPYTHON__ variable
         # referenced here is not resolved and causes massive test failures
-        # and errors. B. Granger, 04/2009.
+        # and errors. B. Granger, 04/2009. XXX
         # See https://bugs.launchpad.net/bugs/362137
         # # vds:>>
         # if have_filedata:
         #     __IPYTHON__.hooks.synchronize_with_editor(filename, lineno, 0)
         # # vds:<<
 
         return list
 
     def _some_str(self, value):
         # Lifted from traceback.py
         try:
             return str(value)
         except:
             return '<unprintable %s object>' % type(value).__name__
 
 #----------------------------------------------------------------------------
 class VerboseTB(TBTools):
     """A port of Ka-Ping Yee's cgitb.py module that outputs color text instead
     of HTML.  Requires inspect and pydoc.  Crazy, man.
 
     Modified version which optionally strips the topmost entries from the
     traceback, to be used with alternate interpreters (because their own code
     would appear in the traceback)."""
 
     def __init__(self,color_scheme = 'Linux',tb_offset=0,long_header=0,
                  call_pdb = 0, include_vars=1):
         """Specify traceback offset, headers and color scheme.
 
         Define how many frames to drop from the tracebacks. Calling it with
         tb_offset=1 allows use of this handler in interpreters which will have
         their own code at the top of the traceback (VerboseTB will first
         remove that frame before printing the traceback info)."""
         TBTools.__init__(self,color_scheme=color_scheme,call_pdb=call_pdb)
         self.tb_offset = tb_offset
         self.long_header = long_header
         self.include_vars = include_vars
 
     def text(self, etype, evalue, etb, context=5):
         """Return a nice text document describing the traceback."""
 
         # some locals
         try:
             etype = etype.__name__
         except AttributeError:
             pass
         Colors        = self.Colors   # just a shorthand + quicker name lookup
         ColorsNormal  = Colors.Normal  # used a lot
         col_scheme    = self.color_scheme_table.active_scheme_name
         indent        = ' '*INDENT_SIZE
         em_normal     = '%s\n%s%s' % (Colors.valEm, indent,ColorsNormal)
         undefined     = '%sundefined%s' % (Colors.em, ColorsNormal)
         exc = '%s%s%s' % (Colors.excName,etype,ColorsNormal)
 
         # some internal-use functions
         def text_repr(value):
             """Hopefully pretty robust repr equivalent."""
             # this is pretty horrible but should always return *something*
             try:
                 return pydoc.text.repr(value)
             except KeyboardInterrupt:
                 raise
             except:
                 try:
                     return repr(value)
                 except KeyboardInterrupt:
                     raise
                 except:
                     try:
                         # all still in an except block so we catch
                         # getattr raising
                         name = getattr(value, '__name__', None)
                         if name:
                             # ick, recursion
                             return text_repr(name)
                         klass = getattr(value, '__class__', None)
                         if klass:
                             return '%s instance' % text_repr(klass)
                     except KeyboardInterrupt:
                         raise
                     except:
                         return 'UNRECOVERABLE REPR FAILURE'
         def eqrepr(value, repr=text_repr): return '=%s' % repr(value)
         def nullrepr(value, repr=text_repr): return ''
 
         # meat of the code begins
         try:
             etype = etype.__name__
         except AttributeError:
             pass
 
         if self.long_header:
             # Header with the exception type, python version, and date
             pyver = 'Python ' + string.split(sys.version)[0] + ': ' + sys.executable
             date = time.ctime(time.time())
             
             head = '%s%s%s\n%s%s%s\n%s' % (Colors.topline, '-'*75, ColorsNormal,
                                            exc, ' '*(75-len(str(etype))-len(pyver)),
                                            pyver, string.rjust(date, 75) )
             head += "\nA problem occured executing Python code.  Here is the sequence of function"\
                     "\ncalls leading up to the error, with the most recent (innermost) call last."
         else:
             # Simplified header
             head = '%s%s%s\n%s%s' % (Colors.topline, '-'*75, ColorsNormal,exc,
                                      string.rjust('Traceback (most recent call last)',
                                                   75 - len(str(etype)) ) )
         frames = []
         # Flush cache before calling inspect.  This helps alleviate some of the
         # problems with python 2.3's inspect.py.
         linecache.checkcache()
         # Drop topmost frames if requested
         try:
             # Try the default getinnerframes and Alex's: Alex's fixes some
             # problems, but it generates empty tracebacks for console errors
             # (5 blanks lines) where none should be returned.
             #records = inspect.getinnerframes(etb, context)[self.tb_offset:]
             #print 'python records:', records # dbg
             records = _fixed_getinnerframes(etb, context,self.tb_offset)
             #print 'alex   records:', records # dbg
         except:
 
             # FIXME: I've been getting many crash reports from python 2.3
             # users, traceable to inspect.py.  If I can find a small test-case
             # to reproduce this, I should either write a better workaround or
             # file a bug report against inspect (if that's the real problem).
             # So far, I haven't been able to find an isolated example to
             # reproduce the problem.
             inspect_error()
             traceback.print_exc(file=Term.cerr)
             info('\nUnfortunately, your original traceback can not be constructed.\n')
             return ''
 
         # build some color string templates outside these nested loops
         tpl_link       = '%s%%s%s' % (Colors.filenameEm,ColorsNormal)
         tpl_call       = 'in %s%%s%s%%s%s' % (Colors.vName, Colors.valEm,
                                               ColorsNormal)
         tpl_call_fail  = 'in %s%%s%s(***failed resolving arguments***)%s' % \
                          (Colors.vName, Colors.valEm, ColorsNormal)
         tpl_local_var  = '%s%%s%s' % (Colors.vName, ColorsNormal)
         tpl_global_var = '%sglobal%s %s%%s%s' % (Colors.em, ColorsNormal,
                                                  Colors.vName, ColorsNormal)
         tpl_name_val   = '%%s %s= %%s%s' % (Colors.valEm, ColorsNormal)
         tpl_line       = '%s%%s%s %%s' % (Colors.lineno, ColorsNormal)
         tpl_line_em    = '%s%%s%s %%s%s' % (Colors.linenoEm,Colors.line,
                                             ColorsNormal)
 
         # now, loop over all records printing context and info
         abspath = os.path.abspath
         for frame, file, lnum, func, lines, index in records:
             #print '*** record:',file,lnum,func,lines,index  # dbg
             try:
                 file = file and abspath(file) or '?'
             except OSError:
                 # if file is '<console>' or something not in the filesystem,
                 # the abspath call will throw an OSError.  Just ignore it and
                 # keep the original file string.
                 pass
             link = tpl_link % file
             try:
                 args, varargs, varkw, locals = inspect.getargvalues(frame)
             except:
                 # This can happen due to a bug in python2.3.  We should be
                 # able to remove this try/except when 2.4 becomes a
                 # requirement.  Bug details at http://python.org/sf/1005466
                 inspect_error()
                 traceback.print_exc(file=Term.cerr)
                 info("\nIPython's exception reporting continues...\n")
                 
             if func == '?':
                 call = ''
             else:
                 # Decide whether to include variable details or not
                 var_repr = self.include_vars and eqrepr or nullrepr
                 try:
                     call = tpl_call % (func,inspect.formatargvalues(args,
                                                 varargs, varkw,
                                                 locals,formatvalue=var_repr))
                 except KeyError:
                     # Very odd crash from inspect.formatargvalues().  The
                     # scenario under which it appeared was a call to
                     # view(array,scale) in NumTut.view.view(), where scale had
                     # been defined as a scalar (it should be a tuple). Somehow
                     # inspect messes up resolving the argument list of view()
                     # and barfs out. At some point I should dig into this one
                     # and file a bug report about it.
                     inspect_error()
                     traceback.print_exc(file=Term.cerr)
                     info("\nIPython's exception reporting continues...\n")
                     call = tpl_call_fail % func
 
             # Initialize a list of names on the current line, which the
             # tokenizer below will populate.
             names = []
 
             def tokeneater(token_type, token, start, end, line):
                 """Stateful tokeneater which builds dotted names.
 
                 The list of names it appends to (from the enclosing scope) can
                 contain repeated composite names.  This is unavoidable, since
                 there is no way to disambguate partial dotted structures until
                 the full list is known.  The caller is responsible for pruning
                 the final list of duplicates before using it."""
                 
                 # build composite names
                 if token == '.':
                     try:
                         names[-1] += '.'
                         # store state so the next token is added for x.y.z names
                         tokeneater.name_cont = True
                         return
                     except IndexError:
                         pass
                 if token_type == tokenize.NAME and token not in keyword.kwlist:
                     if tokeneater.name_cont:
                         # Dotted names
                         names[-1] += token
                         tokeneater.name_cont = False
                     else:
                         # Regular new names.  We append everything, the caller
                         # will be responsible for pruning the list later.  It's
                         # very tricky to try to prune as we go, b/c composite
                         # names can fool us.  The pruning at the end is easy
                         # to do (or the caller can print a list with repeated
                         # names if so desired.
                         names.append(token)
                 elif token_type == tokenize.NEWLINE:
                     raise IndexError
             # we need to store a bit of state in the tokenizer to build
             # dotted names
             tokeneater.name_cont = False
 
             def linereader(file=file, lnum=[lnum], getline=linecache.getline):
                 line = getline(file, lnum[0])
                 lnum[0] += 1
                 return line
 
             # Build the list of names on this line of code where the exception
             # occurred.
             try:
                 # This builds the names list in-place by capturing it from the
                 # enclosing scope.
                 tokenize.tokenize(linereader, tokeneater)
             except IndexError:
                 # signals exit of tokenizer
                 pass
             except tokenize.TokenError,msg:
                 _m = ("An unexpected error occurred while tokenizing input\n"
                       "The following traceback may be corrupted or invalid\n"
                       "The error message is: %s\n" % msg)
                 error(_m)
             
             # prune names list of duplicates, but keep the right order
             unique_names = uniq_stable(names)
 
             # Start loop over vars
             lvals = []
             if self.include_vars:
                 for name_full in unique_names:
                     name_base = name_full.split('.',1)[0]
                     if name_base in frame.f_code.co_varnames:
                         if locals.has_key(name_base):
                             try:
                                 value = repr(eval(name_full,locals))
                             except:
                                 value = undefined
                         else:
                             value = undefined
                         name = tpl_local_var % name_full
                     else:
                         if frame.f_globals.has_key(name_base):
                             try:
                                 value = repr(eval(name_full,frame.f_globals))
                             except:
                                 value = undefined
                         else:
                             value = undefined
                         name = tpl_global_var % name_full
                     lvals.append(tpl_name_val % (name,value))
             if lvals:
                 lvals = '%s%s' % (indent,em_normal.join(lvals))
             else:
                 lvals = ''
 
             level = '%s %s\n' % (link,call)
 
             if index is None:
                 frames.append(level)
             else:
                 frames.append('%s%s' % (level,''.join(
                     _formatTracebackLines(lnum,index,lines,Colors,lvals,
                                           col_scheme))))
 
         # Get (safely) a string form of the exception info
         try:
             etype_str,evalue_str = map(str,(etype,evalue))
         except:
             # User exception is improperly defined.
             etype,evalue = str,sys.exc_info()[:2]
             etype_str,evalue_str = map(str,(etype,evalue))
         # ... and format it
         exception = ['%s%s%s: %s' % (Colors.excName, etype_str,
                                      ColorsNormal, evalue_str)]
         if type(evalue) is types.InstanceType:
             try:
                 names = [w for w in dir(evalue) if isinstance(w, basestring)]
             except:
                 # Every now and then, an object with funny inernals blows up
                 # when dir() is called on it.  We do the best we can to report
                 # the problem and continue
                 _m = '%sException reporting error (object with broken dir())%s:'
                 exception.append(_m % (Colors.excName,ColorsNormal))
                 etype_str,evalue_str = map(str,sys.exc_info()[:2])
                 exception.append('%s%s%s: %s' % (Colors.excName,etype_str,
                                      ColorsNormal, evalue_str))
                 names = []
             for name in names:
                 value = text_repr(getattr(evalue, name))
                 exception.append('\n%s%s = %s' % (indent, name, value))
 
         # This is being commented out for now as the __IPYTHON__ variable
         # referenced here is not resolved and causes massive test failures
-        # and errors. B. Granger, 04/2009.
+        # and errors. B. Granger, 04/2009. XXX
         # See https://bugs.launchpad.net/bugs/362137
         # # vds: >>
         # if records:
         #      filepath, lnum = records[-1][1:3]
         #      #print "file:", str(file), "linenb", str(lnum) # dbg
         #      filepath = os.path.abspath(filepath)
         #      __IPYTHON__.hooks.synchronize_with_editor(filepath, lnum, 0)
         # # vds: <<
                 
         # return all our info assembled as a single string
         return '%s\n\n%s\n%s' % (head,'\n'.join(frames),''.join(exception[0]) )
 
     def debugger(self,force=False):
         """Call up the pdb debugger if desired, always clean up the tb
         reference.
 
         Keywords:
 
           - force(False): by default, this routine checks the instance call_pdb
           flag and does not actually invoke the debugger if the flag is false.
           The 'force' option forces the debugger to activate even if the flag
           is false.
 
         If the call_pdb flag is set, the pdb interactive debugger is
         invoked. In all cases, the self.tb reference to the current traceback
         is deleted to prevent lingering references which hamper memory
         management.
 
         Note that each call to pdb() does an 'import readline', so if your app
         requires a special setup for the readline completers, you'll have to
         fix that by hand after invoking the exception handler."""
 
         if force or self.call_pdb:
             if self.pdb is None:
                 self.pdb = Debugger.Pdb(
                     self.color_scheme_table.active_scheme_name)
             # the system displayhook may have changed, restore the original
             # for pdb
             dhook = sys.displayhook
             sys.displayhook = sys.__displayhook__
             self.pdb.reset()
             # Find the right frame so we don't pop up inside ipython itself
             if hasattr(self,'tb'):
                 etb = self.tb
             else:
                 etb = self.tb = sys.last_traceback
             while self.tb.tb_next is not None:
                 self.tb = self.tb.tb_next
             try:
                 if etb and etb.tb_next:
                     etb = etb.tb_next
                 self.pdb.botframe = etb.tb_frame
                 self.pdb.interaction(self.tb.tb_frame, self.tb)
             finally:
                 sys.displayhook = dhook
             
         if hasattr(self,'tb'):
             del self.tb
 
     def handler(self, info=None):
         (etype, evalue, etb) = info or sys.exc_info()
         self.tb = etb
         Term.cout.flush()
         print >> Term.cerr, self.text(etype, evalue, etb)
         Term.cerr.flush()
 
     # Changed so an instance can just be called as VerboseTB_inst() and print
     # out the right info on its own.
     def __call__(self, etype=None, evalue=None, etb=None):
         """This hook can replace sys.excepthook (for Python 2.1 or higher)."""
         if etb is None:
             self.handler()
         else:
             self.handler((etype, evalue, etb))
         try:
             self.debugger()
         except KeyboardInterrupt:
             print "\nKeyboardInterrupt"
 
 #----------------------------------------------------------------------------
 class FormattedTB(VerboseTB,ListTB):
     """Subclass ListTB but allow calling with a traceback.
 
     It can thus be used as a sys.excepthook for Python > 2.1.
 
     Also adds 'Context' and 'Verbose' modes, not available in ListTB.
 
     Allows a tb_offset to be specified. This is useful for situations where
     one needs to remove a number of topmost frames from the traceback (such as
     occurs with python programs that themselves execute other python code,
     like Python shells).  """
     
     def __init__(self, mode = 'Plain', color_scheme='Linux',
                  tb_offset = 0,long_header=0,call_pdb=0,include_vars=0):
 
         # NEVER change the order of this list. Put new modes at the end:
         self.valid_modes = ['Plain','Context','Verbose']
         self.verbose_modes = self.valid_modes[1:3]
 
         VerboseTB.__init__(self,color_scheme,tb_offset,long_header,
                            call_pdb=call_pdb,include_vars=include_vars)
         self.set_mode(mode)
         
     def _extract_tb(self,tb):
         if tb:
             return traceback.extract_tb(tb)
         else:
             return None
 
     def text(self, etype, value, tb,context=5,mode=None):
         """Return formatted traceback.
 
         If the optional mode parameter is given, it overrides the current
         mode."""
 
         if mode is None:
             mode = self.mode
         if mode in self.verbose_modes:
             # verbose modes need a full traceback
             return VerboseTB.text(self,etype, value, tb,context=5)
         else:
             # We must check the source cache because otherwise we can print
             # out-of-date source code.
             linecache.checkcache()
             # Now we can extract and format the exception
             elist = self._extract_tb(tb)
             if len(elist) > self.tb_offset:
                 del elist[:self.tb_offset]
             return ListTB.text(self,etype,value,elist)
 
     def set_mode(self,mode=None):
         """Switch to the desired mode.
 
         If mode is not specified, cycles through the available modes."""
 
         if not mode:
             new_idx = ( self.valid_modes.index(self.mode) + 1 ) % \
                       len(self.valid_modes)
             self.mode = self.valid_modes[new_idx]
         elif mode not in self.valid_modes:
             raise ValueError, 'Unrecognized mode in FormattedTB: <'+mode+'>\n'\
                   'Valid modes: '+str(self.valid_modes)
         else:
             self.mode = mode
         # include variable details only in 'Verbose' mode
         self.include_vars = (self.mode == self.valid_modes[2])
 
     # some convenient shorcuts
     def plain(self):
         self.set_mode(self.valid_modes[0])
 
     def context(self):
         self.set_mode(self.valid_modes[1])
 
     def verbose(self):
         self.set_mode(self.valid_modes[2])
 
 #----------------------------------------------------------------------------
 class AutoFormattedTB(FormattedTB):
     """A traceback printer which can be called on the fly.
 
     It will find out about exceptions by itself.
 
     A brief example:
     
     AutoTB = AutoFormattedTB(mode = 'Verbose',color_scheme='Linux')
     try:
       ...
     except:
       AutoTB()  # or AutoTB(out=logfile) where logfile is an open file object
     """
     def __call__(self,etype=None,evalue=None,etb=None,
                  out=None,tb_offset=None):
         """Print out a formatted exception traceback.
 
         Optional arguments:
           - out: an open file-like object to direct output to.
 
           - tb_offset: the number of frames to skip over in the stack, on a
           per-call basis (this overrides temporarily the instance's tb_offset
           given at initialization time.  """
         
         if out is None:
             out = Term.cerr
         Term.cout.flush()
         if tb_offset is not None:
             tb_offset, self.tb_offset = self.tb_offset, tb_offset
             print >> out, self.text(etype, evalue, etb)
             self.tb_offset = tb_offset
         else:
             print >> out, self.text(etype, evalue, etb)
         out.flush()
         try:
             self.debugger()
         except KeyboardInterrupt:
             print "\nKeyboardInterrupt"
 
     def text(self,etype=None,value=None,tb=None,context=5,mode=None):
         if etype is None:
             etype,value,tb = sys.exc_info()
         self.tb = tb
         return FormattedTB.text(self,etype,value,tb,context=5,mode=mode)
 
 #---------------------------------------------------------------------------
 # A simple class to preserve Nathan's original functionality.
 class ColorTB(FormattedTB):
     """Shorthand to initialize a FormattedTB in Linux colors mode."""
     def __init__(self,color_scheme='Linux',call_pdb=0):
         FormattedTB.__init__(self,color_scheme=color_scheme,
                              call_pdb=call_pdb)
 
 #----------------------------------------------------------------------------
 # module testing (minimal)
 if __name__ == "__main__":
     def spam(c, (d, e)):
         x = c + d
         y = c * d
         foo(x, y)
 
     def foo(a, b, bar=1):
         eggs(a, b + bar)
 
     def eggs(f, g, z=globals()):
         h = f + g
         i = f - g
         return h / i
 
     print ''
     print '*** Before ***'
     try:
         print spam(1, (2, 3))
     except:
         traceback.print_exc()
     print ''
     
     handler = ColorTB()
     print '*** ColorTB ***'
     try:
         print spam(1, (2, 3))
     except:
         apply(handler, sys.exc_info() )
     print ''
     
     handler = VerboseTB()
     print '*** VerboseTB ***'
     try:
         print spam(1, (2, 3))
     except:
         apply(handler, sys.exc_info() )
     print ''
     

IPython/kernel/scripts/ipcluster.py

@@ -1,814 +1,825 b''
  #!/usr/bin/env python
 # encoding: utf-8
 
 """Start an IPython cluster = (controller + engines)."""
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import os
 import re
 import sys
 import signal
 import tempfile
 pjoin = os.path.join
 
 from twisted.internet import reactor, defer
 from twisted.internet.protocol import ProcessProtocol
 from twisted.internet.error import ProcessDone, ProcessTerminated
 from twisted.internet.utils import getProcessOutput
 from twisted.python import failure, log
 
 from IPython.external import argparse
 from IPython.external import Itpl
-from IPython.genutils import get_ipython_dir, get_log_dir, get_security_dir
-from IPython.genutils import num_cpus
+from IPython.genutils import (
+    get_ipython_dir, 
+    get_log_dir, 
+    get_security_dir, 
+    num_cpus
+)
 from IPython.kernel.fcutil import have_crypto
 
 # Create various ipython directories if they don't exist.
 # This must be done before IPython.kernel.config is imported.
 from IPython.iplib import user_setup
 if os.name == 'posix':
     rc_suffix = ''
 else:
     rc_suffix = '.ini'
 user_setup(get_ipython_dir(), rc_suffix, mode='install', interactive=False)
 get_log_dir()
 get_security_dir()
 
 from IPython.kernel.config import config_manager as kernel_config_manager
 from IPython.kernel.error import SecurityError, FileTimeoutError
 from IPython.kernel.fcutil import have_crypto
 from IPython.kernel.twistedutil import gatherBoth, wait_for_file
 from IPython.kernel.util import printer
 
 
 #-----------------------------------------------------------------------------
 # General process handling code
 #-----------------------------------------------------------------------------
 
 def find_exe(cmd):
     try:
         import win32api
     except ImportError:
         raise ImportError('you need to have pywin32 installed for this to work')
     else:
         try:
             (path, offest) = win32api.SearchPath(os.environ['PATH'],cmd + '.exe')
         except:
             (path, offset) = win32api.SearchPath(os.environ['PATH'],cmd + '.bat')
     return path
 
 class ProcessStateError(Exception):
     pass
 
 class UnknownStatus(Exception):
     pass
 
 class LauncherProcessProtocol(ProcessProtocol):
     """
     A ProcessProtocol to go with the ProcessLauncher.
     """
     def __init__(self, process_launcher):
         self.process_launcher = process_launcher
     
     def connectionMade(self):
         self.process_launcher.fire_start_deferred(self.transport.pid)
     
     def processEnded(self, status):
         value = status.value
         if isinstance(value, ProcessDone):
             self.process_launcher.fire_stop_deferred(0)
         elif isinstance(value, ProcessTerminated):
             self.process_launcher.fire_stop_deferred(
                 {'exit_code':value.exitCode,
                  'signal':value.signal,
                  'status':value.status
                 }
             )
         else:
             raise UnknownStatus("unknown exit status, this is probably a bug in Twisted")
     
     def outReceived(self, data):
         log.msg(data)
     
     def errReceived(self, data):
         log.err(data)
 
 class ProcessLauncher(object):
     """
     Start and stop an external process in an asynchronous manner.
     
     Currently this uses deferreds to notify other parties of process state
     changes.  This is an awkward design and should be moved to using
     a formal NotificationCenter.
     """
     def __init__(self, cmd_and_args):
         self.cmd = cmd_and_args[0]
         self.args = cmd_and_args
         self._reset()
     
     def _reset(self):
         self.process_protocol = None
         self.pid = None
         self.start_deferred = None
         self.stop_deferreds = []
         self.state = 'before' # before, running, or after
     
     @property
     def running(self):
         if self.state == 'running':
             return True
         else:
             return False
     
     def fire_start_deferred(self, pid):
         self.pid = pid
         self.state = 'running'
         log.msg('Process %r has started with pid=%i' % (self.args, pid))
         self.start_deferred.callback(pid)
     
     def start(self):
         if self.state == 'before':
             self.process_protocol = LauncherProcessProtocol(self)
             self.start_deferred = defer.Deferred()
             self.process_transport = reactor.spawnProcess(
                 self.process_protocol,
                 self.cmd,
                 self.args,
                 env=os.environ
             )
             return self.start_deferred
         else:
             s = 'the process has already been started and has state: %r' % \
                 self.state
             return defer.fail(ProcessStateError(s))
     
     def get_stop_deferred(self):
         if self.state == 'running' or self.state == 'before':
             d = defer.Deferred()
             self.stop_deferreds.append(d)
             return d
         else:
             s = 'this process is already complete'
             return defer.fail(ProcessStateError(s))
     
     def fire_stop_deferred(self, exit_code):
         log.msg('Process %r has stopped with %r' % (self.args, exit_code))
         self.state = 'after'
         for d in self.stop_deferreds:
             d.callback(exit_code)
     
     def signal(self, sig):
         """
         Send a signal to the process.
         
         The argument sig can be ('KILL','INT', etc.) or any signal number.
         """
         if self.state == 'running':
             self.process_transport.signalProcess(sig)
     
     # def __del__(self):
     #     self.signal('KILL')
     
     def interrupt_then_kill(self, delay=1.0):
         self.signal('INT')
         reactor.callLater(delay, self.signal, 'KILL')
 
 
 #-----------------------------------------------------------------------------
 # Code for launching controller and engines
 #-----------------------------------------------------------------------------
 
 
 class ControllerLauncher(ProcessLauncher):
     
     def __init__(self, extra_args=None):
         if sys.platform == 'win32':
             # This logic is needed because the ipcontroller script doesn't
             # always get installed in the same way or in the same location.
             from IPython.kernel.scripts import ipcontroller
             script_location = ipcontroller.__file__.replace('.pyc', '.py')
             # The -u option here turns on unbuffered output, which is required
             # on Win32 to prevent wierd conflict and problems with Twisted.
             # Also, use sys.executable to make sure we are picking up the 
             # right python exe.
             args = [sys.executable, '-u', script_location]
         else:
             args = ['ipcontroller']
         self.extra_args = extra_args
         if extra_args is not None:
             args.extend(extra_args)
         
         ProcessLauncher.__init__(self, args)
 
 
 class EngineLauncher(ProcessLauncher):
     
     def __init__(self, extra_args=None):
         if sys.platform == 'win32':
             # This logic is needed because the ipcontroller script doesn't
             # always get installed in the same way or in the same location.
             from IPython.kernel.scripts import ipengine
             script_location = ipengine.__file__.replace('.pyc', '.py')
             # The -u option here turns on unbuffered output, which is required
             # on Win32 to prevent wierd conflict and problems with Twisted.
             # Also, use sys.executable to make sure we are picking up the 
             # right python exe.
             args = [sys.executable, '-u', script_location]
         else:
             args = ['ipengine']
         self.extra_args = extra_args
         if extra_args is not None:
             args.extend(extra_args)
         
         ProcessLauncher.__init__(self, args)
 
 
 class LocalEngineSet(object):
     
     def __init__(self, extra_args=None):
         self.extra_args = extra_args
         self.launchers = []
     
     def start(self, n):
         dlist = []
         for i in range(n):
             el = EngineLauncher(extra_args=self.extra_args)
             d = el.start()
             self.launchers.append(el)
             dlist.append(d)
         dfinal = gatherBoth(dlist, consumeErrors=True)
         dfinal.addCallback(self._handle_start)
         return dfinal
     
     def _handle_start(self, r):
         log.msg('Engines started with pids: %r' % r)
         return r
     
     def _handle_stop(self, r):
         log.msg('Engines received signal: %r' % r)
         return r
     
     def signal(self, sig):
         dlist = []
         for el in self.launchers:
             d = el.get_stop_deferred()
             dlist.append(d)
             el.signal(sig)
         dfinal = gatherBoth(dlist, consumeErrors=True)
         dfinal.addCallback(self._handle_stop)
         return dfinal
     
     def interrupt_then_kill(self, delay=1.0):
         dlist = []
         for el in self.launchers:
             d = el.get_stop_deferred()
             dlist.append(d)
             el.interrupt_then_kill(delay)
         dfinal = gatherBoth(dlist, consumeErrors=True)
         dfinal.addCallback(self._handle_stop)
         return dfinal
 
 
 class BatchEngineSet(object):
     
     # Subclasses must fill these in.  See PBSEngineSet
     submit_command = ''
     delete_command = ''
     job_id_regexp = ''
     
     def __init__(self, template_file, **kwargs):
         self.template_file = template_file
         self.context = {}
         self.context.update(kwargs)
         self.batch_file = self.template_file+'-run'
     
     def parse_job_id(self, output):
         m = re.match(self.job_id_regexp, output)
         if m is not None:
             job_id = m.group()
         else:
             raise Exception("job id couldn't be determined: %s" % output)
         self.job_id = job_id
         log.msg('Job started with job id: %r' % job_id)
         return job_id
     
     def write_batch_script(self, n):
         self.context['n'] = n
         template = open(self.template_file, 'r').read()
         log.msg('Using template for batch script: %s' % self.template_file)
         script_as_string = Itpl.itplns(template, self.context)
         log.msg('Writing instantiated batch script: %s' % self.batch_file)
         f = open(self.batch_file,'w')
         f.write(script_as_string)
         f.close()
     
     def handle_error(self, f):
         f.printTraceback()
         f.raiseException()
     
     def start(self, n):
         self.write_batch_script(n)
         d = getProcessOutput(self.submit_command,
             [self.batch_file],env=os.environ)
         d.addCallback(self.parse_job_id)
         d.addErrback(self.handle_error)
         return d
     
     def kill(self):
         d = getProcessOutput(self.delete_command,
             [self.job_id],env=os.environ)
         return d
 
 class PBSEngineSet(BatchEngineSet):
     
     submit_command = 'qsub'
     delete_command = 'qdel'
     job_id_regexp = '\d+'
     
     def __init__(self, template_file, **kwargs):
         BatchEngineSet.__init__(self, template_file, **kwargs)
 
 
 sshx_template="""#!/bin/sh
 "$@" &> /dev/null &
 echo $!
 """
 
 engine_killer_template="""#!/bin/sh
 ps -fu `whoami` | grep '[i]pengine' | awk '{print $2}' | xargs kill -TERM
 """
 
 class SSHEngineSet(object):
     sshx_template=sshx_template
     engine_killer_template=engine_killer_template
     
     def __init__(self, engine_hosts, sshx=None, ipengine="ipengine"):
         """Start a controller on localhost and engines using ssh.
         
         The engine_hosts argument is a dict with hostnames as keys and
         the number of engine (int) as values.  sshx is the name of a local
         file that will be used to run remote commands.  This file is used
         to setup the environment properly.
         """
         
         self.temp_dir = tempfile.gettempdir()
         if sshx is not None:
             self.sshx = sshx
         else:
             # Write the sshx.sh file locally from our template.
             self.sshx = os.path.join(
                 self.temp_dir,
                 '%s-main-sshx.sh' % os.environ['USER']
             )
             f = open(self.sshx, 'w')
             f.writelines(self.sshx_template)
             f.close()
         self.engine_command = ipengine
         self.engine_hosts = engine_hosts
         # Write the engine killer script file locally from our template.
         self.engine_killer = os.path.join(
             self.temp_dir,  
             '%s-local-engine_killer.sh' % os.environ['USER']
         )
         f = open(self.engine_killer, 'w')
         f.writelines(self.engine_killer_template)
         f.close()
     
     def start(self, send_furl=False):
         dlist = []
         for host in self.engine_hosts.keys():
             count = self.engine_hosts[host]
             d = self._start(host, count, send_furl)
             dlist.append(d)
         return gatherBoth(dlist, consumeErrors=True)
     
     def _start(self, hostname, count=1, send_furl=False):
         if send_furl:
             d = self._scp_furl(hostname)
         else:
             d = defer.succeed(None)
         d.addCallback(lambda r: self._scp_sshx(hostname))
         d.addCallback(lambda r: self._ssh_engine(hostname, count))
         return d
         
     def _scp_furl(self, hostname):
         scp_cmd = "scp ~/.ipython/security/ipcontroller-engine.furl %s:.ipython/security/" % (hostname)
         cmd_list = scp_cmd.split()
         cmd_list[1] = os.path.expanduser(cmd_list[1])
         log.msg('Copying furl file: %s' % scp_cmd)
         d = getProcessOutput(cmd_list[0], cmd_list[1:], env=os.environ) 
         return d
     
     def _scp_sshx(self, hostname):
         scp_cmd = "scp %s %s:%s/%s-sshx.sh" % (
             self.sshx, hostname, 
             self.temp_dir, os.environ['USER']
         )
         print
         log.msg("Copying sshx: %s" % scp_cmd)
         sshx_scp = scp_cmd.split()
         d = getProcessOutput(sshx_scp[0], sshx_scp[1:], env=os.environ)
         return d
     
     def _ssh_engine(self, hostname, count):
         exec_engine = "ssh %s sh %s/%s-sshx.sh %s" % (
             hostname, self.temp_dir, 
             os.environ['USER'], self.engine_command
         )
         cmds = exec_engine.split()
         dlist = []
         log.msg("about to start engines...")
         for i in range(count):
             log.msg('Starting engines: %s' % exec_engine)
             d = getProcessOutput(cmds[0], cmds[1:], env=os.environ)
             dlist.append(d)
         return gatherBoth(dlist, consumeErrors=True)
     
     def kill(self):
         dlist = []
         for host in self.engine_hosts.keys():
             d = self._killall(host)
             dlist.append(d)
         return gatherBoth(dlist, consumeErrors=True)
     
     def _killall(self, hostname):
         d = self._scp_engine_killer(hostname)
         d.addCallback(lambda r: self._ssh_kill(hostname))
         # d.addErrback(self._exec_err)
         return d
 
     def _scp_engine_killer(self, hostname):
         scp_cmd = "scp %s %s:%s/%s-engine_killer.sh" % (
             self.engine_killer, 
             hostname, 
             self.temp_dir, 
             os.environ['USER']
         )
         cmds = scp_cmd.split()
         log.msg('Copying engine_killer: %s' % scp_cmd)
         d = getProcessOutput(cmds[0], cmds[1:], env=os.environ)
         return d
     
     def _ssh_kill(self, hostname):
         kill_cmd = "ssh %s sh %s/%s-engine_killer.sh" % (
             hostname,
             self.temp_dir, 
             os.environ['USER']
         )
         log.msg('Killing engine: %s' % kill_cmd)
         kill_cmd = kill_cmd.split()
         d = getProcessOutput(kill_cmd[0], kill_cmd[1:], env=os.environ)
         return d
 
     def _exec_err(self, r):
         log.msg(r)
 
 #-----------------------------------------------------------------------------
 # Main functions for the different types of clusters
 #-----------------------------------------------------------------------------
 
 # TODO:
 # The logic in these codes should be moved into classes like LocalCluster
 # MpirunCluster, PBSCluster, etc.  This would remove alot of the duplications.
 # The main functions should then just parse the command line arguments, create
 # the appropriate class and call a 'start' method.
 
 
 def check_security(args, cont_args):
+    """Check to see if we should run with SSL support."""
     if (not args.x or not args.y) and not have_crypto:
         log.err("""
 OpenSSL/pyOpenSSL is not available, so we can't run in secure mode.
 Try running ipcluster with the -xy flags:  ipcluster local -xy -n 4""")
         reactor.stop()
         return False
     if args.x:
         cont_args.append('-x')
     if args.y:
         cont_args.append('-y')
     return True
 
 
 def check_reuse(args, cont_args):
+    """Check to see if we should try to resuse FURL files."""
     if args.r:
         cont_args.append('-r')
         if args.client_port == 0 or args.engine_port == 0:
             log.err("""
 To reuse FURL files, you must also set the client and engine ports using
 the --client-port and --engine-port options.""")
             reactor.stop()
             return False
         cont_args.append('--client-port=%i' % args.client_port)
         cont_args.append('--engine-port=%i' % args.engine_port)
     return True
 
 
 def _err_and_stop(f):
+    """Errback to log a failure and halt the reactor on a fatal error."""
     log.err(f)
     reactor.stop()
 
 
 def _delay_start(cont_pid, start_engines, furl_file, reuse):
+    """Wait for controller to create FURL files and the start the engines."""
     if not reuse:
         if os.path.isfile(furl_file):
             os.unlink(furl_file)
     log.msg('Waiting for controller to finish starting...')
     d = wait_for_file(furl_file, delay=0.2, max_tries=50)
     d.addCallback(lambda _: log.msg('Controller started'))
     d.addCallback(lambda _: start_engines(cont_pid))
     return d
 
 
 def main_local(args):
     cont_args = []
     cont_args.append('--logfile=%s' % pjoin(args.logdir,'ipcontroller'))
     
     # Check security settings before proceeding
     if not check_security(args, cont_args):
         return
     
     # See if we are reusing FURL files
     if not check_reuse(args, cont_args):
         return
     
     cl = ControllerLauncher(extra_args=cont_args)
     dstart = cl.start()
     def start_engines(cont_pid):
         engine_args = []
         engine_args.append('--logfile=%s' % \
             pjoin(args.logdir,'ipengine%s-' % cont_pid))
         eset = LocalEngineSet(extra_args=engine_args)
         def shutdown(signum, frame):
             log.msg('Stopping local cluster')
             # We are still playing with the times here, but these seem
             # to be reliable in allowing everything to exit cleanly.
             eset.interrupt_then_kill(0.5)
             cl.interrupt_then_kill(0.5)
             reactor.callLater(1.0, reactor.stop)
         signal.signal(signal.SIGINT,shutdown)
         d = eset.start(args.n)
         return d
     config = kernel_config_manager.get_config_obj()
     furl_file = config['controller']['engine_furl_file']
     dstart.addCallback(_delay_start, start_engines, furl_file, args.r)
     dstart.addErrback(_err_and_stop)
 
 
 def main_mpi(args):
     cont_args = []
     cont_args.append('--logfile=%s' % pjoin(args.logdir,'ipcontroller'))
     
     # Check security settings before proceeding
     if not check_security(args, cont_args):
         return
     
     # See if we are reusing FURL files
     if not check_reuse(args, cont_args):
         return
     
     cl = ControllerLauncher(extra_args=cont_args)
     dstart = cl.start()
     def start_engines(cont_pid):
         raw_args = [args.cmd]
         raw_args.extend(['-n',str(args.n)])
         raw_args.append('ipengine')
         raw_args.append('-l')
         raw_args.append(pjoin(args.logdir,'ipengine%s-' % cont_pid))
         if args.mpi:
             raw_args.append('--mpi=%s' % args.mpi)
         eset = ProcessLauncher(raw_args)
         def shutdown(signum, frame):
             log.msg('Stopping local cluster')
             # We are still playing with the times here, but these seem
             # to be reliable in allowing everything to exit cleanly.
             eset.interrupt_then_kill(1.0)
             cl.interrupt_then_kill(1.0)
             reactor.callLater(2.0, reactor.stop)
         signal.signal(signal.SIGINT,shutdown)
         d = eset.start()
         return d
     config = kernel_config_manager.get_config_obj()
     furl_file = config['controller']['engine_furl_file']
     dstart.addCallback(_delay_start, start_engines, furl_file, args.r)
     dstart.addErrback(_err_and_stop)
 
 
 def main_pbs(args):
     cont_args = []
     cont_args.append('--logfile=%s' % pjoin(args.logdir,'ipcontroller'))
     
     # Check security settings before proceeding
     if not check_security(args, cont_args):
         return
     
     # See if we are reusing FURL files
     if not check_reuse(args, cont_args):
         return
     
     cl = ControllerLauncher(extra_args=cont_args)
     dstart = cl.start()
     def start_engines(r):
         pbs_set =  PBSEngineSet(args.pbsscript)
         def shutdown(signum, frame):
             log.msg('Stopping pbs cluster')
             d = pbs_set.kill()
             d.addBoth(lambda _: cl.interrupt_then_kill(1.0))
             d.addBoth(lambda _: reactor.callLater(2.0, reactor.stop))
         signal.signal(signal.SIGINT,shutdown)
         d = pbs_set.start(args.n)
         return d
     config = kernel_config_manager.get_config_obj()
     furl_file = config['controller']['engine_furl_file']
     dstart.addCallback(_delay_start, start_engines, furl_file, args.r)
     dstart.addErrback(_err_and_stop)
 
 
 def main_ssh(args):
     """Start a controller on localhost and engines using ssh.
     
     Your clusterfile should look like::
     
         send_furl = False # True, if you want 
         engines = {
             'engine_host1' : engine_count, 
             'engine_host2' : engine_count2
         } 
     """
     clusterfile = {}
     execfile(args.clusterfile, clusterfile)
     if not clusterfile.has_key('send_furl'):
         clusterfile['send_furl'] = False
         
     cont_args = []
     cont_args.append('--logfile=%s' % pjoin(args.logdir,'ipcontroller'))
     
     # Check security settings before proceeding
     if not check_security(args, cont_args):
         return
     
     # See if we are reusing FURL files
     if not check_reuse(args, cont_args):
         return
     
     cl = ControllerLauncher(extra_args=cont_args)
     dstart = cl.start()
     def start_engines(cont_pid):
         ssh_set = SSHEngineSet(clusterfile['engines'], sshx=args.sshx)
         def shutdown(signum, frame):
             d = ssh_set.kill()
             cl.interrupt_then_kill(1.0)
             reactor.callLater(2.0, reactor.stop)
         signal.signal(signal.SIGINT,shutdown)
         d = ssh_set.start(clusterfile['send_furl'])
         return d
     config = kernel_config_manager.get_config_obj()
     furl_file = config['controller']['engine_furl_file']
     dstart.addCallback(_delay_start, start_engines, furl_file, args.r)
     dstart.addErrback(_err_and_stop)
 
 
 def get_args():
     base_parser = argparse.ArgumentParser(add_help=False)
     base_parser.add_argument(
         '-r',
         action='store_true',
         dest='r',
         help='try to reuse FURL files.  Use with --client-port and --engine-port'
     )
     base_parser.add_argument(
         '--client-port',
         type=int,
         dest='client_port',
         help='the port the controller will listen on for client connections',
         default=0
     )
     base_parser.add_argument(
         '--engine-port',
         type=int,
         dest='engine_port',
         help='the port the controller will listen on for engine connections',
         default=0
     )
     base_parser.add_argument(
         '-x',
         action='store_true',
         dest='x',
         help='turn off client security'
     )
     base_parser.add_argument(
         '-y',
         action='store_true',
         dest='y',
         help='turn off engine security'
     )
     base_parser.add_argument(
         "--logdir", 
         type=str,
         dest="logdir",
         help="directory to put log files (default=$IPYTHONDIR/log)",
         default=pjoin(get_ipython_dir(),'log')
     )
     base_parser.add_argument(
         "-n",
         "--num", 
         type=int,
         dest="n",
         default=2,
         help="the number of engines to start"
     )
     
     parser = argparse.ArgumentParser(
         description='IPython cluster startup.  This starts a controller and\
-        engines using various approaches.  THIS IS A TECHNOLOGY PREVIEW AND\
-        THE API WILL CHANGE SIGNIFICANTLY BEFORE THE FINAL RELEASE.'
+        engines using various approaches.  Use the IPYTHONDIR environment\
+        variable to change your IPython directory from the default of\
+        .ipython or _ipython.  The log and security subdirectories of your\
+        IPython directory will be used by this script for log files and\
+        security files.'
     )
     subparsers = parser.add_subparsers(
         help='available cluster types.  For help, do "ipcluster TYPE --help"')
     
     parser_local = subparsers.add_parser(
         'local',
         help='run a local cluster',
         parents=[base_parser]
     )
     parser_local.set_defaults(func=main_local)
     
     parser_mpirun = subparsers.add_parser(
         'mpirun',
         help='run a cluster using mpirun (mpiexec also works)',
         parents=[base_parser]
     )
     parser_mpirun.add_argument(
         "--mpi",
         type=str,
         dest="mpi", # Don't put a default here to allow no MPI support
         help="how to call MPI_Init (default=mpi4py)"
     )
     parser_mpirun.set_defaults(func=main_mpi, cmd='mpirun')
 
     parser_mpiexec = subparsers.add_parser(
         'mpiexec',
         help='run a cluster using mpiexec (mpirun also works)',
         parents=[base_parser]
     )
     parser_mpiexec.add_argument(
         "--mpi",
         type=str,
         dest="mpi", # Don't put a default here to allow no MPI support
         help="how to call MPI_Init (default=mpi4py)"
     )
     parser_mpiexec.set_defaults(func=main_mpi, cmd='mpiexec')
     
     parser_pbs = subparsers.add_parser(
         'pbs', 
         help='run a pbs cluster',
         parents=[base_parser]
     )
     parser_pbs.add_argument(
         '--pbs-script',
         type=str, 
         dest='pbsscript',
         help='PBS script template',
         default='pbs.template'
     )
     parser_pbs.set_defaults(func=main_pbs)
     
     parser_ssh = subparsers.add_parser(
         'ssh',
         help='run a cluster using ssh, should have ssh-keys setup',
         parents=[base_parser]
     )
     parser_ssh.add_argument(
         '--clusterfile', 
         type=str,
         dest='clusterfile',
         help='python file describing the cluster',
         default='clusterfile.py',
     )
     parser_ssh.add_argument(
         '--sshx', 
         type=str,
         dest='sshx',
         help='sshx launcher helper'
     )
     parser_ssh.set_defaults(func=main_ssh)
     
     args = parser.parse_args()
     return args
 
 def main():
     args = get_args()
     reactor.callWhenRunning(args.func, args)
     log.startLogging(sys.stdout)
     reactor.run()
 
 if __name__ == '__main__':
     main()

IPython/kernel/scripts/ipcontroller.py

@@ -1,408 +1,416 b''
 #!/usr/bin/env python
 # encoding: utf-8
 
 """The IPython controller."""
 
 __docformat__ = "restructuredtext en"
 
 #-------------------------------------------------------------------------------
 #  Copyright (C) 2008  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-------------------------------------------------------------------------------
 
 #-------------------------------------------------------------------------------
 # Imports
 #-------------------------------------------------------------------------------
 
 # Python looks for an empty string at the beginning of sys.path to enable
 # importing from the cwd.
 import sys
 sys.path.insert(0, '')
 
-import sys, time, os
-import tempfile
 from optparse import OptionParser
+import os
+import time
+import tempfile
 
 from twisted.application import internet, service
 from twisted.internet import reactor, error, defer
 from twisted.python import log
 
 from IPython.kernel.fcutil import Tub, UnauthenticatedTub, have_crypto
 
 # from IPython.tools import growl
 # growl.start("IPython1 Controller")
 
 from IPython.kernel.error import SecurityError
 from IPython.kernel import controllerservice
 from IPython.kernel.fcutil import check_furl_file_security
 
 # Create various ipython directories if they don't exist.
 # This must be done before IPython.kernel.config is imported.
 from IPython.iplib import user_setup
 from IPython.genutils import get_ipython_dir, get_log_dir, get_security_dir
 if os.name == 'posix':
     rc_suffix = ''
 else:
     rc_suffix = '.ini'
 user_setup(get_ipython_dir(), rc_suffix, mode='install', interactive=False)
 get_log_dir()
 get_security_dir()
 
 from IPython.kernel.config import config_manager as kernel_config_manager
 from IPython.config.cutils import import_item
 
 
 #-------------------------------------------------------------------------------
 # Code
 #-------------------------------------------------------------------------------
 
 def get_temp_furlfile(filename):
     return tempfile.mktemp(dir=os.path.dirname(filename),
                            prefix=os.path.basename(filename))
 
 def make_tub(ip, port, secure, cert_file):
     """
     Create a listening tub given an ip, port, and cert_file location.
     
     :Parameters:
         ip : str
             The ip address that the tub should listen on.  Empty means all
         port : int
             The port that the tub should listen on.  A value of 0 means
             pick a random port
         secure: boolean
             Will the connection be secure (in the foolscap sense)
         cert_file:
             A filename of a file to be used for theSSL certificate
     """
     if secure:
         if have_crypto:
             tub = Tub(certFile=cert_file)
         else:
             raise SecurityError("""
 OpenSSL/pyOpenSSL is not available, so we can't run in secure mode.
 Try running without security using 'ipcontroller -xy'.
 """)
     else:
         tub = UnauthenticatedTub()
     
     # Set the strport based on the ip and port and start listening
     if ip == '':
         strport = "tcp:%i" % port
     else:
         strport = "tcp:%i:interface=%s" % (port, ip)
     listener = tub.listenOn(strport)
     
     return tub, listener
 
 def make_client_service(controller_service, config):
     """
     Create a service that will listen for clients.
     
     This service is simply a `foolscap.Tub` instance that has a set of Referenceables
     registered with it.
     """
 
     # Now create the foolscap tub
     ip = config['controller']['client_tub']['ip']
     port = config['controller']['client_tub'].as_int('port')
     location = config['controller']['client_tub']['location']
     secure = config['controller']['client_tub']['secure']
     cert_file = config['controller']['client_tub']['cert_file']
     client_tub, client_listener = make_tub(ip, port, secure, cert_file)
     
     # Set the location in the trivial case of localhost
     if ip == 'localhost' or ip == '127.0.0.1':
         location = "127.0.0.1"
     
     if not secure:
         log.msg("WARNING: you are running the controller with no client security")
     
     def set_location_and_register():
         """Set the location for the tub and return a deferred."""
 
         def register(empty, ref, furl_file):
             # We create and then move to make sure that when the file
             # appears to other processes, the buffer has the flushed
             # and the file has been closed
             temp_furl_file = get_temp_furlfile(furl_file)
             client_tub.registerReference(ref, furlFile=temp_furl_file)
             os.rename(temp_furl_file, furl_file)
         
         if location == '':
             d = client_tub.setLocationAutomatically()
         else:
             d = defer.maybeDeferred(client_tub.setLocation, "%s:%i" % (location, client_listener.getPortnum()))
         
         for ciname, ci in config['controller']['controller_interfaces'].iteritems():
             log.msg("Adapting Controller to interface: %s" % ciname)
             furl_file = ci['furl_file']
             log.msg("Saving furl for interface [%s] to file: %s" % (ciname, furl_file))
             check_furl_file_security(furl_file, secure)
             adapted_controller = import_item(ci['controller_interface'])(controller_service)
             d.addCallback(register, import_item(ci['fc_interface'])(adapted_controller), 
                 furl_file=ci['furl_file'])
     
     reactor.callWhenRunning(set_location_and_register)
     return client_tub
 
 
 def make_engine_service(controller_service, config):
     """
     Create a service that will listen for engines.
     
     This service is simply a `foolscap.Tub` instance that has a set of Referenceables
     registered with it.
     """
 
     # Now create the foolscap tub
     ip = config['controller']['engine_tub']['ip']
     port = config['controller']['engine_tub'].as_int('port')
     location = config['controller']['engine_tub']['location']
     secure = config['controller']['engine_tub']['secure']
     cert_file = config['controller']['engine_tub']['cert_file']
     engine_tub, engine_listener = make_tub(ip, port, secure, cert_file)
     
     # Set the location in the trivial case of localhost
     if ip == 'localhost' or ip == '127.0.0.1':
         location = "127.0.0.1"
     
     if not secure:
         log.msg("WARNING: you are running the controller with no engine security")
     
     def set_location_and_register():
         """Set the location for the tub and return a deferred."""
 
         def register(empty, ref, furl_file):
             # We create and then move to make sure that when the file
             # appears to other processes, the buffer has the flushed
             # and the file has been closed
             temp_furl_file = get_temp_furlfile(furl_file)
             engine_tub.registerReference(ref, furlFile=temp_furl_file)
             os.rename(temp_furl_file, furl_file)
         
         if location == '':
             d = engine_tub.setLocationAutomatically()
         else:
             d = defer.maybeDeferred(engine_tub.setLocation, "%s:%i" % (location, engine_listener.getPortnum()))
     
         furl_file = config['controller']['engine_furl_file']
         engine_fc_interface = import_item(config['controller']['engine_fc_interface'])
         log.msg("Saving furl for the engine to file: %s" % furl_file)
         check_furl_file_security(furl_file, secure)
         fc_controller = engine_fc_interface(controller_service)
         d.addCallback(register, fc_controller, furl_file=furl_file)
     
     reactor.callWhenRunning(set_location_and_register)
     return engine_tub
     
 def start_controller():
     """
     Start the controller by creating the service hierarchy and starting the reactor.
     
     This method does the following:
     
         * It starts the controller logging
         * In execute an import statement for the controller
         * It creates 2 `foolscap.Tub` instances for the client and the engines
           and registers `foolscap.Referenceables` with the tubs to expose the
           controller to engines and clients.
     """
     config = kernel_config_manager.get_config_obj()
     
     # Start logging
     logfile = config['controller']['logfile']
     if logfile:
         logfile = logfile + str(os.getpid()) + '.log'
         try:
             openLogFile = open(logfile, 'w')
         except:
             openLogFile = sys.stdout
     else:
         openLogFile = sys.stdout
     log.startLogging(openLogFile)
     
     # Execute any user defined import statements
     cis = config['controller']['import_statement']
     if cis:
         try:
             exec cis in globals(), locals()
         except:
             log.msg("Error running import_statement: %s" % cis)
     
     # Delete old furl files unless the reuse_furls is set
     reuse = config['controller']['reuse_furls']
     if not reuse:
         paths = (config['controller']['engine_furl_file'],
             config['controller']['controller_interfaces']['task']['furl_file'],
             config['controller']['controller_interfaces']['multiengine']['furl_file']
         )
         for p in paths:
             if os.path.isfile(p):
                 os.remove(p)
     
     # Create the service hierarchy
     main_service = service.MultiService()
     # The controller service
     controller_service = controllerservice.ControllerService()
     controller_service.setServiceParent(main_service)
     # The client tub and all its refereceables
     client_service = make_client_service(controller_service, config)
     client_service.setServiceParent(main_service)
     # The engine tub
     engine_service = make_engine_service(controller_service, config)
     engine_service.setServiceParent(main_service)
     # Start the controller service and set things running
     main_service.startService()
     reactor.run()
 
 def init_config():
     """
     Initialize the configuration using default and command line options.
     """
     
-    parser = OptionParser()
+    parser = OptionParser("""ipcontroller [options]
+
+Start an IPython controller.
+
+Use the IPYTHONDIR environment variable to change your IPython directory 
+from the default of .ipython or _ipython.  The log and security 
+subdirectories of your IPython directory will be used by this script 
+for log files and security files.""")
     
     # Client related options
     parser.add_option(
         "--client-ip", 
         type="string",
         dest="client_ip",
         help="the IP address or hostname the controller will listen on for client connections"
     )
     parser.add_option(
         "--client-port", 
         type="int", 
         dest="client_port",
         help="the port the controller will listen on for client connections"
     )    
     parser.add_option(
         '--client-location',
         type="string",
         dest="client_location",
         help="hostname or ip for clients to connect to"
     )
     parser.add_option(
         "-x",
         action="store_false",
         dest="client_secure",
         help="turn off all client security"
     )
     parser.add_option(
         '--client-cert-file',
         type="string",
         dest="client_cert_file",
         help="file to store the client SSL certificate"
     )
     parser.add_option(
         '--task-furl-file',
         type="string",
         dest="task_furl_file",
         help="file to store the FURL for task clients to connect with"
     )
     parser.add_option(
         '--multiengine-furl-file',
         type="string",
         dest="multiengine_furl_file",
         help="file to store the FURL for multiengine clients to connect with"
     )
     # Engine related options
     parser.add_option(
         "--engine-ip", 
         type="string",
         dest="engine_ip",
         help="the IP address or hostname the controller will listen on for engine connections"
     )
     parser.add_option(
         "--engine-port", 
         type="int", 
         dest="engine_port",
         help="the port the controller will listen on for engine connections"
     )    
     parser.add_option(
         '--engine-location',
         type="string",
         dest="engine_location",
         help="hostname or ip for engines to connect to"
     )
     parser.add_option(
         "-y",
         action="store_false",
         dest="engine_secure",
         help="turn off all engine security"
     )
     parser.add_option(
         '--engine-cert-file',
         type="string",
         dest="engine_cert_file",
         help="file to store the engine SSL certificate"
     )
     parser.add_option(
         '--engine-furl-file',
         type="string",
         dest="engine_furl_file",
         help="file to store the FURL for engines to connect with"
     )
     parser.add_option(
         "-l", "--logfile",
         type="string",
         dest="logfile",
         help="log file name (default is stdout)"
     )
     parser.add_option(
         "-r",
         action="store_true",
         dest="reuse_furls",
         help="try to reuse all furl files"
     )
     
     (options, args) = parser.parse_args()
     
     config = kernel_config_manager.get_config_obj()
     
     # Update with command line options
     if options.client_ip is not None:
         config['controller']['client_tub']['ip'] = options.client_ip
     if options.client_port is not None:
         config['controller']['client_tub']['port'] = options.client_port
     if options.client_location is not None:
         config['controller']['client_tub']['location'] = options.client_location
     if options.client_secure is not None:
         config['controller']['client_tub']['secure'] = options.client_secure
     if options.client_cert_file is not None:
         config['controller']['client_tub']['cert_file'] = options.client_cert_file
     if options.task_furl_file is not None:
         config['controller']['controller_interfaces']['task']['furl_file'] = options.task_furl_file
     if options.multiengine_furl_file is not None:
         config['controller']['controller_interfaces']['multiengine']['furl_file'] = options.multiengine_furl_file
     if options.engine_ip is not None:
         config['controller']['engine_tub']['ip'] = options.engine_ip
     if options.engine_port is not None:
         config['controller']['engine_tub']['port'] = options.engine_port
     if options.engine_location is not None:
         config['controller']['engine_tub']['location'] = options.engine_location
     if options.engine_secure is not None:
         config['controller']['engine_tub']['secure'] = options.engine_secure
     if options.engine_cert_file is not None:
         config['controller']['engine_tub']['cert_file'] = options.engine_cert_file
     if options.engine_furl_file is not None:
         config['controller']['engine_furl_file'] = options.engine_furl_file
     if options.reuse_furls is not None:
             config['controller']['reuse_furls'] = options.reuse_furls
 
     if options.logfile is not None:
         config['controller']['logfile'] = options.logfile
     
     kernel_config_manager.update_config_obj(config)
 
 def main():
     """
     After creating the configuration information, start the controller.
     """
     init_config()
     start_controller()
     
 if __name__ == "__main__":
     main()

IPython/kernel/scripts/ipengine.py

@@ -1,186 +1,193 b''
 #!/usr/bin/env python
 # encoding: utf-8
 
 """Start the IPython Engine."""
 
 __docformat__ = "restructuredtext en"
 
 #-------------------------------------------------------------------------------
 #  Copyright (C) 2008  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-------------------------------------------------------------------------------
 
 #-------------------------------------------------------------------------------
 # Imports
 #-------------------------------------------------------------------------------
 
 # Python looks for an empty string at the beginning of sys.path to enable
 # importing from the cwd.
 import sys
 sys.path.insert(0, '')
 
-import sys, os
 from optparse import OptionParser
+import os
 
 from twisted.application import service
 from twisted.internet import reactor
 from twisted.python import log
 
 from IPython.kernel.fcutil import Tub, UnauthenticatedTub
 
 from IPython.kernel.core.config import config_manager as core_config_manager
 from IPython.config.cutils import import_item
 from IPython.kernel.engineservice import EngineService
 
 # Create various ipython directories if they don't exist.
 # This must be done before IPython.kernel.config is imported.
 from IPython.iplib import user_setup
 from IPython.genutils import get_ipython_dir, get_log_dir, get_security_dir
 if os.name == 'posix':
     rc_suffix = ''
 else:
     rc_suffix = '.ini'
 user_setup(get_ipython_dir(), rc_suffix, mode='install', interactive=False)
 get_log_dir()
 get_security_dir()
 
 from IPython.kernel.config import config_manager as kernel_config_manager
 from IPython.kernel.engineconnector import EngineConnector
 
 
 #-------------------------------------------------------------------------------
 # Code
 #-------------------------------------------------------------------------------
 
 def start_engine():
     """
     Start the engine, by creating it and starting the Twisted reactor.
     
     This method does:
     
         * If it exists, runs the `mpi_import_statement` to call `MPI_Init`
         * Starts the engine logging
         * Creates an IPython shell and wraps it in an `EngineService`
         * Creates a `foolscap.Tub` to use in connecting to a controller.
         * Uses the tub and the `EngineService` along with a Foolscap URL
           (or FURL) to connect to the controller and register the engine
           with the controller
     """
     kernel_config = kernel_config_manager.get_config_obj()
     core_config = core_config_manager.get_config_obj()
     
 
     # Execute the mpi import statement that needs to call MPI_Init
     global mpi
     mpikey = kernel_config['mpi']['default']
     mpi_import_statement = kernel_config['mpi'].get(mpikey, None)
     if mpi_import_statement is not None:
         try:
             exec mpi_import_statement in globals()
         except:
             mpi = None
     else:
         mpi = None
     
     # Start logging
     logfile = kernel_config['engine']['logfile']
     if logfile:
         logfile = logfile + str(os.getpid()) + '.log'
         try:
             openLogFile = open(logfile, 'w')
         except:
             openLogFile = sys.stdout
     else:
         openLogFile = sys.stdout
     log.startLogging(openLogFile)
     
     # Create the underlying shell class and EngineService
     shell_class = import_item(core_config['shell']['shell_class'])
     engine_service = EngineService(shell_class, mpi=mpi)
     shell_import_statement = core_config['shell']['import_statement']
     if shell_import_statement:
         try:
             engine_service.execute(shell_import_statement)
         except:
             log.msg("Error running import_statement: %s" % shell_import_statement)
     
     # Create the service hierarchy
     main_service = service.MultiService()
     engine_service.setServiceParent(main_service)
     tub_service = Tub()
     tub_service.setServiceParent(main_service)
     # This needs to be called before the connection is initiated
     main_service.startService()
     
     # This initiates the connection to the controller and calls
     # register_engine to tell the controller we are ready to do work
     engine_connector = EngineConnector(tub_service)
     furl_file = kernel_config['engine']['furl_file']
     log.msg("Using furl file: %s" % furl_file)
     
     def call_connect(engine_service, furl_file):
         d = engine_connector.connect_to_controller(engine_service, furl_file)
         def handle_error(f):
             # If this print statement is replaced by a log.err(f) I get
             # an unhandled error, which makes no sense.  I shouldn't have
             # to use a print statement here.  My only thought is that
             # at the beginning of the process the logging is still starting up
             print "error connecting to controller:", f.getErrorMessage()
             reactor.callLater(0.1, reactor.stop)
         d.addErrback(handle_error)
     
     reactor.callWhenRunning(call_connect, engine_service, furl_file)
     reactor.run()
 
 
 def init_config():
     """
     Initialize the configuration using default and command line options.
     """
     
-    parser = OptionParser()
+    parser = OptionParser("""ipengine [options]
+
+Start an IPython engine.
+
+Use the IPYTHONDIR environment variable to change your IPython directory 
+from the default of .ipython or _ipython.  The log and security 
+subdirectories of your IPython directory will be used by this script 
+for log files and security files.""")
     
     parser.add_option(
         "--furl-file",
         type="string", 
         dest="furl_file",
         help="The filename containing the FURL of the controller"
     )
     parser.add_option(
         "--mpi",
         type="string",
         dest="mpi",
         help="How to enable MPI (mpi4py, pytrilinos, or empty string to disable)"
     )
     parser.add_option(
         "-l",
         "--logfile",
         type="string",
         dest="logfile",
         help="log file name (default is stdout)"
     )
     
     (options, args) = parser.parse_args()
     
     kernel_config = kernel_config_manager.get_config_obj()
     # Now override with command line options
     if options.furl_file is not None:
         kernel_config['engine']['furl_file'] = options.furl_file
     if options.logfile is not None:
         kernel_config['engine']['logfile'] = options.logfile
     if options.mpi is not None:
         kernel_config['mpi']['default'] = options.mpi
 
 
 def main():
     """
     After creating the configuration information, start the engine.
     """
     init_config()
     start_engine()
 
 
 if __name__ == "__main__":
     main()

IPython/kernel/tests/test_contexts.py

@@ -1,43 +1,46 b''
+# Tell nose to skip this module
+__test__ = {}
+
 #from __future__ import with_statement
 
 # XXX This file is currently disabled to preserve 2.4 compatibility.
 
 #def test_simple():
 if 0:
 
     # XXX - for now, we need a running cluster to be started separately.  The
     # daemon work is almost finished, and will make much of this unnecessary.
     from IPython.kernel import client
     mec = client.MultiEngineClient(('127.0.0.1',10105))
 
     try:
         mec.get_ids()
     except ConnectionRefusedError:
         import os, time
         os.system('ipcluster -n 2 &')
         time.sleep(2)
         mec = client.MultiEngineClient(('127.0.0.1',10105))
 
     mec.block = False
 
     import itertools
     c = itertools.count()
 
     parallel = RemoteMultiEngine(mec)
 
     mec.pushAll()
 
     ## with parallel as pr:
     ##     # A comment
     ##     remote()  # this means the code below only runs remotely
     ##     print 'Hello remote world'
     ##     x = range(10)
     ##     # Comments are OK
     ##     # Even misindented.
     ##     y = x+1
 
 
     ## with pfor('i',sequence) as pr:
     ##     print x[i]
 
     print pr.x + pr.y

IPython/kernel/tests/test_controllerservice.py

@@ -1,44 +1,44 b''
 # encoding: utf-8
 
 """This file contains unittests for the kernel.engineservice.py module.
 
 Things that should be tested:
 
  - Should the EngineService return Deferred objects?
  - Run the same tests that are run in shell.py.
  - Make sure that the Interface is really implemented.
  - The startService and stopService methods.
 """
 
 __docformat__ = "restructuredtext en"
 
 #-------------------------------------------------------------------------------
 #  Copyright (C) 2008  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-------------------------------------------------------------------------------
 
 #-------------------------------------------------------------------------------
 # Imports
 #-------------------------------------------------------------------------------
 
-try:
-    from twisted.application.service import IService
-    from IPython.kernel.controllerservice import ControllerService
-    from IPython.kernel.tests import multienginetest as met
-    from controllertest import IControllerCoreTestCase
-    from IPython.testing.util import DeferredTestCase
-except ImportError:
-    import nose
-    raise nose.SkipTest("This test requires zope.interface, Twisted and Foolscap")
+# Tell nose to skip this module
+__test__ = {}
+
+from twisted.application.service import IService
+from IPython.kernel.controllerservice import ControllerService
+from IPython.kernel.tests import multienginetest as met
+from controllertest import IControllerCoreTestCase
+from IPython.testing.util import DeferredTestCase
+
 
 class BasicControllerServiceTest(DeferredTestCase,
     IControllerCoreTestCase):
 
     def setUp(self):
         self.controller  = ControllerService()
         self.controller.startService()
 
     def tearDown(self):
         self.controller.stopService()

IPython/kernel/tests/test_enginefc.py

@@ -1,93 +1,92 b''
 # encoding: utf-8
 
 """This file contains unittests for the enginepb.py module."""
 
 __docformat__ = "restructuredtext en"
 
 #-------------------------------------------------------------------------------
 #  Copyright (C) 2008  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-------------------------------------------------------------------------------
 
 #-------------------------------------------------------------------------------
 # Imports
 #-------------------------------------------------------------------------------
 
-try:
-    from twisted.python import components
-    from twisted.internet import reactor, defer
-    from twisted.spread import pb
-    from twisted.internet.base import DelayedCall
-    DelayedCall.debug = True
+# Tell nose to skip this module
+__test__ = {}
 
-    import zope.interface as zi
+from twisted.python import components
+from twisted.internet import reactor, defer
+from twisted.spread import pb
+from twisted.internet.base import DelayedCall
+DelayedCall.debug = True
 
-    from IPython.kernel.fcutil import Tub, UnauthenticatedTub
-    from IPython.kernel import engineservice as es
-    from IPython.testing.util import DeferredTestCase
-    from IPython.kernel.controllerservice import IControllerBase
-    from IPython.kernel.enginefc import FCRemoteEngineRefFromService, IEngineBase
-    from IPython.kernel.engineservice import IEngineQueued
-    from IPython.kernel.engineconnector import EngineConnector
-    
-    from IPython.kernel.tests.engineservicetest import \
-        IEngineCoreTestCase, \
-        IEngineSerializedTestCase, \
-        IEngineQueuedTestCase
-except ImportError:
-    import nose
-    raise nose.SkipTest("This test requires zope.interface, Twisted and Foolscap")
+import zope.interface as zi
+
+from IPython.kernel.fcutil import Tub, UnauthenticatedTub
+from IPython.kernel import engineservice as es
+from IPython.testing.util import DeferredTestCase
+from IPython.kernel.controllerservice import IControllerBase
+from IPython.kernel.enginefc import FCRemoteEngineRefFromService, IEngineBase
+from IPython.kernel.engineservice import IEngineQueued
+from IPython.kernel.engineconnector import EngineConnector
+
+from IPython.kernel.tests.engineservicetest import \
+    IEngineCoreTestCase, \
+    IEngineSerializedTestCase, \
+    IEngineQueuedTestCase
 
 
 class EngineFCTest(DeferredTestCase, 
                  IEngineCoreTestCase, 
                  IEngineSerializedTestCase,
                  IEngineQueuedTestCase
                  ):
  
   zi.implements(IControllerBase)
  
   def setUp(self):
  
       # Start a server and append to self.servers
       self.controller_reference = FCRemoteEngineRefFromService(self)
       self.controller_tub = Tub()
       self.controller_tub.listenOn('tcp:10105:interface=127.0.0.1')
       self.controller_tub.setLocation('127.0.0.1:10105')
      
       furl = self.controller_tub.registerReference(self.controller_reference)
       self.controller_tub.startService()
      
       # Start an EngineService and append to services/client
       self.engine_service = es.EngineService()
       self.engine_service.startService()
       self.engine_tub = Tub()
       self.engine_tub.startService()
       engine_connector = EngineConnector(self.engine_tub)
       d = engine_connector.connect_to_controller(self.engine_service, furl)
       # This deferred doesn't fire until after register_engine has returned and
       # thus, self.engine has been defined and the tets can proceed.
       return d
  
   def tearDown(self):
       dlist = []
       # Shut down the engine
       d = self.engine_tub.stopService()
       dlist.append(d)
       # Shut down the controller
       d = self.controller_tub.stopService()
       dlist.append(d)
       return defer.DeferredList(dlist)
  
   #---------------------------------------------------------------------------
   # Make me look like a basic controller
   #---------------------------------------------------------------------------
  
   def register_engine(self, engine_ref, id=None, ip=None, port=None, pid=None):
       self.engine = IEngineQueued(IEngineBase(engine_ref))
       return {'id':id}
  
   def unregister_engine(self, id):
       pass
  No newline at end of file

IPython/kernel/tests/test_engineservice.py

@@ -1,80 +1,79 b''
 # encoding: utf-8
 
 """This file contains unittests for the kernel.engineservice.py module.
 
 Things that should be tested:
 
  - Should the EngineService return Deferred objects?
  - Run the same tests that are run in shell.py.
  - Make sure that the Interface is really implemented.
  - The startService and stopService methods.
 """
 
 __docformat__ = "restructuredtext en"
 
 #-------------------------------------------------------------------------------
 #  Copyright (C) 2008  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-------------------------------------------------------------------------------
 
 #-------------------------------------------------------------------------------
 # Imports
 #-------------------------------------------------------------------------------
 
-try:
-    from twisted.internet import defer
-    from twisted.application.service import IService
-    
-    from IPython.kernel import engineservice as es
-    from IPython.testing.util import DeferredTestCase
-    from IPython.kernel.tests.engineservicetest import \
-        IEngineCoreTestCase, \
-        IEngineSerializedTestCase, \
-        IEngineQueuedTestCase, \
-        IEnginePropertiesTestCase
-except ImportError:
-    import nose
-    raise nose.SkipTest("This test requires zope.interface, Twisted and Foolscap")    
+# Tell nose to skip this module
+__test__ = {}
+
+from twisted.internet import defer
+from twisted.application.service import IService
+
+from IPython.kernel import engineservice as es
+from IPython.testing.util import DeferredTestCase
+from IPython.kernel.tests.engineservicetest import \
+    IEngineCoreTestCase, \
+    IEngineSerializedTestCase, \
+    IEngineQueuedTestCase, \
+    IEnginePropertiesTestCase
 
 
 class BasicEngineServiceTest(DeferredTestCase,
                              IEngineCoreTestCase, 
                              IEngineSerializedTestCase,
                              IEnginePropertiesTestCase):
 
     def setUp(self):
         self.engine = es.EngineService()
         self.engine.startService()
 
     def tearDown(self):
         return self.engine.stopService()
 
 class ThreadedEngineServiceTest(DeferredTestCase,
                              IEngineCoreTestCase, 
                              IEngineSerializedTestCase,
                              IEnginePropertiesTestCase):
 
     def setUp(self):
         self.engine = es.ThreadedEngineService()
         self.engine.startService()
 
     def tearDown(self):
         return self.engine.stopService()
 
 class QueuedEngineServiceTest(DeferredTestCase,
                               IEngineCoreTestCase, 
                               IEngineSerializedTestCase,
                               IEnginePropertiesTestCase,
                               IEngineQueuedTestCase):
                           
     def setUp(self):
         self.rawEngine = es.EngineService()
         self.rawEngine.startService()
         self.engine = es.IEngineQueued(self.rawEngine)
     
     def tearDown(self):
         return self.rawEngine.stopService()
 
 

IPython/kernel/tests/test_multiengine.py

@@ -1,56 +1,55 b''
 # encoding: utf-8
 
 """"""
 
 __docformat__ = "restructuredtext en"
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
-try:
-    from twisted.internet import defer
-    from IPython.testing.util import DeferredTestCase
-    from IPython.kernel.controllerservice import ControllerService
-    from IPython.kernel import multiengine as me
-    from IPython.kernel.tests.multienginetest import (IMultiEngineTestCase,
-        ISynchronousMultiEngineTestCase)
-except ImportError:
-    import nose
-    raise nose.SkipTest("This test requires zope.interface, Twisted and Foolscap")
-
- 
+# Tell nose to skip this module
+__test__ = {}
+
+from twisted.internet import defer
+from IPython.testing.util import DeferredTestCase
+from IPython.kernel.controllerservice import ControllerService
+from IPython.kernel import multiengine as me
+from IPython.kernel.tests.multienginetest import (IMultiEngineTestCase,
+    ISynchronousMultiEngineTestCase)
+
+
 class BasicMultiEngineTestCase(DeferredTestCase, IMultiEngineTestCase):
 
     def setUp(self):
         self.controller = ControllerService()
         self.controller.startService()
         self.multiengine = me.IMultiEngine(self.controller)
         self.engines = []
     
     def tearDown(self):
         self.controller.stopService()
         for e in self.engines:
             e.stopService()
 
 
 class SynchronousMultiEngineTestCase(DeferredTestCase, ISynchronousMultiEngineTestCase):
 
     def setUp(self):
         self.controller = ControllerService()
         self.controller.startService()
         self.multiengine = me.ISynchronousMultiEngine(me.IMultiEngine(self.controller))
         self.engines = []
     
     def tearDown(self):
         self.controller.stopService()
         for e in self.engines:
             e.stopService()
 

IPython/kernel/tests/test_multienginefc.py

@@ -1,144 +1,144 b''
 #!/usr/bin/env python
 # encoding: utf-8
 
 __docformat__ = "restructuredtext en"
 
 #-------------------------------------------------------------------------------
 #  Copyright (C) 2008  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-------------------------------------------------------------------------------
 
 #-------------------------------------------------------------------------------
 # Imports
 #-------------------------------------------------------------------------------
 
-try:
-    from twisted.internet import defer, reactor
+# Tell nose to skip this module
+__test__ = {}
+
+from twisted.internet import defer, reactor
+
+from IPython.kernel.fcutil import Tub, UnauthenticatedTub
+
+from IPython.testing.util import DeferredTestCase
+from IPython.kernel.controllerservice import ControllerService
+from IPython.kernel.multiengine import IMultiEngine
+from IPython.kernel.tests.multienginetest import IFullSynchronousMultiEngineTestCase
+from IPython.kernel.multienginefc import IFCSynchronousMultiEngine
+from IPython.kernel import multiengine as me
+from IPython.kernel.clientconnector import ClientConnector
+from IPython.kernel.parallelfunction import ParallelFunction
+from IPython.kernel.error import CompositeError
+from IPython.kernel.util import printer
 
-    from IPython.kernel.fcutil import Tub, UnauthenticatedTub
 
-    from IPython.testing.util import DeferredTestCase
-    from IPython.kernel.controllerservice import ControllerService
-    from IPython.kernel.multiengine import IMultiEngine
-    from IPython.kernel.tests.multienginetest import IFullSynchronousMultiEngineTestCase
-    from IPython.kernel.multienginefc import IFCSynchronousMultiEngine
-    from IPython.kernel import multiengine as me
-    from IPython.kernel.clientconnector import ClientConnector
-    from IPython.kernel.parallelfunction import ParallelFunction
-    from IPython.kernel.error import CompositeError
-    from IPython.kernel.util import printer
-except ImportError:
-    import nose
-    raise nose.SkipTest("This test requires zope.interface, Twisted and Foolscap")
-    
 def _raise_it(f):
     try:
         f.raiseException()
     except CompositeError, e:
         e.raise_exception()
 
 
 class FullSynchronousMultiEngineTestCase(DeferredTestCase, IFullSynchronousMultiEngineTestCase):
 
     def setUp(self):
     
         self.engines = []
             
         self.controller = ControllerService()
         self.controller.startService()
         self.imultiengine = IMultiEngine(self.controller)
         self.mec_referenceable = IFCSynchronousMultiEngine(self.imultiengine)
 
         self.controller_tub = Tub()
         self.controller_tub.listenOn('tcp:10105:interface=127.0.0.1')
         self.controller_tub.setLocation('127.0.0.1:10105')
     
         furl = self.controller_tub.registerReference(self.mec_referenceable)
         self.controller_tub.startService()
     
         self.client_tub = ClientConnector()
         d = self.client_tub.get_multiengine_client(furl)
         d.addCallback(self.handle_got_client)
         return d
     
     def handle_got_client(self, client):
         self.multiengine = client
 
     def tearDown(self):
         dlist = []
         # Shut down the multiengine client
         d = self.client_tub.tub.stopService()
         dlist.append(d)
         # Shut down the engines
         for e in self.engines:
             e.stopService()
         # Shut down the controller
         d = self.controller_tub.stopService()
         d.addBoth(lambda _: self.controller.stopService())
         dlist.append(d)
         return defer.DeferredList(dlist)
 
     def test_mapper(self):
         self.addEngine(4)
         m = self.multiengine.mapper()
         self.assertEquals(m.multiengine,self.multiengine)
         self.assertEquals(m.dist,'b')
         self.assertEquals(m.targets,'all')
         self.assertEquals(m.block,True)
     
     def test_map_default(self):
         self.addEngine(4)
         m = self.multiengine.mapper()
         d = m.map(lambda x: 2*x, range(10))
         d.addCallback(lambda r: self.assertEquals(r,[2*x for x in range(10)]))
         d.addCallback(lambda _: self.multiengine.map(lambda x: 2*x, range(10)))
         d.addCallback(lambda r: self.assertEquals(r,[2*x for x in range(10)]))
         return d
     
     def test_map_noblock(self):
         self.addEngine(4)
         m = self.multiengine.mapper(block=False)
         d = m.map(lambda x: 2*x, range(10))
         d.addCallback(lambda did: self.multiengine.get_pending_deferred(did, True))
         d.addCallback(lambda r: self.assertEquals(r,[2*x for x in range(10)]))
         return d
             
     def test_mapper_fail(self):
         self.addEngine(4)
         m = self.multiengine.mapper()
         d = m.map(lambda x: 1/0, range(10))
         d.addBoth(lambda f: self.assertRaises(ZeroDivisionError, _raise_it, f))
         return d
     
     def test_parallel(self):
         self.addEngine(4)
         p = self.multiengine.parallel()
         self.assert_(isinstance(p, ParallelFunction))
         @p
         def f(x): return 2*x
         d = f(range(10))
         d.addCallback(lambda r: self.assertEquals(r,[2*x for x in range(10)]))
         return d
     
     def test_parallel_noblock(self):
         self.addEngine(1)
         p = self.multiengine.parallel(block=False)
         self.assert_(isinstance(p, ParallelFunction))
         @p
         def f(x): return 2*x
         d = f(range(10))
         d.addCallback(lambda did: self.multiengine.get_pending_deferred(did, True))
         d.addCallback(lambda r: self.assertEquals(r,[2*x for x in range(10)]))
         return d
     
     def test_parallel_fail(self):
         self.addEngine(4)
         p = self.multiengine.parallel()
         self.assert_(isinstance(p, ParallelFunction))
         @p
         def f(x): return 1/0
         d = f(range(10))
         d.addBoth(lambda f: self.assertRaises(ZeroDivisionError, _raise_it, f))
         return d
  No newline at end of file

IPython/kernel/tests/test_newserialized.py

@@ -1,102 +1,102 b''
 # encoding: utf-8
 
 """This file contains unittests for the shell.py module."""
 
 __docformat__ = "restructuredtext en"
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
-try:
-    import zope.interface as zi
-    from twisted.trial import unittest
-    from IPython.testing.util import DeferredTestCase
+# Tell nose to skip this module
+__test__ = {}
+
+import zope.interface as zi
+from twisted.trial import unittest
+from IPython.testing.util import DeferredTestCase
+
+from IPython.kernel.newserialized import \
+    ISerialized, \
+    IUnSerialized, \
+    Serialized, \
+    UnSerialized, \
+    SerializeIt, \
+    UnSerializeIt
 
-    from IPython.kernel.newserialized import \
-        ISerialized, \
-        IUnSerialized, \
-        Serialized, \
-        UnSerialized, \
-        SerializeIt, \
-        UnSerializeIt
-except ImportError:
-    import nose
-    raise nose.SkipTest("This test requires zope.interface, Twisted and Foolscap")
 
 #-----------------------------------------------------------------------------
 # Tests
 #-----------------------------------------------------------------------------
 
 class SerializedTestCase(unittest.TestCase):
 
     def setUp(self):
         pass
     
     def tearDown(self):
         pass
     
     def testSerializedInterfaces(self):
 
         us = UnSerialized({'a':10, 'b':range(10)})
         s = ISerialized(us)
         uss = IUnSerialized(s)
         self.assert_(ISerialized.providedBy(s))
         self.assert_(IUnSerialized.providedBy(us))
         self.assert_(IUnSerialized.providedBy(uss))
         for m in list(ISerialized):
             self.assert_(hasattr(s, m))
         for m in list(IUnSerialized):
             self.assert_(hasattr(us, m))
         for m in list(IUnSerialized):
             self.assert_(hasattr(uss, m))
 
     def testPickleSerialized(self):
         obj = {'a':1.45345, 'b':'asdfsdf', 'c':10000L}
         original = UnSerialized(obj)
         originalSer = ISerialized(original)
         firstData = originalSer.getData()
         firstTD = originalSer.getTypeDescriptor()
         firstMD = originalSer.getMetadata()
         self.assert_(firstTD == 'pickle')
         self.assert_(firstMD == {})
         unSerialized = IUnSerialized(originalSer)
         secondObj = unSerialized.getObject()
         for k, v in secondObj.iteritems():
             self.assert_(obj[k] == v)
         secondSer = ISerialized(UnSerialized(secondObj))
         self.assert_(firstData == secondSer.getData())
         self.assert_(firstTD == secondSer.getTypeDescriptor() )
         self.assert_(firstMD == secondSer.getMetadata())
     
     def testNDArraySerialized(self):
         try:
             import numpy
         except ImportError:
             pass
         else:
             a = numpy.linspace(0.0, 1.0, 1000)
             unSer1 = UnSerialized(a)
             ser1 = ISerialized(unSer1)
             td = ser1.getTypeDescriptor()
             self.assert_(td == 'ndarray')
             md = ser1.getMetadata()
             self.assert_(md['shape'] == a.shape)
             self.assert_(md['dtype'] == a.dtype.str)
             buff = ser1.getData()
             self.assert_(buff == numpy.getbuffer(a))
             s = Serialized(buff, td, md)
             us = IUnSerialized(s)
             final = us.getObject()
             self.assert_(numpy.getbuffer(a) == numpy.getbuffer(final))
             self.assert_(a.dtype.str == final.dtype.str)
             self.assert_(a.shape == final.shape)
         
         

IPython/kernel/tests/test_pendingdeferred.py

@@ -1,186 +1,186 b''
 #!/usr/bin/env python
 # encoding: utf-8
 
 """Tests for pendingdeferred.py"""
 
 __docformat__ = "restructuredtext en"
 
 #-------------------------------------------------------------------------------
 #  Copyright (C) 2008  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-------------------------------------------------------------------------------
 
 #-------------------------------------------------------------------------------
 # Imports
 #-------------------------------------------------------------------------------
 
-try:
-    from twisted.internet import defer
-    from twisted.python import failure
-
-    from IPython.testing.util import DeferredTestCase
-    import IPython.kernel.pendingdeferred as pd
-    from IPython.kernel import error
-    from IPython.kernel.util import printer
-except ImportError:
-    import nose
-    raise nose.SkipTest("This test requires zope.interface, Twisted and Foolscap")
-    
+# Tell nose to skip this module
+__test__ = {}
+
+from twisted.internet import defer
+from twisted.python import failure
+
+from IPython.testing.util import DeferredTestCase
+import IPython.kernel.pendingdeferred as pd
+from IPython.kernel import error
+from IPython.kernel.util import printer
+
+
 class Foo(object):
 
     def bar(self, bahz):
         return defer.succeed('blahblah: %s' % bahz)
 
 class TwoPhaseFoo(pd.PendingDeferredManager):
 
     def __init__(self, foo):
         self.foo = foo
         pd.PendingDeferredManager.__init__(self)
 
     @pd.two_phase
     def bar(self, bahz):
         return self.foo.bar(bahz)
 
 class PendingDeferredManagerTest(DeferredTestCase):
 
     def setUp(self):
         self.pdm = pd.PendingDeferredManager()
     
     def tearDown(self):
         pass
 
     def testBasic(self):
         dDict = {}
         # Create 10 deferreds and save them
         for i in range(10):
             d = defer.Deferred()
             did = self.pdm.save_pending_deferred(d)
             dDict[did] = d
         # Make sure they are begin saved
         for k in dDict.keys():
             self.assert_(self.pdm.quick_has_id(k))
         # Get the pending deferred (block=True), then callback with 'foo' and compare
         for did in dDict.keys()[0:5]:
             d = self.pdm.get_pending_deferred(did,block=True)
             dDict[did].callback('foo')
             d.addCallback(lambda r: self.assert_(r=='foo'))
         # Get the pending deferreds with (block=False) and make sure ResultNotCompleted is raised
         for did in dDict.keys()[5:10]:
             d = self.pdm.get_pending_deferred(did,block=False)
             d.addErrback(lambda f: self.assertRaises(error.ResultNotCompleted, f.raiseException))
         # Now callback the last 5, get them and compare.
         for did in dDict.keys()[5:10]:
             dDict[did].callback('foo')
             d = self.pdm.get_pending_deferred(did,block=False)
             d.addCallback(lambda r: self.assert_(r=='foo'))
 
     def test_save_then_delete(self):
         d = defer.Deferred()
         did = self.pdm.save_pending_deferred(d)
         self.assert_(self.pdm.quick_has_id(did))
         self.pdm.delete_pending_deferred(did)
         self.assert_(not self.pdm.quick_has_id(did))
 
     def test_save_get_delete(self):
         d = defer.Deferred()
         did = self.pdm.save_pending_deferred(d)
         d2 = self.pdm.get_pending_deferred(did,True)
         d2.addErrback(lambda f: self.assertRaises(error.AbortedPendingDeferredError, f.raiseException))
         self.pdm.delete_pending_deferred(did)
         return d2
 
     def test_double_get(self):
         d = defer.Deferred()
         did = self.pdm.save_pending_deferred(d)
         d2 = self.pdm.get_pending_deferred(did,True)
         d3 = self.pdm.get_pending_deferred(did,True)
         d3.addErrback(lambda f: self.assertRaises(error.InvalidDeferredID, f.raiseException))
 
     def test_get_after_callback(self):
         d = defer.Deferred()
         did = self.pdm.save_pending_deferred(d)
         d.callback('foo')
         d2 = self.pdm.get_pending_deferred(did,True)
         d2.addCallback(lambda r: self.assertEquals(r,'foo'))
         self.assert_(not self.pdm.quick_has_id(did))
 
     def test_get_before_callback(self):
         d = defer.Deferred()
         did = self.pdm.save_pending_deferred(d)
         d2 = self.pdm.get_pending_deferred(did,True)
         d.callback('foo')
         d2.addCallback(lambda r: self.assertEquals(r,'foo'))
         self.assert_(not self.pdm.quick_has_id(did))
         d = defer.Deferred()
         did = self.pdm.save_pending_deferred(d)
         d2 = self.pdm.get_pending_deferred(did,True)
         d2.addCallback(lambda r: self.assertEquals(r,'foo'))
         d.callback('foo')
         self.assert_(not self.pdm.quick_has_id(did))
 
     def test_get_after_errback(self):
         class MyError(Exception):
             pass
         d = defer.Deferred()
         did = self.pdm.save_pending_deferred(d)
         d.errback(failure.Failure(MyError('foo')))
         d2 = self.pdm.get_pending_deferred(did,True)
         d2.addErrback(lambda f: self.assertRaises(MyError, f.raiseException))
         self.assert_(not self.pdm.quick_has_id(did))
 
     def test_get_before_errback(self):
         class MyError(Exception):
             pass
         d = defer.Deferred()
         did = self.pdm.save_pending_deferred(d)
         d2 = self.pdm.get_pending_deferred(did,True)
         d.errback(failure.Failure(MyError('foo')))
         d2.addErrback(lambda f: self.assertRaises(MyError, f.raiseException))
         self.assert_(not self.pdm.quick_has_id(did))
         d = defer.Deferred()
         did = self.pdm.save_pending_deferred(d)
         d2 = self.pdm.get_pending_deferred(did,True)
         d2.addErrback(lambda f: self.assertRaises(MyError, f.raiseException))
         d.errback(failure.Failure(MyError('foo')))
         self.assert_(not self.pdm.quick_has_id(did))
     
     def test_noresult_noblock(self):
         d = defer.Deferred()
         did = self.pdm.save_pending_deferred(d)
         d2 = self.pdm.get_pending_deferred(did,False)
         d2.addErrback(lambda f: self.assertRaises(error.ResultNotCompleted, f.raiseException))
 
     def test_with_callbacks(self):
         d = defer.Deferred()
         d.addCallback(lambda r: r+' foo')
         d.addCallback(lambda r: r+' bar')
         did = self.pdm.save_pending_deferred(d)
         d2 = self.pdm.get_pending_deferred(did,True)
         d.callback('bam')
         d2.addCallback(lambda r: self.assertEquals(r,'bam foo bar'))
 
     def test_with_errbacks(self):
         class MyError(Exception):
             pass
         d = defer.Deferred()
         d.addCallback(lambda r: 'foo')
         d.addErrback(lambda f: 'caught error')
         did = self.pdm.save_pending_deferred(d)
         d2 = self.pdm.get_pending_deferred(did,True)
         d.errback(failure.Failure(MyError('bam')))
         d2.addErrback(lambda f: self.assertRaises(MyError, f.raiseException))
 
     def test_nested_deferreds(self):
         d = defer.Deferred()
         d2 = defer.Deferred()
         d.addCallback(lambda r: d2)
         did = self.pdm.save_pending_deferred(d)
         d.callback('foo')
         d3 = self.pdm.get_pending_deferred(did,False)
         d3.addErrback(lambda f: self.assertRaises(error.ResultNotCompleted, f.raiseException))
         d2.callback('bar')
         d3 = self.pdm.get_pending_deferred(did,False)
         d3.addCallback(lambda r: self.assertEquals(r,'bar'))
 

IPython/kernel/tests/test_task.py

@@ -1,51 +1,51 b''
 # encoding: utf-8
 
 """This file contains unittests for the kernel.task.py module."""
 
 __docformat__ = "restructuredtext en"
 
 #-------------------------------------------------------------------------------
 #  Copyright (C) 2008  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-------------------------------------------------------------------------------
 
 #-------------------------------------------------------------------------------
 # Imports
 #-------------------------------------------------------------------------------
 
-try:
-    import time
-
-    from twisted.internet import defer
-    from twisted.trial import unittest
-
-    from IPython.kernel import task, controllerservice as cs, engineservice as es
-    from IPython.kernel.multiengine import IMultiEngine
-    from IPython.testing.util import DeferredTestCase
-    from IPython.kernel.tests.tasktest import ITaskControllerTestCase
-except ImportError:
-    import nose
-    raise nose.SkipTest("This test requires zope.interface, Twisted and Foolscap")
+# Tell nose to skip this module
+__test__ = {}
+
+import time
+
+from twisted.internet import defer
+from twisted.trial import unittest
+
+from IPython.kernel import task, controllerservice as cs, engineservice as es
+from IPython.kernel.multiengine import IMultiEngine
+from IPython.testing.util import DeferredTestCase
+from IPython.kernel.tests.tasktest import ITaskControllerTestCase
+
 
 #-------------------------------------------------------------------------------
 # Tests
 #-------------------------------------------------------------------------------
 
 class BasicTaskControllerTestCase(DeferredTestCase, ITaskControllerTestCase):
 
     def setUp(self):
         self.controller  = cs.ControllerService()
         self.controller.startService()
         self.multiengine = IMultiEngine(self.controller)
         self.tc = task.ITaskController(self.controller)
         self.tc.failurePenalty = 0
         self.engines=[]
     
     def tearDown(self):
         self.controller.stopService()
         for e in self.engines:
             e.stopService()
 
 

IPython/kernel/tests/test_taskfc.py

@@ -1,162 +1,161 b''
 #!/usr/bin/env python
 # encoding: utf-8
 
 __docformat__ = "restructuredtext en"
 
 #-------------------------------------------------------------------------------
 #  Copyright (C) 2008  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-------------------------------------------------------------------------------
 
 #-------------------------------------------------------------------------------
 # Imports
 #-------------------------------------------------------------------------------
 
-try:
-    import time
+# Tell nose to skip this module
+__test__ = {}
 
-    from twisted.internet import defer, reactor
+import time
 
-    from IPython.kernel.fcutil import Tub, UnauthenticatedTub
+from twisted.internet import defer, reactor
 
-    from IPython.kernel import task as taskmodule
-    from IPython.kernel import controllerservice as cs
-    import IPython.kernel.multiengine as me
-    from IPython.testing.util import DeferredTestCase
-    from IPython.kernel.multienginefc import IFCSynchronousMultiEngine
-    from IPython.kernel.taskfc import IFCTaskController
-    from IPython.kernel.util import printer
-    from IPython.kernel.tests.tasktest import ITaskControllerTestCase
-    from IPython.kernel.clientconnector import ClientConnector
-    from IPython.kernel.error import CompositeError
-    from IPython.kernel.parallelfunction import ParallelFunction
-except ImportError:
-    import nose
-    raise nose.SkipTest("This test requires zope.interface, Twisted and Foolscap")
+from IPython.kernel.fcutil import Tub, UnauthenticatedTub
+
+from IPython.kernel import task as taskmodule
+from IPython.kernel import controllerservice as cs
+import IPython.kernel.multiengine as me
+from IPython.testing.util import DeferredTestCase
+from IPython.kernel.multienginefc import IFCSynchronousMultiEngine
+from IPython.kernel.taskfc import IFCTaskController
+from IPython.kernel.util import printer
+from IPython.kernel.tests.tasktest import ITaskControllerTestCase
+from IPython.kernel.clientconnector import ClientConnector
+from IPython.kernel.error import CompositeError
+from IPython.kernel.parallelfunction import ParallelFunction
 
 
 #-------------------------------------------------------------------------------
 # Tests
 #-------------------------------------------------------------------------------
 
 def _raise_it(f):
     try:
         f.raiseException()
     except CompositeError, e:
         e.raise_exception()
 
 class TaskTest(DeferredTestCase, ITaskControllerTestCase):
 
     def setUp(self):
     
         self.engines = []
             
         self.controller = cs.ControllerService()
         self.controller.startService()
         self.imultiengine = me.IMultiEngine(self.controller)
         self.itc = taskmodule.ITaskController(self.controller)
         self.itc.failurePenalty = 0
     
         self.mec_referenceable = IFCSynchronousMultiEngine(self.imultiengine)
         self.tc_referenceable = IFCTaskController(self.itc)
     
         self.controller_tub = Tub()
         self.controller_tub.listenOn('tcp:10105:interface=127.0.0.1')
         self.controller_tub.setLocation('127.0.0.1:10105')
     
         mec_furl = self.controller_tub.registerReference(self.mec_referenceable)
         tc_furl = self.controller_tub.registerReference(self.tc_referenceable)
         self.controller_tub.startService()
     
         self.client_tub = ClientConnector()
         d = self.client_tub.get_multiengine_client(mec_furl)
         d.addCallback(self.handle_mec_client)
         d.addCallback(lambda _: self.client_tub.get_task_client(tc_furl))
         d.addCallback(self.handle_tc_client)
         return d
     
     def handle_mec_client(self, client):
         self.multiengine = client
 
     def handle_tc_client(self, client):
         self.tc = client
 
     def tearDown(self):
         dlist = []
         # Shut down the multiengine client
         d = self.client_tub.tub.stopService()
         dlist.append(d)
         # Shut down the engines
         for e in self.engines:
             e.stopService()
         # Shut down the controller
         d = self.controller_tub.stopService()
         d.addBoth(lambda _: self.controller.stopService())
         dlist.append(d)
         return defer.DeferredList(dlist)
     
     def test_mapper(self):
         self.addEngine(1)
         m = self.tc.mapper()
         self.assertEquals(m.task_controller,self.tc)
         self.assertEquals(m.clear_before,False)
         self.assertEquals(m.clear_after,False)
         self.assertEquals(m.retries,0)
         self.assertEquals(m.recovery_task,None)
         self.assertEquals(m.depend,None)
         self.assertEquals(m.block,True)
     
     def test_map_default(self):
         self.addEngine(1)
         m = self.tc.mapper()
         d = m.map(lambda x: 2*x, range(10))
         d.addCallback(lambda r: self.assertEquals(r,[2*x for x in range(10)]))
         d.addCallback(lambda _: self.tc.map(lambda x: 2*x, range(10)))
         d.addCallback(lambda r: self.assertEquals(r,[2*x for x in range(10)]))
         return d
     
     def test_map_noblock(self):
         self.addEngine(1)
         m = self.tc.mapper(block=False)
         d = m.map(lambda x: 2*x, range(10))
         d.addCallback(lambda r: self.assertEquals(r,[x for x in range(10)]))
         return d
             
     def test_mapper_fail(self):
         self.addEngine(1)
         m = self.tc.mapper()
         d = m.map(lambda x: 1/0, range(10))
         d.addBoth(lambda f: self.assertRaises(ZeroDivisionError, _raise_it, f))
         return d
     
     def test_parallel(self):
         self.addEngine(1)
         p = self.tc.parallel()
         self.assert_(isinstance(p, ParallelFunction))
         @p
         def f(x): return 2*x
         d = f(range(10))
         d.addCallback(lambda r: self.assertEquals(r,[2*x for x in range(10)]))
         return d
     
     def test_parallel_noblock(self):
         self.addEngine(1)
         p = self.tc.parallel(block=False)
         self.assert_(isinstance(p, ParallelFunction))
         @p
         def f(x): return 2*x
         d = f(range(10))
         d.addCallback(lambda r: self.assertEquals(r,[x for x in range(10)]))
         return d
     
     def test_parallel_fail(self):
         self.addEngine(1)
         p = self.tc.parallel()
         self.assert_(isinstance(p, ParallelFunction))
         @p
         def f(x): return 1/0
         d = f(range(10))
         d.addBoth(lambda f: self.assertRaises(ZeroDivisionError, _raise_it, f))
         return d
  No newline at end of file

IPython/kernel/tests/test_twistedutil.py

@@ -1,48 +1,51 b''
 #!/usr/bin/env python
 # encoding: utf-8
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
+# Tell nose to skip this module
+__test__ = {}
+
 import tempfile
 import os, sys
 
 from twisted.internet import reactor
 from twisted.trial import unittest
 
 from IPython.kernel.error import FileTimeoutError
 from IPython.kernel.twistedutil import wait_for_file
 
 #-----------------------------------------------------------------------------
 # Tests
 #-----------------------------------------------------------------------------
 
 class TestWaitForFile(unittest.TestCase):
 
     def test_delay(self):
         filename = tempfile.mktemp()
         def _create_file():
             open(filename,'w').write('####')
         dcall = reactor.callLater(0.5, _create_file)
         d = wait_for_file(filename,delay=0.1)
         d.addCallback(lambda r: self.assert_(r))
         def _cancel_dcall(r):
             if dcall.active():
                 dcall.cancel()
         d.addCallback(_cancel_dcall)
         return d
     
     def test_timeout(self):
         filename = tempfile.mktemp()
         d = wait_for_file(filename,delay=0.1,max_tries=1)
         d.addErrback(lambda f: self.assertRaises(FileTimeoutError,f.raiseException))
         return d
         
  No newline at end of file

IPython/testing/decorators.py

@@ -1,254 +1,254 b''
 """Decorators for labeling test objects.
 
 Decorators that merely return a modified version of the original function
 object are straightforward.  Decorators that return a new function object need
 to use nose.tools.make_decorator(original_function)(decorator) in returning the
 decorator, in order to preserve metadata such as function name, setup and
 teardown functions and so on - see nose.tools for more information.
 
 This module provides a set of useful decorators meant to be ready to use in
 your own tests.  See the bottom of the file for the ready-made ones, and if you
 find yourself writing a new one that may be of generic use, add it here.
 
 NOTE: This file contains IPython-specific decorators and imports the
 numpy.testing.decorators file, which we've copied verbatim.  Any of our own
 code will be added at the bottom if we end up extending this.
 """
 
 # Stdlib imports
 import inspect
 import sys
 
 # Third-party imports
 
 # This is Michele Simionato's decorator module, also kept verbatim.
 from decorator_msim import decorator, update_wrapper
 
 # Grab the numpy-specific decorators which we keep in a file that we
 # occasionally update from upstream: decorators_numpy.py is an IDENTICAL copy
 # of numpy.testing.decorators.
 from decorators_numpy import *
 
 ##############################################################################
 # Local code begins
 
 # Utility functions
 
 def apply_wrapper(wrapper,func):
     """Apply a wrapper to a function for decoration.
 
     This mixes Michele Simionato's decorator tool with nose's make_decorator,
     to apply a wrapper in a decorator so that all nose attributes, as well as
     function signature and other properties, survive the decoration cleanly.
     This will ensure that wrapped functions can still be well introspected via
     IPython, for example.
     """
     import nose.tools
 
     return decorator(wrapper,nose.tools.make_decorator(func)(wrapper))
 
 
 def make_label_dec(label,ds=None):
     """Factory function to create a decorator that applies one or more labels.
 
     :Parameters:
       label : string or sequence
       One or more labels that will be applied by the decorator to the functions
     it decorates.  Labels are attributes of the decorated function with their
     value set to True.
 
     :Keywords:
       ds : string
       An optional docstring for the resulting decorator.  If not given, a
       default docstring is auto-generated.
 
     :Returns:
       A decorator.
 
     :Examples:
 
     A simple labeling decorator:
     >>> slow = make_label_dec('slow')
     >>> print slow.__doc__
     Labels a test as 'slow'.
 
     And one that uses multiple labels and a custom docstring:
     >>> rare = make_label_dec(['slow','hard'],
     ... "Mix labels 'slow' and 'hard' for rare tests.")
     >>> print rare.__doc__
     Mix labels 'slow' and 'hard' for rare tests.
 
     Now, let's test using this one:
     >>> @rare
     ... def f(): pass
     ...
     >>>
     >>> f.slow
     True
     >>> f.hard
     True
     """
 
     if isinstance(label,basestring):
         labels = [label]
     else:
         labels = label
         
     # Validate that the given label(s) are OK for use in setattr() by doing a
     # dry run on a dummy function.
     tmp = lambda : None
     for label in labels:
         setattr(tmp,label,True)
 
     # This is the actual decorator we'll return
     def decor(f):
         for label in labels:
             setattr(f,label,True)
         return f
     
     # Apply the user's docstring, or autogenerate a basic one
     if ds is None:
         ds = "Labels a test as %r." % label
     decor.__doc__ = ds
     
     return decor
 
 
 # Inspired by numpy's skipif, but uses the full apply_wrapper utility to
 # preserve function metadata better and allows the skip condition to be a
 # callable.
 def skipif(skip_condition, msg=None):
     ''' Make function raise SkipTest exception if skip_condition is true
 
     Parameters
     ----------
     skip_condition : bool or callable. 
- 	Flag to determine whether to skip test.  If the condition is a 
- 	callable, it is used at runtime to dynamically make the decision.  This 
- 	is useful for tests that may require costly imports, to delay the cost 
- 	until the test suite is actually executed.        
+    Flag to determine whether to skip test.  If the condition is a 
+    callable, it is used at runtime to dynamically make the decision.  This 
+    is useful for tests that may require costly imports, to delay the cost 
+    until the test suite is actually executed.        
     msg : string
         Message to give on raising a SkipTest exception
 
    Returns
    -------
    decorator : function
        Decorator, which, when applied to a function, causes SkipTest
        to be raised when the skip_condition was True, and the function
        to be called normally otherwise.
 
     Notes
     -----
     You will see from the code that we had to further decorate the
     decorator with the nose.tools.make_decorator function in order to
     transmit function name, and various other metadata.
     '''
 
     def skip_decorator(f):
         # Local import to avoid a hard nose dependency and only incur the 
         # import time overhead at actual test-time. 
         import nose
 
         # Allow for both boolean or callable skip conditions.
         if callable(skip_condition):
             skip_val = lambda : skip_condition()
         else:
             skip_val = lambda : skip_condition
 
         def get_msg(func,msg=None):
             """Skip message with information about function being skipped."""
             if msg is None: out = 'Test skipped due to test condition.'
             else: out = msg
             return "Skipping test: %s. %s" % (func.__name__,out)
 
         # We need to define *two* skippers because Python doesn't allow both
         # return with value and yield inside the same function.
         def skipper_func(*args, **kwargs):
             """Skipper for normal test functions."""
             if skip_val():
                 raise nose.SkipTest(get_msg(f,msg))
             else:
                 return f(*args, **kwargs) 
 
         def skipper_gen(*args, **kwargs):
             """Skipper for test generators."""
             if skip_val():
                 raise nose.SkipTest(get_msg(f,msg))
             else:
                 for x in f(*args, **kwargs):
                     yield x
 
         # Choose the right skipper to use when building the actual generator.
         if nose.util.isgenerator(f):
             skipper = skipper_gen
         else:
             skipper = skipper_func
             
         return nose.tools.make_decorator(f)(skipper)
 
     return skip_decorator
 
 # A version with the condition set to true, common case just to attacha message
 # to a skip decorator
 def skip(msg=None):
     """Decorator factory - mark a test function for skipping from test suite.
 
     :Parameters:
       msg : string
         Optional message to be added.
 
     :Returns:
        decorator : function
          Decorator, which, when applied to a function, causes SkipTest
          to be raised, with the optional message added.
       """
 
     return skipif(True,msg)
 
 
 #-----------------------------------------------------------------------------
 # Utility functions for decorators
 def numpy_not_available():
     """Can numpy be imported?  Returns true if numpy does NOT import.
 
     This is used to make a decorator to skip tests that require numpy to be
     available, but delay the 'import numpy' to test execution time.
     """
     try:
         import numpy
         np_not_avail = False
     except ImportError:
         np_not_avail = True
 
     return np_not_avail
 
 #-----------------------------------------------------------------------------
 # Decorators for public use
 
 skip_doctest = make_label_dec('skip_doctest',
     """Decorator - mark a function or method for skipping its doctest.
 
     This decorator allows you to mark a function whose docstring you wish to
     omit from testing, while preserving the docstring for introspection, help,
     etc.""")                              
 
 # Decorators to skip certain tests on specific platforms.
 skip_win32 = skipif(sys.platform == 'win32',
                     "This test does not run under Windows")
 skip_linux = skipif(sys.platform == 'linux2',
                     "This test does not run under Linux")
 skip_osx = skipif(sys.platform == 'darwin',"This test does not run under OS X")
 
 
 # Decorators to skip tests if not on specific platforms.
 skip_if_not_win32 = skipif(sys.platform != 'win32',
                            "This test only runs under Windows")
 skip_if_not_linux = skipif(sys.platform != 'linux2',
                            "This test only runs under Linux")
 skip_if_not_osx = skipif(sys.platform != 'darwin',
                          "This test only runs under OSX")
 
 # Other skip decorators
 skipif_not_numpy = skipif(numpy_not_available,"This test requires numpy")
 
 skipknownfailure = skip('This test is known to fail')

docs/source/parallel/parallel_process.txt

@@ -1,351 +1,351 b''
 .. _parallel_process:
 
 ===========================================
 Starting the IPython controller and engines
 ===========================================
 
 To use IPython for parallel computing, you need to start one instance of
 the controller and one or more instances of the engine. The controller
 and each engine can run on different machines or on the same machine.
 Because of this, there are many different possibilities.
 
 Broadly speaking, there are two ways of going about starting a controller and engines:
 
 * In an automated manner using the :command:`ipcluster` command.
 * In a more manual way using the :command:`ipcontroller` and 
   :command:`ipengine` commands.
 
 This document describes both of these methods.  We recommend that new users start with the :command:`ipcluster` command as it simplifies many common usage cases.
 
 General considerations
 ======================
 
 Before delving into the details about how you can start a controller and engines using the various methods, we outline some of the general issues that come up when starting the controller and engines.  These things come up no matter which method you use to start your IPython cluster.
 
 Let's say that you want to start the controller on ``host0`` and engines on hosts ``host1``-``hostn``.  The following steps are then required:
 
 1. Start the controller on ``host0`` by running :command:`ipcontroller` on
    ``host0``.
 2. Move the FURL file (:file:`ipcontroller-engine.furl`) created by the 
    controller from ``host0`` to hosts ``host1``-``hostn``.
 3. Start the engines on hosts ``host1``-``hostn`` by running 
    :command:`ipengine`.  This command has to be told where the FURL file 
    (:file:`ipcontroller-engine.furl`) is located.
 
 At this point, the controller and engines will be connected. By default, the
 FURL files created by the controller are put into the
 :file:`~/.ipython/security` directory. If the engines share a filesystem with
 the controller, step 2 can be skipped as the engines will automatically look
 at that location.
 
 The final step required required to actually use the running controller from a
 client is to move the FURL files :file:`ipcontroller-mec.furl` and
 :file:`ipcontroller-tc.furl` from ``host0`` to the host where the clients will
 be run.  If these file are put into the :file:`~/.ipython/security` directory of the client's host, they will be found automatically.  Otherwise, the full path to them has to be passed to the client's constructor.
 
 Using :command:`ipcluster`
 ==========================
 
 The :command:`ipcluster` command provides a simple way of starting a controller and engines in the following situations:
 
 1. When the controller and engines are all run on localhost. This is useful
    for testing or running on a multicore computer.
 2. When engines are started using the :command:`mpirun` command that comes 
    with most MPI [MPI]_ implementations
 3. When engines are started using the PBS [PBS]_ batch system.
 4. When the controller is started on localhost and the engines are started on 
    remote nodes using :command:`ssh`.
 
 .. note::
 
     It is also possible for advanced users to add support to
     :command:`ipcluster` for starting controllers and engines using other
     methods (like Sun's Grid Engine for example).
 
 .. note::
 
     Currently :command:`ipcluster` requires that the
     :file:`~/.ipython/security` directory live on a shared filesystem that is
     seen by both the controller and engines. If you don't have a shared file
     system you will need to use :command:`ipcontroller` and
     :command:`ipengine` directly.  This constraint can be relaxed if you are 
     using the :command:`ssh` method to start the cluster.
 
 Underneath the hood, :command:`ipcluster` just uses :command:`ipcontroller`
 and :command:`ipengine` to perform the steps described above.
 
 Using :command:`ipcluster` in local mode
 ----------------------------------------
 
 To start one controller and 4 engines on localhost, just do::
 
     $ ipcluster local -n 4
 
 To see other command line options for the local mode, do::
 
     $ ipcluster local -h
 
 Using :command:`ipcluster` in mpiexec/mpirun mode
 -------------------------------------------------
 
 The mpiexec/mpirun mode is useful if you:
 
 1. Have MPI installed.
 2. Your systems are configured to use the :command:`mpiexec` or
    :command:`mpirun` commands to start MPI processes.
 
 .. note::
 
     The preferred command to use is :command:`mpiexec`.  However, we also 
     support :command:`mpirun` for backwards compatibility.  The underlying
     logic used is exactly the same, the only difference being the name of the 
     command line program that is called.
 
 If these are satisfied, you can start an IPython cluster using::
 
     $ ipcluster mpiexec -n 4
 
 This does the following:
 
 1. Starts the IPython controller on current host.
 2. Uses :command:`mpiexec` to start 4 engines.
 
 On newer MPI implementations (such as OpenMPI), this will work even if you don't make any calls to MPI or call :func:`MPI_Init`.  However, older MPI implementations actually require each process to call :func:`MPI_Init` upon starting.  The easiest way of having this done is to install the mpi4py [mpi4py]_ package and then call ipcluster with the ``--mpi`` option::
 
     $ ipcluster mpiexec -n 4 --mpi=mpi4py
 
 Unfortunately, even this won't work for some MPI implementations.  If you are having problems with this, you will likely have to use a custom Python executable that itself calls :func:`MPI_Init` at the appropriate time.  Fortunately, mpi4py comes with such a custom Python executable that is easy to install and use.  However, this custom Python executable approach will not work with :command:`ipcluster` currently.
 
 Additional command line options for this mode can be found by doing::
 
     $ ipcluster mpiexec -h
 
 More details on using MPI with IPython can be found :ref:`here <parallelmpi>`.
 
 
 Using :command:`ipcluster` in PBS mode
 --------------------------------------
 
 The PBS mode uses the Portable Batch System [PBS]_ to start the engines.  To use this mode, you first need to create a PBS script template that will be used to start the engines.  Here is a sample PBS script template:
 
 .. sourcecode:: bash
 
     #PBS -N ipython
     #PBS -j oe
     #PBS -l walltime=00:10:00
     #PBS -l nodes=${n/4}:ppn=4
     #PBS -q parallel
 
     cd $$PBS_O_WORKDIR
     export PATH=$$HOME/usr/local/bin
     export PYTHONPATH=$$HOME/usr/local/lib/python2.4/site-packages
     /usr/local/bin/mpiexec -n ${n} ipengine --logfile=$$PBS_O_WORKDIR/ipengine
 
 There are a few important points about this template:
 
 1. This template will be rendered at runtime using IPython's :mod:`Itpl` 
    template engine.
 
 2. Instead of putting in the actual number of engines, use the notation
    ``${n}`` to indicate the number of engines to be started. You can also uses
    expressions like ``${n/4}`` in the template to indicate the number of 
    nodes.
 
 3. Because ``$`` is a special character used by the template engine, you must  
    escape any ``$`` by using ``$$``.  This is important when referring to 
    environment variables in the template.
 
 4. Any options to :command:`ipengine` should be given in the batch script 
    template.
 
 5. Depending on the configuration of you system, you may have to set 
    environment variables in the script template.
 
 Once you have created such a script, save it with a name like :file:`pbs.template`.  Now you are ready to start your job::
 
     $ ipcluster pbs -n 128 --pbs-script=pbs.template
 
 Additional command line options for this mode can be found by doing::
 
     $ ipcluster pbs -h
 
 Using :command:`ipcluster` in SSH mode
 --------------------------------------
 
 The SSH mode uses :command:`ssh` to execute :command:`ipengine` on remote
 nodes and the :command:`ipcontroller` on localhost.
 
 When using using this mode it highly recommended that you have set up SSH keys and are using ssh-agent [SSH]_ for password-less logins.
 
 To use this mode you need a python file describing the cluster, here is an example of such a "clusterfile":
 
 .. sourcecode:: python
     
     send_furl = True
     engines = { 'host1.example.com' : 2,
                 'host2.example.com' : 5,
                 'host3.example.com' : 1,
                 'host4.example.com' : 8 }
 
 Since this is a regular python file usual python syntax applies. Things to note:
 
 * The `engines` dict, where the keys is the host we want to run engines on and
   the value is the number of engines to run on that host.
 * send_furl can either be `True` or `False`, if `True` it will copy over the
   furl needed for :command:`ipengine` to each host.
 
 The ``--clusterfile`` command line option lets you specify the file to use for
 the cluster definition. Once you have your cluster file and you can
 :command:`ssh` into the remote hosts with out an password you are ready to
 start your cluster like so:
 
 .. sourcecode:: bash 
 
    $ ipcluster ssh --clusterfile /path/to/my/clusterfile.py
 
 
 Two helper shell scripts are used to start and stop :command:`ipengine` on remote hosts:
 
 * sshx.sh 
 * engine_killer.sh
 
 Defaults for both of these are contained in the source code for :command:`ipcluster`. The default scripts are written to a local file in a tmep directory and then copied to a temp directory on the remote host and executed from there.  On most Unix, Linux and OS X systems this is /tmp.
 
 The default sshx.sh is the following:
 
 .. sourcecode:: bash
 
    #!/bin/sh
    "$@" &> /dev/null &
    echo $!
 
 If you want to use a custom sshx.sh script you need to use the ``--sshx``
 option and specify the file to use. Using a custom sshx.sh file could be
 helpful when you need to setup the environment on the remote host before
 executing :command:`ipengine`.
 
 For a detailed options list:
 
 .. sourcecode:: bash
 
    $ ipcluster ssh -h
 
 Current limitations of the SSH mode of :command:`ipcluster` are:
 
 * Untested on Windows. Would require a working :command:`ssh` on Windows.
   Also, we are using shell scripts to setup and execute commands on remote
   hosts.
 * :command:`ipcontroller` is started on localhost, with no option to start it
   on a remote node.
 
 Using the :command:`ipcontroller` and :command:`ipengine` commands
 ==================================================================
 
 It is also possible to use the :command:`ipcontroller` and :command:`ipengine` commands to start your controller and engines.  This approach gives you full control over all aspects of the startup process.
 
 Starting the controller and engine on your local machine
 --------------------------------------------------------
 
 To use :command:`ipcontroller` and :command:`ipengine` to start things on your
 local machine, do the following.
 
 First start the controller::
 
 	$ ipcontroller
 	
 Next, start however many instances of the engine you want using (repeatedly) the command::
 
 	$ ipengine
 
 The engines should start and automatically connect to the controller using the FURL files in :file:`~./ipython/security`. You are now ready to use the controller and engines from IPython.
 
 .. warning:: 
 	
 	The order of the above operations is very important.  You *must*
  	start the controller before the engines, since the engines connect 
 	to the controller as they get started.
 
 .. note::
 
     On some platforms (OS X), to put the controller and engine into the
     background you may need to give these commands in the form ``(ipcontroller
     &)`` and ``(ipengine &)`` (with the parentheses) for them to work
     properly.
 
 Starting the controller and engines on different hosts
 ------------------------------------------------------
 
 When the controller and engines are running on different hosts, things are
 slightly more complicated, but the underlying ideas are the same:
 
 1. Start the controller on a host using :command:`ipcontroller`.
 2. Copy :file:`ipcontroller-engine.furl` from :file:`~./ipython/security` on the controller's host to the host where the engines will run.
 3. Use :command:`ipengine` on the engine's hosts to start the engines.
 
 The only thing you have to be careful of is to tell :command:`ipengine` where the :file:`ipcontroller-engine.furl` file is located.  There are two ways you can do this:
 
 * Put :file:`ipcontroller-engine.furl` in the :file:`~./ipython/security`
   directory on the engine's host, where it will be found automatically.
 * Call :command:`ipengine` with the ``--furl-file=full_path_to_the_file``
   flag.
 
 The ``--furl-file`` flag works like this::
 
     $ ipengine --furl-file=/path/to/my/ipcontroller-engine.furl
 
 .. note::
     
     If the controller's and engine's hosts all have a shared file system  
     (:file:`~./ipython/security` is the same on all of them), then things
     will just work!
 
 Make FURL files persistent
 ---------------------------
 
 At fist glance it may seem that that managing the FURL files is a bit
 annoying. Going back to the house and key analogy, copying the FURL around
 each time you start the controller is like having to make a new key every time
 you want to unlock the door and enter your house. As with your house, you want
 to be able to create the key (or FURL file) once, and then simply use it at
 any point in the future.
 
-This is possible. but before you do this, you **must** remove any old FURL
+This is possible, but before you do this, you **must** remove any old FURL
 files in the :file:`~/.ipython/security` directory.
 
 .. warning::
 
     You **must** remove old FURL files before using persistent FURL files.
 
 Then, The only thing you have to do is decide what ports the controller will
 listen on for the engines and clients. This is done as follows::
 
     $ ipcontroller -r --client-port=10101 --engine-port=10102
 
 These options also work with all of the various modes of
 :command:`ipcluster`::
 
     $ ipcluster local -n 2 -r --client-port=10101 --engine-port=10102
 
 Then, just copy the furl files over the first time and you are set. You can
 start and stop the controller and engines any many times as you want in the
 future, just make sure to tell the controller to use the *same* ports.
 
 .. note::
 
     You may ask the question: what ports does the controller listen on if you
     don't tell is to use specific ones? The default is to use high random port
     numbers. We do this for two reasons: i) to increase security through
     obscurity and ii) to multiple controllers on a given host to start and
     automatically use different ports.
 
 Log files
 ---------
 
 All of the components of IPython have log files associated with them.
 These log files can be extremely useful in debugging problems with
 IPython and can be found in the directory :file:`~/.ipython/log`.  Sending
 the log files to us will often help us to debug any problems.
 
 
 .. [PBS] Portable Batch System.  http://www.openpbs.org/
 .. [SSH] SSH-Agent http://en.wikipedia.org/wiki/Ssh-agent