Checkpoint before merging with upstream

IPython/Shell.py

@@ -1,1236 +1,1238 b''
 # -*- coding: utf-8 -*-
 """IPython Shell classes.
 
 All the matplotlib support code was co-developed with John Hunter,
 matplotlib's author.
 
 $Id: Shell.py 3024 2008-02-07 15:34:42Z darren.dale $"""
 
 #*****************************************************************************
 #       Copyright (C) 2001-2006 Fernando Perez <[e-mail]>
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #*****************************************************************************
 
 from IPython import Release
 __author__  = '%s <%s>' % Release.authors['Fernando']
 __license__ = Release.license
 
 # Code begins
 # Stdlib imports
 import __builtin__
 import __main__
 import Queue
 import inspect
 import os
 import sys
 import thread
 import threading
 import time
 
 from signal import signal, SIGINT
 
 try:
     import ctypes
     HAS_CTYPES = True
 except ImportError:
     HAS_CTYPES = False
 
 # IPython imports
 import IPython
 from IPython import ultraTB, ipapi
+from IPython.Magic import Magic
 from IPython.genutils import Term,warn,error,flag_calls, ask_yes_no
 from IPython.iplib import InteractiveShell
 from IPython.ipmaker import make_IPython
-from IPython.Magic import Magic
 from IPython.ipstruct import Struct
+from IPython.testing import decorators as testdec
 
 # Globals
 # global flag to pass around information about Ctrl-C without exceptions
 KBINT = False
 
 # global flag to turn on/off Tk support.
 USE_TK = False
 
 # ID for the main thread, used for cross-thread exceptions
 MAIN_THREAD_ID = thread.get_ident()
 
 # Tag when runcode() is active, for exception handling
 CODE_RUN = None
 
 # Default timeout for waiting for multithreaded shells (in seconds)
 GUI_TIMEOUT = 10
 
 #-----------------------------------------------------------------------------
 # This class is trivial now, but I want to have it in to publish a clean
 # interface. Later when the internals are reorganized, code that uses this
 # shouldn't have to change.
 
 class IPShell:
     """Create an IPython instance."""
     
     def __init__(self,argv=None,user_ns=None,user_global_ns=None,
                  debug=1,shell_class=InteractiveShell):
         self.IP = make_IPython(argv,user_ns=user_ns,
                                user_global_ns=user_global_ns,
                                debug=debug,shell_class=shell_class)
 
     def mainloop(self,sys_exit=0,banner=None):
         self.IP.mainloop(banner)
         if sys_exit:
             sys.exit()
 
 #-----------------------------------------------------------------------------
 def kill_embedded(self,parameter_s=''):
     """%kill_embedded : deactivate for good the current embedded IPython.
 
     This function (after asking for confirmation) sets an internal flag so that
     an embedded IPython will never activate again.  This is useful to
     permanently disable a shell that is being called inside a loop: once you've
     figured out what you needed from it, you may then kill it and the program
     will then continue to run without the interactive shell interfering again.
     """
     
     kill = ask_yes_no("Are you sure you want to kill this embedded instance "
                      "(y/n)? [y/N] ",'n')
     if kill:
         self.shell.embedded_active = False
         print "This embedded IPython will not reactivate anymore once you exit."
     
 class IPShellEmbed:
     """Allow embedding an IPython shell into a running program.
 
     Instances of this class are callable, with the __call__ method being an
     alias to the embed() method of an InteractiveShell instance.
 
     Usage (see also the example-embed.py file for a running example):
 
     ipshell = IPShellEmbed([argv,banner,exit_msg,rc_override])
 
     - argv: list containing valid command-line options for IPython, as they
     would appear in sys.argv[1:].
 
     For example, the following command-line options:
 
       $ ipython -prompt_in1 'Input <\\#>' -colors LightBG
 
     would be passed in the argv list as:
 
       ['-prompt_in1','Input <\\#>','-colors','LightBG']
 
     - banner: string which gets printed every time the interpreter starts.
 
     - exit_msg: string which gets printed every time the interpreter exits.
 
     - rc_override: a dict or Struct of configuration options such as those
     used by IPython. These options are read from your ~/.ipython/ipythonrc
     file when the Shell object is created. Passing an explicit rc_override
     dict with any options you want allows you to override those values at
     creation time without having to modify the file. This way you can create
     embeddable instances configured in any way you want without editing any
     global files (thus keeping your interactive IPython configuration
     unchanged).
 
     Then the ipshell instance can be called anywhere inside your code:
     
     ipshell(header='') -> Opens up an IPython shell.
 
     - header: string printed by the IPython shell upon startup. This can let
     you know where in your code you are when dropping into the shell. Note
     that 'banner' gets prepended to all calls, so header is used for
     location-specific information.
 
     For more details, see the __call__ method below.
 
     When the IPython shell is exited with Ctrl-D, normal program execution
     resumes.
 
     This functionality was inspired by a posting on comp.lang.python by cmkl
     <[e-mail]> on Dec. 06/01 concerning similar uses of pyrepl, and
     by the IDL stop/continue commands."""
 
     def __init__(self,argv=None,banner='',exit_msg=None,rc_override=None,
                  user_ns=None):
         """Note that argv here is a string, NOT a list."""
         self.set_banner(banner)
         self.set_exit_msg(exit_msg)
         self.set_dummy_mode(0)
 
         # sys.displayhook is a global, we need to save the user's original
         # Don't rely on __displayhook__, as the user may have changed that.
         self.sys_displayhook_ori = sys.displayhook
 
         # save readline completer status
         try:
             #print 'Save completer',sys.ipcompleter  # dbg
             self.sys_ipcompleter_ori = sys.ipcompleter
         except:
             pass # not nested with IPython
         
         self.IP = make_IPython(argv,rc_override=rc_override,
                                embedded=True,
                                user_ns=user_ns)
 
         ip = ipapi.IPApi(self.IP)
         ip.expose_magic("kill_embedded",kill_embedded)
 
         # copy our own displayhook also
         self.sys_displayhook_embed = sys.displayhook
         # and leave the system's display hook clean
         sys.displayhook = self.sys_displayhook_ori
         # don't use the ipython crash handler so that user exceptions aren't
         # trapped
         sys.excepthook = ultraTB.FormattedTB(color_scheme = self.IP.rc.colors,
                                              mode = self.IP.rc.xmode,
                                              call_pdb = self.IP.rc.pdb)
         self.restore_system_completer()
 
     def restore_system_completer(self):
         """Restores the readline completer which was in place.
 
         This allows embedded IPython within IPython not to disrupt the
         parent's completion.
         """
         
         try:
             self.IP.readline.set_completer(self.sys_ipcompleter_ori)
             sys.ipcompleter = self.sys_ipcompleter_ori
         except:
             pass
 
     def __call__(self,header='',local_ns=None,global_ns=None,dummy=None):
         """Activate the interactive interpreter.
 
         __call__(self,header='',local_ns=None,global_ns,dummy=None) -> Start
         the interpreter shell with the given local and global namespaces, and
         optionally print a header string at startup.
 
         The shell can be globally activated/deactivated using the
         set/get_dummy_mode methods. This allows you to turn off a shell used
         for debugging globally.
 
         However, *each* time you call the shell you can override the current
         state of dummy_mode with the optional keyword parameter 'dummy'. For
         example, if you set dummy mode on with IPShell.set_dummy_mode(1), you
         can still have a specific call work by making it as IPShell(dummy=0).
 
         The optional keyword parameter dummy controls whether the call
         actually does anything.  """
 
         # If the user has turned it off, go away
         if not self.IP.embedded_active:
             return
 
         # Normal exits from interactive mode set this flag, so the shell can't
         # re-enter (it checks this variable at the start of interactive mode).
         self.IP.exit_now = False
 
         # Allow the dummy parameter to override the global __dummy_mode
         if dummy or (dummy != 0 and self.__dummy_mode):
             return
 
         # Set global subsystems (display,completions) to our values
         sys.displayhook = self.sys_displayhook_embed
         if self.IP.has_readline:
             self.IP.set_completer()
 
         if self.banner and header:
             format = '%s\n%s\n'
         else:
             format = '%s%s\n'
         banner =  format % (self.banner,header)
 
         # Call the embedding code with a stack depth of 1 so it can skip over
         # our call and get the original caller's namespaces.
         self.IP.embed_mainloop(banner,local_ns,global_ns,stack_depth=1)
 
         if self.exit_msg:
             print self.exit_msg
             
         # Restore global systems (display, completion)
         sys.displayhook = self.sys_displayhook_ori
         self.restore_system_completer()
     
     def set_dummy_mode(self,dummy):
         """Sets the embeddable shell's dummy mode parameter.
 
         set_dummy_mode(dummy): dummy = 0 or 1.
 
         This parameter is persistent and makes calls to the embeddable shell
         silently return without performing any action. This allows you to
         globally activate or deactivate a shell you're using with a single call.
 
         If you need to manually"""
 
         if dummy not in [0,1,False,True]:
             raise ValueError,'dummy parameter must be boolean'
         self.__dummy_mode = dummy
 
     def get_dummy_mode(self):
         """Return the current value of the dummy mode parameter.
         """
         return self.__dummy_mode
     
     def set_banner(self,banner):
         """Sets the global banner.
 
         This banner gets prepended to every header printed when the shell
         instance is called."""
 
         self.banner = banner
 
     def set_exit_msg(self,exit_msg):
         """Sets the global exit_msg.
 
         This exit message gets printed upon exiting every time the embedded
         shell is called. It is None by default. """
 
         self.exit_msg = exit_msg
 
 #-----------------------------------------------------------------------------
 if HAS_CTYPES:
     #  Add async exception support.  Trick taken from:
     # http://sebulba.wikispaces.com/recipe+thread2
     def _async_raise(tid, exctype):
         """raises the exception, performs cleanup if needed"""
         if not inspect.isclass(exctype):
             raise TypeError("Only types can be raised (not instances)")
         res = ctypes.pythonapi.PyThreadState_SetAsyncExc(tid,
                                                          ctypes.py_object(exctype))
         if res == 0:
             raise ValueError("invalid thread id")
         elif res != 1:
             # """if it returns a number greater than one, you're in trouble, 
             # and you should call it again with exc=NULL to revert the effect"""
             ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, 0)
             raise SystemError("PyThreadState_SetAsyncExc failed")
 
     def sigint_handler (signum,stack_frame):
         """Sigint handler for threaded apps.
 
         This is a horrible hack to pass information about SIGINT _without_
         using exceptions, since I haven't been able to properly manage
         cross-thread exceptions in GTK/WX.  In fact, I don't think it can be
         done (or at least that's my understanding from a c.l.py thread where
         this was discussed)."""
 
         global KBINT
 
         if CODE_RUN:
             _async_raise(MAIN_THREAD_ID,KeyboardInterrupt)
         else:
             KBINT = True
             print '\nKeyboardInterrupt - Press <Enter> to continue.',
             Term.cout.flush()
 
 else:
     def sigint_handler (signum,stack_frame):
         """Sigint handler for threaded apps.
 
         This is a horrible hack to pass information about SIGINT _without_
         using exceptions, since I haven't been able to properly manage
         cross-thread exceptions in GTK/WX.  In fact, I don't think it can be
         done (or at least that's my understanding from a c.l.py thread where
         this was discussed)."""
 
         global KBINT
 
         print '\nKeyboardInterrupt - Press <Enter> to continue.',
         Term.cout.flush()
         # Set global flag so that runsource can know that Ctrl-C was hit
         KBINT = True
 
 
 class MTInteractiveShell(InteractiveShell):
     """Simple multi-threaded shell."""
 
     # Threading strategy taken from:
     # http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/65109, by Brian
     # McErlean and John Finlay.  Modified with corrections by Antoon Pardon,
     # from the pygtk mailing list, to avoid lockups with system calls.
 
     # class attribute to indicate whether the class supports threads or not.
     # Subclasses with thread support should override this as needed.
     isthreaded = True
 
     def __init__(self,name,usage=None,rc=Struct(opts=None,args=None),
                  user_ns=None,user_global_ns=None,banner2='',
                  gui_timeout=GUI_TIMEOUT,**kw):
         """Similar to the normal InteractiveShell, but with threading control"""
         
         InteractiveShell.__init__(self,name,usage,rc,user_ns,
                                   user_global_ns,banner2)
 
         # Timeout we wait for GUI thread
         self.gui_timeout = gui_timeout
 
         # A queue to hold the code to be executed. 
         self.code_queue = Queue.Queue()
 
         # Stuff to do at closing time
         self._kill = None
         on_kill = kw.get('on_kill', [])
         # Check that all things to kill are callable:
         for t in on_kill:
             if not callable(t):
                 raise TypeError,'on_kill must be a list of callables'
         self.on_kill = on_kill
         # thread identity of the "worker thread" (that may execute code directly)
         self.worker_ident = None
         
     def runsource(self, source, filename="<input>", symbol="single"):
         """Compile and run some source in the interpreter.
 
         Modified version of code.py's runsource(), to handle threading issues.
         See the original for full docstring details."""
         
         global KBINT
         
         # If Ctrl-C was typed, we reset the flag and return right away
         if KBINT:
             KBINT = False
             return False
 
         if self._kill:
             # can't queue new code if we are being killed
             return True
         
         try:
             code = self.compile(source, filename, symbol)
         except (OverflowError, SyntaxError, ValueError):
             # Case 1
             self.showsyntaxerror(filename)
             return False
 
         if code is None:
             # Case 2
             return True
 
         # shortcut - if we are in worker thread, or the worker thread is not
         # running, execute directly (to allow recursion and prevent deadlock if
         # code is run early in IPython construction)
         
         if (self.worker_ident is None
             or self.worker_ident == thread.get_ident() ):
             InteractiveShell.runcode(self,code)
             return
 
         # Case 3
         # Store code in queue, so the execution thread can handle it.
 
         completed_ev, received_ev = threading.Event(), threading.Event() 
         
         self.code_queue.put((code,completed_ev, received_ev))
         # first make sure the message was received, with timeout
         received_ev.wait(self.gui_timeout)
         if not received_ev.isSet():
             # the mainloop is dead, start executing code directly
             print "Warning: Timeout for mainloop thread exceeded"
             print "switching to nonthreaded mode (until mainloop wakes up again)"
             self.worker_ident = None
         else:
             completed_ev.wait()
         return False
 
     def runcode(self):
         """Execute a code object.
 
         Multithreaded wrapper around IPython's runcode()."""
         
         global CODE_RUN
         
         # we are in worker thread, stash out the id for runsource() 
         self.worker_ident = thread.get_ident()
 
         if self._kill:
             print >>Term.cout, 'Closing threads...',
             Term.cout.flush()
             for tokill in self.on_kill:
                 tokill()
             print >>Term.cout, 'Done.'
             # allow kill() to return
             self._kill.set()
             return True
 
         # Install sigint handler.  We do it every time to ensure that if user
         # code modifies it, we restore our own handling.
         try:
             signal(SIGINT,sigint_handler)
         except SystemError:
             # This happens under Windows, which seems to have all sorts
             # of problems with signal handling.  Oh well...
             pass
 
         # Flush queue of pending code by calling the run methood of the parent
         # class with all items which may be in the queue.
         code_to_run = None
         while 1:
             try:
                 code_to_run, completed_ev, received_ev = self.code_queue.get_nowait()                
             except Queue.Empty:
                 break
             received_ev.set()
             
             # Exceptions need to be raised differently depending on which
             # thread is active.  This convoluted try/except is only there to
             # protect against asynchronous exceptions, to ensure that a KBINT
             # at the wrong time doesn't deadlock everything.  The global
             # CODE_TO_RUN is set to true/false as close as possible to the
             # runcode() call, so that the KBINT handler is correctly informed.
             try:
                 try:
                    CODE_RUN = True
                    InteractiveShell.runcode(self,code_to_run)
                 except KeyboardInterrupt:
                    print "Keyboard interrupted in mainloop"
                    while not self.code_queue.empty():
                       code, ev1,ev2 = self.code_queue.get_nowait()
                       ev1.set()
                       ev2.set()                      
                    break
             finally:
                 CODE_RUN = False
                 # allow runsource() return from wait
                 completed_ev.set()
                 
         
         # This MUST return true for gtk threading to work
         return True
 
     def kill(self):
         """Kill the thread, returning when it has been shut down."""
         self._kill = threading.Event()
         self._kill.wait()
 
 class MatplotlibShellBase:
     """Mixin class to provide the necessary modifications to regular IPython
     shell classes for matplotlib support.
 
     Given Python's MRO, this should be used as the FIRST class in the
     inheritance hierarchy, so that it overrides the relevant methods."""
     
     def _matplotlib_config(self,name,user_ns,user_global_ns=None):
         """Return items needed to setup the user's shell with matplotlib"""
 
         # Initialize matplotlib to interactive mode always
         import matplotlib
         from matplotlib import backends
         matplotlib.interactive(True)
 
         def use(arg):
             """IPython wrapper for matplotlib's backend switcher.
 
             In interactive use, we can not allow switching to a different
             interactive backend, since thread conflicts will most likely crash
             the python interpreter.  This routine does a safety check first,
             and refuses to perform a dangerous switch.  It still allows
             switching to non-interactive backends."""
 
             if arg in backends.interactive_bk and arg != self.mpl_backend:
                 m=('invalid matplotlib backend switch.\n'
                    'This script attempted to switch to the interactive '
                    'backend: `%s`\n'
                    'Your current choice of interactive backend is: `%s`\n\n'
                    'Switching interactive matplotlib backends at runtime\n'
                    'would crash the python interpreter, '
                    'and IPython has blocked it.\n\n'
                    'You need to either change your choice of matplotlib backend\n'
                    'by editing your .matplotlibrc file, or run this script as a \n'
                    'standalone file from the command line, not using IPython.\n' %
                    (arg,self.mpl_backend) )
                 raise RuntimeError, m
             else:
                 self.mpl_use(arg)
                 self.mpl_use._called = True
         
         self.matplotlib = matplotlib
         self.mpl_backend = matplotlib.rcParams['backend']
 
         # we also need to block switching of interactive backends by use()
         self.mpl_use = matplotlib.use
         self.mpl_use._called = False
         # overwrite the original matplotlib.use with our wrapper
         matplotlib.use = use
 
         # This must be imported last in the matplotlib series, after
         # backend/interactivity choices have been made
         import matplotlib.pylab as pylab
         self.pylab = pylab
 
         self.pylab.show._needmain = False
         # We need to detect at runtime whether show() is called by the user.
         # For this, we wrap it into a decorator which adds a 'called' flag.
         self.pylab.draw_if_interactive = flag_calls(self.pylab.draw_if_interactive)
 
         # Build a user namespace initialized with matplotlib/matlab features.
         user_ns, user_global_ns = IPython.ipapi.make_user_namespaces(user_ns,
             user_global_ns)
 
         # Import numpy as np/pyplot as plt are conventions we're trying to
         # somewhat standardize on.  Making them available to users by default
         # will greatly help this. 
         exec ("import numpy\n"
               "import numpy as np\n"
               "import matplotlib\n"
               "import matplotlib.pylab as pylab\n"
               "try:\n"
               "    import matplotlib.pyplot as plt\n"
               "except ImportError:\n"
               "    pass\n"
               ) in user_ns
         
         # Build matplotlib info banner
         b="""
   Welcome to pylab, a matplotlib-based Python environment.
   For more information, type 'help(pylab)'.
 """
         return user_ns,user_global_ns,b
 
     def mplot_exec(self,fname,*where,**kw):
         """Execute a matplotlib script.
 
         This is a call to execfile(), but wrapped in safeties to properly
         handle interactive rendering and backend switching."""
 
         #print '*** Matplotlib runner ***' # dbg
         # turn off rendering until end of script
         isInteractive = self.matplotlib.rcParams['interactive']
         self.matplotlib.interactive(False)
         self.safe_execfile(fname,*where,**kw)
         self.matplotlib.interactive(isInteractive)
         # make rendering call now, if the user tried to do it
         if self.pylab.draw_if_interactive.called:
             self.pylab.draw()
             self.pylab.draw_if_interactive.called = False
                 
         # if a backend switch was performed, reverse it now
         if self.mpl_use._called:
             self.matplotlib.rcParams['backend'] = self.mpl_backend
         
+    @testdec.skip_doctest
     def magic_run(self,parameter_s=''):
         Magic.magic_run(self,parameter_s,runner=self.mplot_exec)
 
     # Fix the docstring so users see the original as well
     magic_run.__doc__ = "%s\n%s" % (Magic.magic_run.__doc__,
                                     "\n        *** Modified %run for Matplotlib,"
                                     " with proper interactive handling ***")
 
 # Now we provide 2 versions of a matplotlib-aware IPython base shells, single
 # and multithreaded.  Note that these are meant for internal use, the IPShell*
 # classes below are the ones meant for public consumption.
 
 class MatplotlibShell(MatplotlibShellBase,InteractiveShell):
     """Single-threaded shell with matplotlib support."""
 
     def __init__(self,name,usage=None,rc=Struct(opts=None,args=None),
                  user_ns=None,user_global_ns=None,**kw):
         user_ns,user_global_ns,b2 = self._matplotlib_config(name,user_ns,user_global_ns)
         InteractiveShell.__init__(self,name,usage,rc,user_ns,user_global_ns,
                                   banner2=b2,**kw)
 
 class MatplotlibMTShell(MatplotlibShellBase,MTInteractiveShell):
     """Multi-threaded shell with matplotlib support."""
 
     def __init__(self,name,usage=None,rc=Struct(opts=None,args=None),
                  user_ns=None,user_global_ns=None, **kw):
         user_ns,user_global_ns,b2 = self._matplotlib_config(name,user_ns,user_global_ns)
         MTInteractiveShell.__init__(self,name,usage,rc,user_ns,user_global_ns,
                                     banner2=b2,**kw)
 
 #-----------------------------------------------------------------------------
 # Utility functions for the different GUI enabled IPShell* classes.
 
 def get_tk():
     """Tries to import Tkinter and returns a withdrawn Tkinter root
     window.  If Tkinter is already imported or not available, this
     returns None.  This function calls `hijack_tk` underneath.
     """
     if not USE_TK or sys.modules.has_key('Tkinter'):
         return None
     else:
         try:
             import Tkinter
         except ImportError:
             return None
         else:
             hijack_tk()
             r = Tkinter.Tk()
             r.withdraw()
             return r
 
 def hijack_tk():
     """Modifies Tkinter's mainloop with a dummy so when a module calls
     mainloop, it does not block.
 
     """
     def misc_mainloop(self, n=0):
         pass
     def tkinter_mainloop(n=0):
         pass
     
     import Tkinter
     Tkinter.Misc.mainloop = misc_mainloop
     Tkinter.mainloop = tkinter_mainloop
 
 def update_tk(tk):
     """Updates the Tkinter event loop.  This is typically called from
     the respective WX or GTK mainloops.
     """    
     if tk:
         tk.update()
 
 def hijack_wx():
     """Modifies wxPython's MainLoop with a dummy so user code does not
     block IPython.  The hijacked mainloop function is returned.
     """    
     def dummy_mainloop(*args, **kw):
         pass
 
     try:
         import wx
     except ImportError:
         # For very old versions of WX
         import wxPython as wx
         
     ver = wx.__version__
     orig_mainloop = None
     if ver[:3] >= '2.5':
         import wx
         if hasattr(wx, '_core_'): core = getattr(wx, '_core_')
         elif hasattr(wx, '_core'): core = getattr(wx, '_core')
         else: raise AttributeError('Could not find wx core module')
         orig_mainloop = core.PyApp_MainLoop
         core.PyApp_MainLoop = dummy_mainloop
     elif ver[:3] == '2.4':
         orig_mainloop = wx.wxc.wxPyApp_MainLoop
         wx.wxc.wxPyApp_MainLoop = dummy_mainloop
     else:
         warn("Unable to find either wxPython version 2.4 or >= 2.5.")
     return orig_mainloop
 
 def hijack_gtk():
     """Modifies pyGTK's mainloop with a dummy so user code does not
     block IPython.  This function returns the original `gtk.mainloop`
     function that has been hijacked.
     """    
     def dummy_mainloop(*args, **kw):
         pass
     import gtk
     if gtk.pygtk_version >= (2,4,0): orig_mainloop = gtk.main
     else:                            orig_mainloop = gtk.mainloop
     gtk.mainloop = dummy_mainloop
     gtk.main = dummy_mainloop
     return orig_mainloop
 
 def hijack_qt():
     """Modifies PyQt's mainloop with a dummy so user code does not 
     block IPython.  This function returns the original 
     `qt.qApp.exec_loop` function that has been hijacked.
     """    
     def dummy_mainloop(*args, **kw):
         pass
     import qt
     orig_mainloop = qt.qApp.exec_loop
     qt.qApp.exec_loop = dummy_mainloop
     qt.QApplication.exec_loop = dummy_mainloop
     return orig_mainloop
 
 def hijack_qt4():
     """Modifies PyQt4's mainloop with a dummy so user code does not
     block IPython.  This function returns the original 
     `QtGui.qApp.exec_` function that has been hijacked.
     """    
     def dummy_mainloop(*args, **kw):
         pass
     from PyQt4 import QtGui, QtCore
     orig_mainloop = QtGui.qApp.exec_
     QtGui.qApp.exec_ = dummy_mainloop
     QtGui.QApplication.exec_ = dummy_mainloop
     QtCore.QCoreApplication.exec_ = dummy_mainloop
     return orig_mainloop
 
 #-----------------------------------------------------------------------------
 # The IPShell* classes below are the ones meant to be run by external code as
 # IPython instances.  Note that unless a specific threading strategy is
 # desired, the factory function start() below should be used instead (it
 # selects the proper threaded class).
 
 class IPThread(threading.Thread):
     def run(self):
         self.IP.mainloop(self._banner)
         self.IP.kill()
 
 class IPShellGTK(IPThread):
     """Run a gtk mainloop() in a separate thread.
     
     Python commands can be passed to the thread where they will be executed.
     This is implemented by periodically checking for passed code using a
     GTK timeout callback."""
     
     TIMEOUT = 100 # Millisecond interval between timeouts.
 
     def __init__(self,argv=None,user_ns=None,user_global_ns=None,
                  debug=1,shell_class=MTInteractiveShell):
 
         import gtk
         
         self.gtk = gtk
         self.gtk_mainloop = hijack_gtk()
 
         # Allows us to use both Tk and GTK.
         self.tk = get_tk()
         
         if gtk.pygtk_version >= (2,4,0): mainquit = self.gtk.main_quit
         else:                            mainquit = self.gtk.mainquit
 
         self.IP = make_IPython(argv,user_ns=user_ns,
                                user_global_ns=user_global_ns,
                                debug=debug,
                                shell_class=shell_class,
                                on_kill=[mainquit])
 
         # HACK: slot for banner in self; it will be passed to the mainloop
         # method only and .run() needs it.  The actual value will be set by
         # .mainloop().
         self._banner = None 
 
         threading.Thread.__init__(self)
 
     def mainloop(self,sys_exit=0,banner=None):
 
         self._banner = banner
         
         if self.gtk.pygtk_version >= (2,4,0):
             import gobject
             gobject.idle_add(self.on_timer)
         else:
             self.gtk.idle_add(self.on_timer)
 
         if sys.platform != 'win32':
             try:
                 if self.gtk.gtk_version[0] >= 2:
                     self.gtk.gdk.threads_init()
             except AttributeError:
                 pass
             except RuntimeError:
                 error('Your pyGTK likely has not been compiled with '
                       'threading support.\n'
                       'The exception printout is below.\n'
                       'You can either rebuild pyGTK with threads, or '
                       'try using \n'
                       'matplotlib with a different backend (like Tk or WX).\n'
                       'Note that matplotlib will most likely not work in its '
                       'current state!')
                 self.IP.InteractiveTB()
 
         self.start()
         self.gtk.gdk.threads_enter()
         self.gtk_mainloop()
         self.gtk.gdk.threads_leave()
         self.join()
 
     def on_timer(self):
         """Called when GTK is idle.
 
         Must return True always, otherwise GTK stops calling it"""
         
         update_tk(self.tk)
         self.IP.runcode()
         time.sleep(0.01)
         return True
 
 
 class IPShellWX(IPThread):
     """Run a wx mainloop() in a separate thread.
     
     Python commands can be passed to the thread where they will be executed.
     This is implemented by periodically checking for passed code using a
     GTK timeout callback."""
     
     TIMEOUT = 100 # Millisecond interval between timeouts.
 
     def __init__(self,argv=None,user_ns=None,user_global_ns=None,
                  debug=1,shell_class=MTInteractiveShell):
 
         self.IP = make_IPython(argv,user_ns=user_ns,
                                user_global_ns=user_global_ns,
                                debug=debug,
                                shell_class=shell_class,
                                on_kill=[self.wxexit])
 
         wantedwxversion=self.IP.rc.wxversion
         if wantedwxversion!="0":
             try:
                 import wxversion
             except ImportError:
                 error('The wxversion module is needed for WX version selection')
             else:
                 try:
                     wxversion.select(wantedwxversion)
                 except:
                     self.IP.InteractiveTB()
                     error('Requested wxPython version %s could not be loaded' %
                                                                wantedwxversion)
 
         import wx
 
         threading.Thread.__init__(self)
         self.wx = wx
         self.wx_mainloop = hijack_wx()
 
         # Allows us to use both Tk and GTK.
         self.tk = get_tk()
         
         # HACK: slot for banner in self; it will be passed to the mainloop
         # method only and .run() needs it.  The actual value will be set by
         # .mainloop().
         self._banner = None 
 
         self.app = None
 
     def wxexit(self, *args):
         if self.app is not None:
             self.app.agent.timer.Stop()
             self.app.ExitMainLoop()
 
     def mainloop(self,sys_exit=0,banner=None):
 
         self._banner = banner
         
         self.start()
 
         class TimerAgent(self.wx.MiniFrame):
             wx = self.wx
             IP = self.IP
             tk = self.tk
             def __init__(self, parent, interval):
                 style = self.wx.DEFAULT_FRAME_STYLE | self.wx.TINY_CAPTION_HORIZ
                 self.wx.MiniFrame.__init__(self, parent, -1, ' ', pos=(200, 200),
                                              size=(100, 100),style=style)
                 self.Show(False)
                 self.interval = interval
                 self.timerId = self.wx.NewId()                                
 
             def StartWork(self):
                 self.timer = self.wx.Timer(self, self.timerId)
                 self.wx.EVT_TIMER(self,  self.timerId, self.OnTimer)
                 self.timer.Start(self.interval)
 
             def OnTimer(self, event):
                 update_tk(self.tk)
                 self.IP.runcode()
 
         class App(self.wx.App):
             wx = self.wx
             TIMEOUT = self.TIMEOUT
             def OnInit(self):
                 'Create the main window and insert the custom frame'
                 self.agent = TimerAgent(None, self.TIMEOUT)
                 self.agent.Show(False)
                 self.agent.StartWork()
                 return True
 
         self.app = App(redirect=False)
         self.wx_mainloop(self.app)
         self.join()
 
 
 class IPShellQt(IPThread):
     """Run a Qt event loop in a separate thread.
     
     Python commands can be passed to the thread where they will be executed.
     This is implemented by periodically checking for passed code using a
     Qt timer / slot."""
     
     TIMEOUT = 100 # Millisecond interval between timeouts.
 
     def __init__(self, argv=None, user_ns=None, user_global_ns=None,
                  debug=0, shell_class=MTInteractiveShell):
 
         import qt
 
         self.exec_loop = hijack_qt()
 
         # Allows us to use both Tk and QT.
         self.tk = get_tk()
 
         self.IP = make_IPython(argv,
                                user_ns=user_ns,
                                user_global_ns=user_global_ns,
                                debug=debug,
                                shell_class=shell_class,
                                on_kill=[qt.qApp.exit])
 
         # HACK: slot for banner in self; it will be passed to the mainloop
         # method only and .run() needs it.  The actual value will be set by
         # .mainloop().
         self._banner = None 
         
         threading.Thread.__init__(self)
 
     def mainloop(self, sys_exit=0, banner=None):
 
         import qt
 
         self._banner = banner
 
         if qt.QApplication.startingUp():
             a = qt.QApplication(sys.argv)
 
         self.timer = qt.QTimer()
         qt.QObject.connect(self.timer,
                            qt.SIGNAL('timeout()'),
                            self.on_timer)
 
         self.start()
         self.timer.start(self.TIMEOUT, True)
         while True:
             if self.IP._kill: break
             self.exec_loop()
         self.join()
 
     def on_timer(self):
         update_tk(self.tk)
         result = self.IP.runcode()
         self.timer.start(self.TIMEOUT, True)
         return result
 
 
 class IPShellQt4(IPThread):
     """Run a Qt event loop in a separate thread.
 
     Python commands can be passed to the thread where they will be executed.
     This is implemented by periodically checking for passed code using a
     Qt timer / slot."""
 
     TIMEOUT = 100 # Millisecond interval between timeouts.
 
     def __init__(self, argv=None, user_ns=None, user_global_ns=None,
                  debug=0, shell_class=MTInteractiveShell):
 
         from PyQt4 import QtCore, QtGui
 
         try:
             # present in PyQt4-4.2.1 or later
             QtCore.pyqtRemoveInputHook()
         except AttributeError:
             pass
 
         if QtCore.PYQT_VERSION_STR == '4.3':
             warn('''PyQt4 version 4.3 detected.
 If you experience repeated threading warnings, please update PyQt4.
 ''')
 
         self.exec_ = hijack_qt4()
 
         # Allows us to use both Tk and QT.
         self.tk = get_tk()
 
         self.IP = make_IPython(argv,
                                user_ns=user_ns,
                                user_global_ns=user_global_ns,
                                debug=debug,
                                shell_class=shell_class,
                                on_kill=[QtGui.qApp.exit])
 
         # HACK: slot for banner in self; it will be passed to the mainloop
         # method only and .run() needs it.  The actual value will be set by
         # .mainloop().
         self._banner = None
 
         threading.Thread.__init__(self)
 
     def mainloop(self, sys_exit=0, banner=None):
 
         from PyQt4 import QtCore, QtGui
 
         self._banner = banner
 
         if QtGui.QApplication.startingUp():
             a = QtGui.QApplication(sys.argv)
 
         self.timer = QtCore.QTimer()
         QtCore.QObject.connect(self.timer,
                                QtCore.SIGNAL('timeout()'),
                                self.on_timer)
 
         self.start()
         self.timer.start(self.TIMEOUT)
         while True:
             if self.IP._kill: break
             self.exec_()
         self.join()
 
     def on_timer(self):
         update_tk(self.tk)
         result = self.IP.runcode()
         self.timer.start(self.TIMEOUT)
         return result
 
 
 # A set of matplotlib public IPython shell classes, for single-threaded (Tk*
 # and FLTK*) and multithreaded (GTK*, WX* and Qt*) backends to use.
 def _load_pylab(user_ns):
     """Allow users to disable pulling all of pylab into the top-level
     namespace.
 
     This little utility must be called AFTER the actual ipython instance is
     running, since only then will the options file have been fully parsed."""
     
     ip = IPython.ipapi.get()
     if ip.options.pylab_import_all:
         ip.ex("from matplotlib.pylab import *")
         ip.IP.user_config_ns.update(ip.user_ns)
         
 
 class IPShellMatplotlib(IPShell):
     """Subclass IPShell with MatplotlibShell as the internal shell.
 
     Single-threaded class, meant for the Tk* and FLTK* backends.
 
     Having this on a separate class simplifies the external driver code."""
     
     def __init__(self,argv=None,user_ns=None,user_global_ns=None,debug=1):
         IPShell.__init__(self,argv,user_ns,user_global_ns,debug,
                          shell_class=MatplotlibShell)
         _load_pylab(self.IP.user_ns)
 
 class IPShellMatplotlibGTK(IPShellGTK):
     """Subclass IPShellGTK with MatplotlibMTShell as the internal shell.
 
     Multi-threaded class, meant for the GTK* backends."""
     
     def __init__(self,argv=None,user_ns=None,user_global_ns=None,debug=1):
         IPShellGTK.__init__(self,argv,user_ns,user_global_ns,debug,
                             shell_class=MatplotlibMTShell)
         _load_pylab(self.IP.user_ns)
 
 class IPShellMatplotlibWX(IPShellWX):
     """Subclass IPShellWX with MatplotlibMTShell as the internal shell.
 
     Multi-threaded class, meant for the WX* backends."""
     
     def __init__(self,argv=None,user_ns=None,user_global_ns=None,debug=1):
         IPShellWX.__init__(self,argv,user_ns,user_global_ns,debug,
                            shell_class=MatplotlibMTShell)
         _load_pylab(self.IP.user_ns)
 
 class IPShellMatplotlibQt(IPShellQt):
     """Subclass IPShellQt with MatplotlibMTShell as the internal shell.
 
     Multi-threaded class, meant for the Qt* backends."""
     
     def __init__(self,argv=None,user_ns=None,user_global_ns=None,debug=1):
         IPShellQt.__init__(self,argv,user_ns,user_global_ns,debug,
                            shell_class=MatplotlibMTShell)
         _load_pylab(self.IP.user_ns)
 
 class IPShellMatplotlibQt4(IPShellQt4):
     """Subclass IPShellQt4 with MatplotlibMTShell as the internal shell.
 
     Multi-threaded class, meant for the Qt4* backends."""
 
     def __init__(self,argv=None,user_ns=None,user_global_ns=None,debug=1):
         IPShellQt4.__init__(self,argv,user_ns,user_global_ns,debug,
                            shell_class=MatplotlibMTShell)
         _load_pylab(self.IP.user_ns)
 
 #-----------------------------------------------------------------------------
 # Factory functions to actually start the proper thread-aware shell
 
 def _select_shell(argv):
     """Select a shell from the given argv vector.
 
     This function implements the threading selection policy, allowing runtime
     control of the threading mode, both for general users and for matplotlib.
 
     Return:
       Shell class to be instantiated for runtime operation.
     """
     
     global USE_TK
 
     mpl_shell = {'gthread' : IPShellMatplotlibGTK,
                  'wthread' : IPShellMatplotlibWX,
                  'qthread' : IPShellMatplotlibQt,
                  'q4thread' : IPShellMatplotlibQt4,
                  'tkthread' : IPShellMatplotlib,  # Tk is built-in
                  }
 
     th_shell = {'gthread' : IPShellGTK,
                 'wthread' : IPShellWX,
                 'qthread' : IPShellQt,
                 'q4thread' : IPShellQt4,
                 'tkthread' : IPShell, # Tk is built-in
                 }
 
     backends = {'gthread' : 'GTKAgg',
                 'wthread' : 'WXAgg',
                 'qthread' : 'QtAgg',
                 'q4thread' :'Qt4Agg',
                 'tkthread' :'TkAgg',
                 }
 
     all_opts = set(['tk','pylab','gthread','qthread','q4thread','wthread',
                     'tkthread'])
     user_opts = set([s.replace('-','') for s in argv[:3]])
     special_opts = user_opts & all_opts            
 
     if 'tk' in special_opts:
         USE_TK = True
         special_opts.remove('tk')
 
     if 'pylab' in special_opts:
 
         try:
             import matplotlib
         except ImportError:
             error('matplotlib could NOT be imported!  Starting normal IPython.')
             return IPShell
         
         special_opts.remove('pylab')
         # If there's any option left, it means the user wants to force the
         # threading backend, else it's auto-selected from the rc file
         if special_opts:
             th_mode = special_opts.pop()
             matplotlib.rcParams['backend'] = backends[th_mode]
         else:
             backend = matplotlib.rcParams['backend']
             if backend.startswith('GTK'):
                 th_mode = 'gthread'
             elif backend.startswith('WX'):
                 th_mode = 'wthread'
             elif backend.startswith('Qt4'):
                 th_mode = 'q4thread'
             elif backend.startswith('Qt'):
                 th_mode = 'qthread'
             else:
                 # Any other backend, use plain Tk
                 th_mode = 'tkthread'
                 
         return mpl_shell[th_mode]
     else:
         # No pylab requested, just plain threads
         try:
             th_mode = special_opts.pop()
         except KeyError:
             th_mode = 'tkthread'
         return th_shell[th_mode]
 
 
 # This is the one which should be called by external code.
 def start(user_ns = None):
     """Return a running shell instance, dealing with threading options.
 
     This is a factory function which will instantiate the proper IPython shell
     based on the user's threading choice.  Such a selector is needed because
     different GUI toolkits require different thread handling details."""
 
     shell = _select_shell(sys.argv)
     return shell(user_ns = user_ns)
 
 # Some aliases for backwards compatibility
 IPythonShell = IPShell
 IPythonShellEmbed = IPShellEmbed
 #************************ End of file <Shell.py> ***************************

IPython/testing/plugin/ipdoctest.py

@@ -1,784 +1,806 b''
 """Nose Plugin that supports IPython doctests.
 
 Limitations:
 
 - When generating examples for use as doctests, make sure that you have
   pretty-printing OFF.  This can be done either by starting ipython with the
   flag '--nopprint', by setting pprint to 0 in your ipythonrc file, or by
   interactively disabling it with %Pprint.  This is required so that IPython
   output matches that of normal Python, which is used by doctest for internal
   execution.
 
 - Do not rely on specific prompt numbers for results (such as using
   '_34==True', for example).  For IPython tests run via an external process the
   prompt numbers may be different, and IPython tests run as normal python code
   won't even have these special _NN variables set at all.
 """
 
 
 #-----------------------------------------------------------------------------
 # Module imports
 
 # From the standard library
 import __builtin__
 import commands
 import doctest
 import inspect
 import logging
 import os
 import re
 import sys
 import traceback
 import unittest
 
 from inspect import getmodule
 from StringIO import StringIO
 
 # We are overriding the default doctest runner, so we need to import a few
 # things from doctest directly
 from doctest import (REPORTING_FLAGS, REPORT_ONLY_FIRST_FAILURE,
                      _unittest_reportflags, DocTestRunner,
                      _extract_future_flags, pdb, _OutputRedirectingPdb,
                      _exception_traceback,
                      linecache)
 
 # Third-party modules
 import nose.core
 
 from nose.plugins import doctests, Plugin
 from nose.util import anyp, getpackage, test_address, resolve_name, tolist
 
 #-----------------------------------------------------------------------------
 # Module globals and other constants
 
 log = logging.getLogger(__name__)
 
 ###########################################################################
 # *** HACK ***
 # We must start our own ipython object and heavily muck with it so that all the
 # modifications IPython makes to system behavior don't send the doctest
 # machinery into a fit.  This code should be considered a gross hack, but it
 # gets the job done.
 
 
 # Hack to modify the %run command so we can sync the user's namespace with the
 # test globals.  Once we move over to a clean magic system, this will be done
 # with much less ugliness.
 
 def _run_ns_sync(self,arg_s,runner=None):
     """Modified version of %run that syncs testing namespaces.
 
     This is strictly needed for running doctests that call %run.
     """
 
     out = _ip.IP.magic_run_ori(arg_s,runner)
     _run_ns_sync.test_globs.update(_ip.user_ns)
     return out
 
 
 class ipnsdict(dict):
     """A special subclass of dict for use as an IPython namespace in doctests.
 
     This subclass adds a simple checkpointing capability so that when testing
     machinery clears it (we use it as the test execution context), it doesn't
     get completely destroyed.
     """
     
     def __init__(self,*a):
         dict.__init__(self,*a)
         self._savedict = {}
         
     def clear(self):
         dict.clear(self)
         self.update(self._savedict)
 
     def _checkpoint(self):
         self._savedict.clear()
         self._savedict.update(self)
 
     def update(self,other):
         self._checkpoint()
         dict.update(self,other)
         # If '_' is in the namespace, python won't set it when executing code,
         # and we have examples that test it.  So we ensure that the namespace
         # is always 'clean' of it before it's used for test code execution.
         self.pop('_',None)
         
 
 def start_ipython():
     """Start a global IPython shell, which we need for IPython-specific syntax.
     """
     import new
 
     import IPython
 
     def xsys(cmd):
         """Execute a command and print its output.
 
         This is just a convenience function to replace the IPython system call
         with one that is more doctest-friendly.
         """
         cmd = _ip.IP.var_expand(cmd,depth=1)
         sys.stdout.write(commands.getoutput(cmd))
         sys.stdout.flush()
 
     # Store certain global objects that IPython modifies
     _displayhook = sys.displayhook
     _excepthook = sys.excepthook
     _main = sys.modules.get('__main__')
 
     # Start IPython instance.  We customize it to start with minimal frills.
     user_ns,global_ns = IPython.ipapi.make_user_namespaces(ipnsdict(),dict())
     
     IPython.Shell.IPShell(['--classic','--noterm_title'],
                           user_ns,global_ns)
 
     # Deactivate the various python system hooks added by ipython for
     # interactive convenience so we don't confuse the doctest system
     sys.modules['__main__'] = _main
     sys.displayhook = _displayhook
     sys.excepthook = _excepthook
 
     # So that ipython magics and aliases can be doctested (they work by making
     # a call into a global _ip object)
     _ip = IPython.ipapi.get()
     __builtin__._ip = _ip
 
     # Modify the IPython system call with one that uses getoutput, so that we
     # can capture subcommands and print them to Python's stdout, otherwise the
     # doctest machinery would miss them.
     _ip.system = xsys
 
     # Also patch our %run function in.
     im = new.instancemethod(_run_ns_sync,_ip.IP, _ip.IP.__class__)
     _ip.IP.magic_run_ori = _ip.IP.magic_run
     _ip.IP.magic_run = im
 
 # The start call MUST be made here.  I'm not sure yet why it doesn't work if
 # it is made later, at plugin initialization time, but in all my tests, that's
 # the case.
 start_ipython()
 
 # *** END HACK ***
 ###########################################################################
 
 # Classes and functions
 
 def is_extension_module(filename):
     """Return whether the given filename is an extension module.
 
     This simply checks that the extension is either .so or .pyd.
     """
     return os.path.splitext(filename)[1].lower() in ('.so','.pyd')
 
 
 class nodoc(object):
     def __init__(self,obj):
         self.obj = obj
 
     def __getattribute__(self,key):
         if key == '__doc__':
             return None
         else:
             return getattr(object.__getattribute__(self,'obj'),key)
 
 # Modified version of the one in the stdlib, that fixes a python bug (doctests
 # not found in extension modules, http://bugs.python.org/issue3158)
 class DocTestFinder(doctest.DocTestFinder):
 
     def _from_module(self, module, object):
         """
         Return true if the given object is defined in the given
         module.
         """
         if module is None:
             return True
         elif inspect.isfunction(object):
             return module.__dict__ is object.func_globals
         elif inspect.isbuiltin(object):
             return module.__name__ == object.__module__
         elif inspect.isclass(object):
             return module.__name__ == object.__module__
         elif inspect.ismethod(object):
             # This one may be a bug in cython that fails to correctly set the
             # __module__ attribute of methods, but since the same error is easy
             # to make by extension code writers, having this safety in place
             # isn't such a bad idea
             return module.__name__ == object.im_class.__module__
         elif inspect.getmodule(object) is not None:
             return module is inspect.getmodule(object)
         elif hasattr(object, '__module__'):
             return module.__name__ == object.__module__
         elif isinstance(object, property):
             return True # [XX] no way not be sure.
         else:
             raise ValueError("object must be a class or function")
 
     def _find(self, tests, obj, name, module, source_lines, globs, seen):
         """
         Find tests for the given object and any contained objects, and
         add them to `tests`.
         """
 
         if hasattr(obj,"skip_doctest"):
             #print 'SKIPPING DOCTEST FOR:',obj  # dbg
             obj = nodoc(obj)
 
         doctest.DocTestFinder._find(self,tests, obj, name, module,
                                     source_lines, globs, seen)
 
         # Below we re-run pieces of the above method with manual modifications,
         # because the original code is buggy and fails to correctly identify
         # doctests in extension modules.
 
         # Local shorthands
         from inspect import isroutine, isclass, ismodule
 
         # Look for tests in a module's contained objects.
         if inspect.ismodule(obj) and self._recurse:
             for valname, val in obj.__dict__.items():
                 valname1 = '%s.%s' % (name, valname)
                 if ( (isroutine(val) or isclass(val))
                      and self._from_module(module, val) ):
 
                     self._find(tests, val, valname1, module, source_lines,
                                globs, seen)
 
         # Look for tests in a class's contained objects.
         if inspect.isclass(obj) and self._recurse:
             #print 'RECURSE into class:',obj  # dbg
             for valname, val in obj.__dict__.items():
                 # Special handling for staticmethod/classmethod.
                 if isinstance(val, staticmethod):
                     val = getattr(obj, valname)
                 if isinstance(val, classmethod):
                     val = getattr(obj, valname).im_func
 
                 # Recurse to methods, properties, and nested classes.
                 if ((inspect.isfunction(val) or inspect.isclass(val) or
                      inspect.ismethod(val) or
                       isinstance(val, property)) and
                       self._from_module(module, val)):
                     valname = '%s.%s' % (name, valname)
                     self._find(tests, val, valname, module, source_lines,
                                globs, seen)
 
 
 class IPDoctestOutputChecker(doctest.OutputChecker):
     """Second-chance checker with support for random tests.
     
     If the default comparison doesn't pass, this checker looks in the expected
     output string for flags that tell us to ignore the output.
     """
 
     random_re = re.compile(r'#\s*random\s+')
     
     def check_output(self, want, got, optionflags):
         """Check output, accepting special markers embedded in the output.
 
         If the output didn't pass the default validation but the special string
         '#random' is included, we accept it."""
 
         # Let the original tester verify first, in case people have valid tests
         # that happen to have a comment saying '#random' embedded in.
         ret = doctest.OutputChecker.check_output(self, want, got,
                                                  optionflags)
         if not ret and self.random_re.search(want):
             #print >> sys.stderr, 'RANDOM OK:',want  # dbg
             return True
 
         return ret
 
 
 class DocTestCase(doctests.DocTestCase):
     """Proxy for DocTestCase: provides an address() method that
     returns the correct address for the doctest case. Otherwise
     acts as a proxy to the test case. To provide hints for address(),
     an obj may also be passed -- this will be used as the test object
     for purposes of determining the test address, if it is provided.
     """
 
     # Note: this method was taken from numpy's nosetester module.
 
     # Subclass nose.plugins.doctests.DocTestCase to work around a bug in
     # its constructor that blocks non-default arguments from being passed
     # down into doctest.DocTestCase
 
     def __init__(self, test, optionflags=0, setUp=None, tearDown=None,
                  checker=None, obj=None, result_var='_'):
         self._result_var = result_var
         doctests.DocTestCase.__init__(self, test,
                                       optionflags=optionflags,
                                       setUp=setUp, tearDown=tearDown,
                                       checker=checker)
         # Now we must actually copy the original constructor from the stdlib
         # doctest class, because we can't call it directly and a bug in nose
         # means it never gets passed the right arguments.
 
         self._dt_optionflags = optionflags
         self._dt_checker = checker
         self._dt_test = test
         self._dt_setUp = setUp
         self._dt_tearDown = tearDown
 
         # XXX - store this runner once in the object!
         runner = IPDocTestRunner(optionflags=optionflags,
                                  checker=checker, verbose=False)
         self._dt_runner = runner
 
 
         # Each doctest should remember what directory it was loaded from...
         self._ori_dir = os.getcwd()
 
     # Modified runTest from the default stdlib
     def runTest(self):
         test = self._dt_test
         runner = self._dt_runner
         
         old = sys.stdout
         new = StringIO()
         optionflags = self._dt_optionflags
 
         if not (optionflags & REPORTING_FLAGS):
             # The option flags don't include any reporting flags,
             # so add the default reporting flags
             optionflags |= _unittest_reportflags
 
         try:
             # Save our current directory and switch out to the one where the
             # test was originally created, in case another doctest did a
             # directory change.  We'll restore this in the finally clause.
             curdir = os.getcwd()
             os.chdir(self._ori_dir)
 
             runner.DIVIDER = "-"*70
             failures, tries = runner.run(test,out=new.write,
                                          clear_globs=False)
         finally:
             sys.stdout = old
             os.chdir(curdir)
 
         if failures:
             raise self.failureException(self.format_failure(new.getvalue()))
 
     def setUp(self):
         """Modified test setup that syncs with ipython namespace"""
         
         if isinstance(self._dt_test.examples[0],IPExample):
             # for IPython examples *only*, we swap the globals with the ipython
             # namespace, after updating it with the globals (which doctest
             # fills with the necessary info from the module being tested).
             _ip.IP.user_ns.update(self._dt_test.globs)
             self._dt_test.globs = _ip.IP.user_ns
 
         doctests.DocTestCase.setUp(self)
     
 
 
 # A simple subclassing of the original with a different class name, so we can
 # distinguish and treat differently IPython examples from pure python ones.
 class IPExample(doctest.Example): pass
 
 
 class IPExternalExample(doctest.Example):
     """Doctest examples to be run in an external process."""
 
     def __init__(self, source, want, exc_msg=None, lineno=0, indent=0,
                  options=None):
         # Parent constructor
         doctest.Example.__init__(self,source,want,exc_msg,lineno,indent,options)
 
         # An EXTRA newline is needed to prevent pexpect hangs
         self.source += '\n'
 
 
 class IPDocTestParser(doctest.DocTestParser):
     """
     A class used to parse strings containing doctest examples.
 
     Note: This is a version modified to properly recognize IPython input and
     convert any IPython examples into valid Python ones.
     """
     # This regular expression is used to find doctest examples in a
     # string.  It defines three groups: `source` is the source code
     # (including leading indentation and prompts); `indent` is the
     # indentation of the first (PS1) line of the source code; and
     # `want` is the expected output (including leading indentation).
 
     # Classic Python prompts or default IPython ones
     _PS1_PY = r'>>>'
     _PS2_PY = r'\.\.\.'
 
     _PS1_IP = r'In\ \[\d+\]:'
     _PS2_IP = r'\ \ \ \.\.\.+:'
 
     _RE_TPL = r'''
         # Source consists of a PS1 line followed by zero or more PS2 lines.
         (?P<source>
             (?:^(?P<indent> [ ]*) (?P<ps1> %s) .*)    # PS1 line
             (?:\n           [ ]*  (?P<ps2> %s) .*)*)  # PS2 lines
         \n? # a newline
         # Want consists of any non-blank lines that do not start with PS1.
         (?P<want> (?:(?![ ]*$)    # Not a blank line
                      (?![ ]*%s)   # Not a line starting with PS1
                      (?![ ]*%s)   # Not a line starting with PS2
                      .*$\n?       # But any other line
                   )*)
                   '''
 
     _EXAMPLE_RE_PY = re.compile( _RE_TPL % (_PS1_PY,_PS2_PY,_PS1_PY,_PS2_PY),
                                  re.MULTILINE | re.VERBOSE)
 
     _EXAMPLE_RE_IP = re.compile( _RE_TPL % (_PS1_IP,_PS2_IP,_PS1_IP,_PS2_IP),
                                  re.MULTILINE | re.VERBOSE)
 
     # Mark a test as being fully random.  In this case, we simply append the
     # random marker ('#random') to each individual example's output.  This way
     # we don't need to modify any other code.
     _RANDOM_TEST = re.compile(r'#\s*all-random\s+')
 
     # Mark tests to be executed in an external process - currently unsupported.
     _EXTERNAL_IP = re.compile(r'#\s*ipdoctest:\s*EXTERNAL')
     
     def ip2py(self,source):
         """Convert input IPython source into valid Python."""
         out = []
         newline = out.append
         for lnum,line in enumerate(source.splitlines()):
             newline(_ip.IP.prefilter(line,lnum>0))
         newline('')  # ensure a closing newline, needed by doctest
         #print "PYSRC:", '\n'.join(out)  # dbg
         return '\n'.join(out)
 
     def parse(self, string, name='<string>'):
         """
         Divide the given string into examples and intervening text,
         and return them as a list of alternating Examples and strings.
         Line numbers for the Examples are 0-based.  The optional
         argument `name` is a name identifying this string, and is only
         used for error messages.
         """
 
         #print 'Parse string:\n',string # dbg
 
         string = string.expandtabs()
         # If all lines begin with the same indentation, then strip it.
         min_indent = self._min_indent(string)
         if min_indent > 0:
             string = '\n'.join([l[min_indent:] for l in string.split('\n')])
 
         output = []
         charno, lineno = 0, 0
 
         # We make 'all random' tests by adding the '# random' mark to every
         # block of output in the test.
         if self._RANDOM_TEST.search(string):
             random_marker = '\n# random'
         else:
             random_marker = ''
 
         # Whether to convert the input from ipython to python syntax
         ip2py = False
         # Find all doctest examples in the string.  First, try them as Python
         # examples, then as IPython ones
         terms = list(self._EXAMPLE_RE_PY.finditer(string))
         if terms:
             # Normal Python example
             #print '-'*70  # dbg
             #print 'PyExample, Source:\n',string  # dbg
             #print '-'*70  # dbg
             Example = doctest.Example
         else:
             # It's an ipython example.  Note that IPExamples are run
             # in-process, so their syntax must be turned into valid python.
             # IPExternalExamples are run out-of-process (via pexpect) so they
             # don't need any filtering (a real ipython will be executing them).
             terms = list(self._EXAMPLE_RE_IP.finditer(string))
             if self._EXTERNAL_IP.search(string):
                 #print '-'*70  # dbg
                 #print 'IPExternalExample, Source:\n',string  # dbg
                 #print '-'*70  # dbg
                 Example = IPExternalExample
             else:
                 #print '-'*70  # dbg
                 #print 'IPExample, Source:\n',string  # dbg
                 #print '-'*70  # dbg
                 Example = IPExample
                 ip2py = True
 
         for m in terms:
             # Add the pre-example text to `output`.
             output.append(string[charno:m.start()])
             # Update lineno (lines before this example)
             lineno += string.count('\n', charno, m.start())
             # Extract info from the regexp match.
             (source, options, want, exc_msg) = \
                      self._parse_example(m, name, lineno,ip2py)
 
             # Append the random-output marker (it defaults to empty in most
             # cases, it's only non-empty for 'all-random' tests):
             want += random_marker
                 
             if Example is IPExternalExample:
                 options[doctest.NORMALIZE_WHITESPACE] = True
                 want += '\n'
 
             # Create an Example, and add it to the list.
             if not self._IS_BLANK_OR_COMMENT(source):
                 output.append(Example(source, want, exc_msg,
                                       lineno=lineno,
                                       indent=min_indent+len(m.group('indent')),
                                       options=options))
             # Update lineno (lines inside this example)
             lineno += string.count('\n', m.start(), m.end())
             # Update charno.
             charno = m.end()
         # Add any remaining post-example text to `output`.
         output.append(string[charno:])
         return output
 
     def _parse_example(self, m, name, lineno,ip2py=False):
         """
         Given a regular expression match from `_EXAMPLE_RE` (`m`),
         return a pair `(source, want)`, where `source` is the matched
         example's source code (with prompts and indentation stripped);
         and `want` is the example's expected output (with indentation
         stripped).
 
         `name` is the string's name, and `lineno` is the line number
         where the example starts; both are used for error messages.
 
         Optional:
         `ip2py`: if true, filter the input via IPython to convert the syntax
         into valid python.
         """
 
         # Get the example's indentation level.
         indent = len(m.group('indent'))
 
         # Divide source into lines; check that they're properly
         # indented; and then strip their indentation & prompts.
         source_lines = m.group('source').split('\n')
 
         # We're using variable-length input prompts
         ps1 = m.group('ps1')
         ps2 = m.group('ps2')
         ps1_len = len(ps1)
 
         self._check_prompt_blank(source_lines, indent, name, lineno,ps1_len)
         if ps2:
             self._check_prefix(source_lines[1:], ' '*indent + ps2, name, lineno)
 
         source = '\n'.join([sl[indent+ps1_len+1:] for sl in source_lines])
 
         if ip2py:
             # Convert source input from IPython into valid Python syntax
             source = self.ip2py(source)
 
         # Divide want into lines; check that it's properly indented; and
         # then strip the indentation.  Spaces before the last newline should
         # be preserved, so plain rstrip() isn't good enough.
         want = m.group('want')
         want_lines = want.split('\n')
         if len(want_lines) > 1 and re.match(r' *$', want_lines[-1]):
             del want_lines[-1]  # forget final newline & spaces after it
         self._check_prefix(want_lines, ' '*indent, name,
                            lineno + len(source_lines))
 
         # Remove ipython output prompt that might be present in the first line
         want_lines[0] = re.sub(r'Out\[\d+\]: \s*?\n?','',want_lines[0])
 
         want = '\n'.join([wl[indent:] for wl in want_lines])
 
         # If `want` contains a traceback message, then extract it.
         m = self._EXCEPTION_RE.match(want)
         if m:
             exc_msg = m.group('msg')
         else:
             exc_msg = None
 
         # Extract options from the source.
         options = self._find_options(source, name, lineno)
 
         return source, options, want, exc_msg
 
     def _check_prompt_blank(self, lines, indent, name, lineno, ps1_len):
         """
         Given the lines of a source string (including prompts and
         leading indentation), check to make sure that every prompt is
         followed by a space character.  If any line is not followed by
         a space character, then raise ValueError.
 
         Note: IPython-modified version which takes the input prompt length as a
         parameter, so that prompts of variable length can be dealt with.
         """
         space_idx = indent+ps1_len
         min_len = space_idx+1
         for i, line in enumerate(lines):
             if len(line) >=  min_len and line[space_idx] != ' ':
                 raise ValueError('line %r of the docstring for %s '
                                  'lacks blank after %s: %r' %
                                  (lineno+i+1, name,
                                   line[indent:space_idx], line))
 
 
 SKIP = doctest.register_optionflag('SKIP')
 
 
 class IPDocTestRunner(doctest.DocTestRunner,object):
     """Test runner that synchronizes the IPython namespace with test globals.
     """
     
     def run(self, test, compileflags=None, out=None, clear_globs=True):
 
         # Hack: ipython needs access to the execution context of the example,
         # so that it can propagate user variables loaded by %run into
         # test.globs.  We put them here into our modified %run as a function
         # attribute.  Our new %run will then only make the namespace update
         # when called (rather than unconconditionally updating test.globs here
         # for all examples, most of which won't be calling %run anyway).
         _run_ns_sync.test_globs = test.globs
 
         return super(IPDocTestRunner,self).run(test,
                                                compileflags,out,clear_globs)
 
 
 class DocFileCase(doctest.DocFileCase):
     """Overrides to provide filename
     """
     def address(self):
         return (self._dt_test.filename, None, None)
 
 
 class ExtensionDoctest(doctests.Doctest):
     """Nose Plugin that supports doctests in extension modules.
     """
     name = 'extdoctest'   # call nosetests with --with-extdoctest
     enabled = True
 
     def options(self, parser, env=os.environ):
         Plugin.options(self, parser, env)
+        parser.add_option('--doctest-tests', action='store_true',
+                          dest='doctest_tests',
+                          default=env.get('NOSE_DOCTEST_TESTS',True),
+                          help="Also look for doctests in test modules. "
+                          "Note that classes, methods and functions should "
+                          "have either doctests or non-doctest tests, "
+                          "not both. [NOSE_DOCTEST_TESTS]")
+        parser.add_option('--doctest-extension', action="append",
+                          dest="doctestExtension",
+                          help="Also look for doctests in files with "
+                          "this extension [NOSE_DOCTEST_EXTENSION]")
+        # Set the default as a list, if given in env; otherwise
+        # an additional value set on the command line will cause
+        # an error.
+        env_setting = env.get('NOSE_DOCTEST_EXTENSION')
+        if env_setting is not None:
+            parser.set_defaults(doctestExtension=tolist(env_setting))
+
 
     def configure(self, options, config):
         Plugin.configure(self, options, config)
         self.doctest_tests = options.doctest_tests
         self.extension = tolist(options.doctestExtension)
 
         self.parser = doctest.DocTestParser()
         self.finder = DocTestFinder()
         self.checker = IPDoctestOutputChecker()
         self.globs = None
         self.extraglobs = None
 
     def loadTestsFromExtensionModule(self,filename):
         bpath,mod = os.path.split(filename)
         modname = os.path.splitext(mod)[0]
         try:
             sys.path.append(bpath)
             module = __import__(modname)
             tests = list(self.loadTestsFromModule(module))
         finally:
             sys.path.pop()
         return tests
 
     # NOTE: the method below is almost a copy of the original one in nose, with
     # a  few modifications to control output checking.
 
     def loadTestsFromModule(self, module):
         #print 'lTM',module  # dbg
 
         if not self.matches(module.__name__):
             log.debug("Doctest doesn't want module %s", module)
             return
 
         tests = self.finder.find(module,globs=self.globs,
                                  extraglobs=self.extraglobs)
         if not tests:
             return
 
         # always use whitespace and ellipsis options
         optionflags = doctest.NORMALIZE_WHITESPACE | doctest.ELLIPSIS
 
         tests.sort()
         module_file = module.__file__
         if module_file[-4:] in ('.pyc', '.pyo'):
             module_file = module_file[:-1]
         for test in tests:
             if not test.examples:
                 continue
             if not test.filename:
                 test.filename = module_file
             
             yield DocTestCase(test,
                               optionflags=optionflags,
                               checker=self.checker)
 
 
     def loadTestsFromFile(self, filename):
         #print 'lTF',filename  # dbg
 
         if is_extension_module(filename):
             for t in self.loadTestsFromExtensionModule(filename):
                 yield t
         else:
             if self.extension and anyp(filename.endswith, self.extension):
                 name = os.path.basename(filename)
                 dh = open(filename)
                 try:
                     doc = dh.read()
                 finally:
                     dh.close()
                 test = self.parser.get_doctest(
                     doc, globs={'__file__': filename}, name=name,
                     filename=filename, lineno=0)
                 if test.examples:
                     #print 'FileCase:',test.examples  # dbg
                     yield DocFileCase(test)
                 else:
                     yield False # no tests to load
 
     def wantFile(self,filename):
         """Return whether the given filename should be scanned for tests.
 
         Modified version that accepts extension modules as valid containers for
         doctests.
         """
-        #print 'Filename:',filename  # dbg
+        print 'Filename:',filename  # dbg
 
         # XXX - temporarily hardcoded list, will move to driver later
         exclude = ['IPython/external/',
-                   'IPython/Extensions/ipy_',
                    'IPython/platutils_win32',
                    'IPython/frontend/cocoa',
                    'IPython_doctest_plugin',
                    'IPython/Gnuplot',
-                   'IPython/Extensions/PhysicalQIn']
+                   'IPython/Extensions/ipy_',
+                   'IPython/Extensions/PhysicalQIn',
+                   'IPython/Extensions/scitedirector',
+                   'IPython/testing/plugin',
+                   ]
 
         for fex in exclude:
             if fex in filename:  # substring
                 #print '###>>> SKIP:',filename  # dbg
                 return False
 
         if is_extension_module(filename):
             return True
         else:
             return doctests.Doctest.wantFile(self,filename)
 
 
 class IPythonDoctest(ExtensionDoctest):
     """Nose Plugin that supports doctests in extension modules.
     """
     name = 'ipdoctest'   # call nosetests with --with-ipdoctest
     enabled = True
 
     def configure(self, options, config):
 
         Plugin.configure(self, options, config)
         self.doctest_tests = options.doctest_tests
         self.extension = tolist(options.doctestExtension)
 
         self.parser = IPDocTestParser()
         self.finder = DocTestFinder(parser=self.parser)
         self.checker = IPDoctestOutputChecker()
         self.globs = None
         self.extraglobs = None
+

IPython/testing/tutils.py

@@ -1,86 +1,87 b''
 """DEPRECATED - use IPython.testing.util instead.
 
 Utilities for testing code.
 """
 
 #############################################################################
 
 # This was old testing code we never really used in IPython.  The pieces of
 # testing machinery from snakeoil that were good have already been merged into
 # the nose plugin, so this can be taken away soon.  Leave a warning for now,
 # we'll remove it in a later release (around 0.10 or so).
+
 from warnings import warn
 warn('This will be removed soon.  Use IPython.testing.util instead',
      DeprecationWarning)
 
 #############################################################################
 
 # Required modules and packages
 
 # Standard Python lib
 import os
 import sys
 
 # From this project
 from IPython.tools import utils
 
 # path to our own installation, so we can find source files under this.
 TEST_PATH = os.path.dirname(os.path.abspath(__file__))
 
 # Global flag, used by vprint
 VERBOSE = '-v' in sys.argv or '--verbose' in sys.argv
 
 ##########################################################################
 # Code begins
 
 # Some utility functions
 def vprint(*args):
     """Print-like function which relies on a global VERBOSE flag."""
     if not VERBOSE:
         return
 
     write = sys.stdout.write
     for item in args:
         write(str(item))
     write('\n')
     sys.stdout.flush()
 
 def test_path(path):
     """Return a path as a subdir of the test package.
 
     This finds the correct path of the test package on disk, and prepends it
     to the input path."""
 
     return os.path.join(TEST_PATH,path)
 
 def fullPath(startPath,files):
     """Make full paths for all the listed files, based on startPath.
 
     Only the base part of startPath is kept, since this routine is typically
     used with a script's __file__ variable as startPath.  The base of startPath
     is then prepended to all the listed files, forming the output list.
 
     :Parameters:
       startPath : string
         Initial path to use as the base for the results.  This path is split
       using os.path.split() and only its first component is kept.
 
       files : string or list
         One or more files.
 
     :Examples:
 
     >>> fullPath('/foo/bar.py',['a.txt','b.txt'])
     ['/foo/a.txt', '/foo/b.txt']
 
     >>> fullPath('/foo',['a.txt','b.txt'])
     ['/a.txt', '/b.txt']
 
     If a single file is given, the output is still a list:
     >>> fullPath('/foo','a.txt')
     ['/a.txt']
     """
 
     files = utils.list_strings(files)
     base = os.path.split(startPath)[0]
     return [ os.path.join(base,f) for f in files ]

IPython/ultraTB.py

@@ -1,1061 +1,1062 b''
 # -*- coding: utf-8 -*-
 """
 ultraTB.py -- Spice up your tracebacks!
 
 * ColorTB
 I've always found it a bit hard to visually parse tracebacks in Python.  The
 ColorTB class is a solution to that problem.  It colors the different parts of a
 traceback in a manner similar to what you would expect from a syntax-highlighting
 text editor.
 
 Installation instructions for ColorTB:
     import sys,ultraTB
     sys.excepthook = ultraTB.ColorTB()
 
 * VerboseTB  
 I've also included a port of Ka-Ping Yee's "cgitb.py" that produces all kinds
 of useful info when a traceback occurs.  Ping originally had it spit out HTML
 and intended it for CGI programmers, but why should they have all the fun?  I
 altered it to spit out colored text to the terminal.  It's a bit overwhelming,
 but kind of neat, and maybe useful for long-running programs that you believe
 are bug-free.  If a crash *does* occur in that type of program you want details.
 Give it a shot--you'll love it or you'll hate it.
 
 Note:
 
   The Verbose mode prints the variables currently visible where the exception
   happened (shortening their strings if too long). This can potentially be
   very slow, if you happen to have a huge data structure whose string
   representation is complex to compute. Your computer may appear to freeze for
   a while with cpu usage at 100%. If this occurs, you can cancel the traceback
   with Ctrl-C (maybe hitting it more than once).
 
   If you encounter this kind of situation often, you may want to use the
   Verbose_novars mode instead of the regular Verbose, which avoids formatting
   variables (but otherwise includes the information and context given by
   Verbose).
   
 
 Installation instructions for ColorTB:
     import sys,ultraTB
     sys.excepthook = ultraTB.VerboseTB()
 
 Note:  Much of the code in this module was lifted verbatim from the standard
 library module 'traceback.py' and Ka-Ping Yee's 'cgitb.py'.
 
 * Color schemes
 The colors are defined in the class TBTools through the use of the
 ColorSchemeTable class. Currently the following exist:
 
   - NoColor: allows all of this module to be used in any terminal (the color
   escapes are just dummy blank strings).
 
   - Linux: is meant to look good in a terminal like the Linux console (black
   or very dark background).
 
   - LightBG: similar to Linux but swaps dark/light colors to be more readable
   in light background terminals.
 
 You can implement other color schemes easily, the syntax is fairly
 self-explanatory. Please send back new schemes you develop to the author for
 possible inclusion in future releases.
 
 $Id: ultraTB.py 2908 2007-12-30 21:07:46Z vivainio $"""
 
 #*****************************************************************************
 #       Copyright (C) 2001 Nathaniel Gray <[e-mail]>
 #       Copyright (C) 2001-2004 Fernando Perez <[e-mail]>
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #*****************************************************************************
 
 from IPython import Release
 __author__  = '%s <%s>\n%s <%s>' % (Release.authors['Nathan']+
                                     Release.authors['Fernando'])
 __license__ = Release.license
 
 # Required modules
 import inspect
 import keyword
 import linecache
 import os
 import pydoc
 import re
 import string
 import sys
 import time
 import tokenize
 import traceback
 import types
 
 # For purposes of monkeypatching inspect to fix a bug in it.
 from inspect import getsourcefile, getfile, getmodule,\
      ismodule,  isclass, ismethod, isfunction, istraceback, isframe, iscode
 
 
 # IPython's own modules
 # Modified pdb which doesn't damage IPython's readline handling
 from IPython import Debugger, PyColorize
 from IPython.ipstruct import Struct
 from IPython.excolors import ExceptionColors
 from IPython.genutils import Term,uniq_stable,error,info
 
 # Globals
 # amount of space to put line numbers before verbose tracebacks
 INDENT_SIZE = 8
 
 # Default color scheme.  This is used, for example, by the traceback
 # formatter.  When running in an actual IPython instance, the user's rc.colors
 # value is used, but havinga module global makes this functionality available
 # to users of ultraTB who are NOT running inside ipython.
 DEFAULT_SCHEME = 'NoColor'
 
 #---------------------------------------------------------------------------
 # Code begins
 
 # Utility functions
 def inspect_error():
     """Print a message about internal inspect errors.
 
     These are unfortunately quite common."""
     
     error('Internal Python error in the inspect module.\n'
           'Below is the traceback from this internal error.\n')
 
 
 def findsource(object):
     """Return the entire source file and starting line number for an object.
 
     The argument may be a module, class, method, function, traceback, frame,
     or code object.  The source code is returned as a list of all the lines
     in the file and the line number indexes a line in that list.  An IOError
     is raised if the source code cannot be retrieved.
 
     FIXED version with which we monkeypatch the stdlib to work around a bug."""
 
     file = getsourcefile(object) or getfile(object)
     # If the object is a frame, then trying to get the globals dict from its
     # module won't work. Instead, the frame object itself has the globals
     # dictionary.
     globals_dict = None
     if inspect.isframe(object):
         # XXX: can this ever be false?
         globals_dict = object.f_globals
     else:
         module = getmodule(object, file)
         if module:
             globals_dict = module.__dict__
     lines = linecache.getlines(file, globals_dict)
     if not lines:
         raise IOError('could not get source code')
 
     if ismodule(object):
         return lines, 0
 
     if isclass(object):
         name = object.__name__
         pat = re.compile(r'^(\s*)class\s*' + name + r'\b')
         # make some effort to find the best matching class definition:
         # use the one with the least indentation, which is the one
         # that's most probably not inside a function definition.
         candidates = []
         for i in range(len(lines)):
             match = pat.match(lines[i])
             if match:
                 # if it's at toplevel, it's already the best one
                 if lines[i][0] == 'c':
                     return lines, i
                 # else add whitespace to candidate list
                 candidates.append((match.group(1), i))
         if candidates:
             # this will sort by whitespace, and by line number,
             # less whitespace first
             candidates.sort()
             return lines, candidates[0][1]
         else:
             raise IOError('could not find class definition')
 
     if ismethod(object):
         object = object.im_func
     if isfunction(object):
         object = object.func_code
     if istraceback(object):
         object = object.tb_frame
     if isframe(object):
         object = object.f_code
     if iscode(object):
         if not hasattr(object, 'co_firstlineno'):
             raise IOError('could not find function definition')
         pat = re.compile(r'^(\s*def\s)|(.*(?<!\w)lambda(:|\s))|^(\s*@)')
         pmatch = pat.match
         # fperez - fix: sometimes, co_firstlineno can give a number larger than
         # the length of lines, which causes an error.  Safeguard against that.
         lnum = min(object.co_firstlineno,len(lines))-1
         while lnum > 0:
             if pmatch(lines[lnum]): break
             lnum -= 1
  
         return lines, lnum
     raise IOError('could not find code object')
 
 # Monkeypatch inspect to apply our bugfix.  This code only works with py25
 if sys.version_info[:2] >= (2,5):
     inspect.findsource = findsource
 
 def fix_frame_records_filenames(records):
     """Try to fix the filenames in each record from inspect.getinnerframes().
 
     Particularly, modules loaded from within zip files have useless filenames
     attached to their code object, and inspect.getinnerframes() just uses it.
     """
     fixed_records = []
     for frame, filename, line_no, func_name, lines, index in records:
         # Look inside the frame's globals dictionary for __file__, which should
         # be better.
         better_fn = frame.f_globals.get('__file__', None)
         if isinstance(better_fn, str):
             # Check the type just in case someone did something weird with
             # __file__. It might also be None if the error occurred during
             # import.
             filename = better_fn
         fixed_records.append((frame, filename, line_no, func_name, lines, index))
     return fixed_records
 
 
 def _fixed_getinnerframes(etb, context=1,tb_offset=0):
     import linecache
     LNUM_POS, LINES_POS, INDEX_POS =  2, 4, 5
 
     records  = fix_frame_records_filenames(inspect.getinnerframes(etb, context))
 
     # If the error is at the console, don't build any context, since it would
     # otherwise produce 5 blank lines printed out (there is no file at the
     # console)
     rec_check = records[tb_offset:]
     try:
         rname = rec_check[0][1]
         if rname == '<ipython console>' or rname.endswith('<string>'):
             return rec_check
     except IndexError:
         pass
 
     aux = traceback.extract_tb(etb)
     assert len(records) == len(aux)
     for i, (file, lnum, _, _) in zip(range(len(records)), aux):
         maybeStart = lnum-1 - context//2
         start =  max(maybeStart, 0)
         end   = start + context
         lines = linecache.getlines(file)[start:end]
         # pad with empty lines if necessary
         if maybeStart < 0:
             lines = (['\n'] * -maybeStart) + lines
         if len(lines) < context:
             lines += ['\n'] * (context - len(lines))
         buf = list(records[i])
         buf[LNUM_POS] = lnum
         buf[INDEX_POS] = lnum - 1 - start
         buf[LINES_POS] = lines
         records[i] = tuple(buf)
     return records[tb_offset:]
 
 # Helper function -- largely belongs to VerboseTB, but we need the same
 # functionality to produce a pseudo verbose TB for SyntaxErrors, so that they
 # can be recognized properly by ipython.el's py-traceback-line-re
 # (SyntaxErrors have to be treated specially because they have no traceback)
 
 _parser = PyColorize.Parser()
     
 def _formatTracebackLines(lnum, index, lines, Colors, lvals=None,scheme=None):
     numbers_width = INDENT_SIZE - 1
     res = []
     i = lnum - index
 
     # This lets us get fully syntax-highlighted tracebacks.
     if scheme is None:
         try:
             scheme = __IPYTHON__.rc.colors
         except:
             scheme = DEFAULT_SCHEME
     _line_format = _parser.format2
 
     for line in lines:
         new_line, err = _line_format(line,'str',scheme)
         if not err: line = new_line
         
         if i == lnum:
             # This is the line with the error
             pad = numbers_width - len(str(i))
             if pad >= 3:
                 marker = '-'*(pad-3) + '-> '
             elif pad == 2:
                 marker = '> '
             elif pad == 1:
                 marker = '>'
             else:
                 marker = ''
             num = marker + str(i)
             line = '%s%s%s %s%s' %(Colors.linenoEm, num, 
                                    Colors.line, line, Colors.Normal)
         else:
             num = '%*s' % (numbers_width,i)
             line = '%s%s%s %s' %(Colors.lineno, num, 
                                  Colors.Normal, line)
 
         res.append(line)
         if lvals and i == lnum:
             res.append(lvals + '\n')
         i = i + 1
     return res
 
 
 #---------------------------------------------------------------------------
 # Module classes
 class TBTools:
     """Basic tools used by all traceback printer classes."""
 
     def __init__(self,color_scheme = 'NoColor',call_pdb=False):
         # Whether to call the interactive pdb debugger after printing
         # tracebacks or not
         self.call_pdb = call_pdb
 
         # Create color table
         self.color_scheme_table = ExceptionColors 
 
         self.set_colors(color_scheme)
         self.old_scheme = color_scheme  # save initial value for toggles
 
         if call_pdb:
             self.pdb = Debugger.Pdb(self.color_scheme_table.active_scheme_name)
         else:
             self.pdb = None
 
     def set_colors(self,*args,**kw):
         """Shorthand access to the color table scheme selector method."""
 
         # Set own color table
         self.color_scheme_table.set_active_scheme(*args,**kw)
         # for convenience, set Colors to the active scheme
         self.Colors = self.color_scheme_table.active_colors
         # Also set colors of debugger
         if hasattr(self,'pdb') and self.pdb is not None:
             self.pdb.set_colors(*args,**kw)
 
     def color_toggle(self):
         """Toggle between the currently active color scheme and NoColor."""
         
         if self.color_scheme_table.active_scheme_name == 'NoColor':
             self.color_scheme_table.set_active_scheme(self.old_scheme)
             self.Colors = self.color_scheme_table.active_colors
         else:
             self.old_scheme = self.color_scheme_table.active_scheme_name
             self.color_scheme_table.set_active_scheme('NoColor')
             self.Colors = self.color_scheme_table.active_colors
 
 #---------------------------------------------------------------------------
 class ListTB(TBTools):
     """Print traceback information from a traceback list, with optional color.
         
     Calling: requires 3 arguments:
       (etype, evalue, elist)
     as would be obtained by:
       etype, evalue, tb = sys.exc_info()
       if tb:
         elist = traceback.extract_tb(tb)
       else:
         elist = None
 
     It can thus be used by programs which need to process the traceback before
     printing (such as console replacements based on the code module from the
     standard library).
 
     Because they are meant to be called without a full traceback (only a
     list), instances of this class can't call the interactive pdb debugger."""
 
     def __init__(self,color_scheme = 'NoColor'):
         TBTools.__init__(self,color_scheme = color_scheme,call_pdb=0)
         
     def __call__(self, etype, value, elist):
         Term.cout.flush()
         print >> Term.cerr, self.text(etype,value,elist)
         Term.cerr.flush()
 
     def text(self,etype, value, elist,context=5):
         """Return a color formatted string with the traceback info."""
 
         Colors = self.Colors
         out_string = ['%s%s%s\n' % (Colors.topline,'-'*60,Colors.Normal)]
         if elist:
             out_string.append('Traceback %s(most recent call last)%s:' % \
                                 (Colors.normalEm, Colors.Normal) + '\n')
             out_string.extend(self._format_list(elist))
         lines = self._format_exception_only(etype, value)
         for line in lines[:-1]:
             out_string.append(" "+line)
         out_string.append(lines[-1])
         return ''.join(out_string)
 
     def _format_list(self, extracted_list):
         """Format a list of traceback entry tuples for printing.
 
         Given a list of tuples as returned by extract_tb() or
         extract_stack(), return a list of strings ready for printing.
         Each string in the resulting list corresponds to the item with the
         same index in the argument list.  Each string ends in a newline;
         the strings may contain internal newlines as well, for those items
         whose source text line is not None.
         
         Lifted almost verbatim from traceback.py
         """
 
         Colors = self.Colors
         list = []
         for filename, lineno, name, line in extracted_list[:-1]:
             item = '  File %s"%s"%s, line %s%d%s, in %s%s%s\n' % \
                     (Colors.filename, filename, Colors.Normal, 
                      Colors.lineno, lineno, Colors.Normal,
                      Colors.name, name, Colors.Normal)
             if line:
                 item = item + '    %s\n' % line.strip()
             list.append(item)
         # Emphasize the last entry
         filename, lineno, name, line = extracted_list[-1]
         item = '%s  File %s"%s"%s, line %s%d%s, in %s%s%s%s\n' % \
                 (Colors.normalEm,
                  Colors.filenameEm, filename, Colors.normalEm,
                  Colors.linenoEm, lineno, Colors.normalEm,
                  Colors.nameEm, name, Colors.normalEm,
                  Colors.Normal)
         if line:
             item = item + '%s    %s%s\n' % (Colors.line, line.strip(),
                                             Colors.Normal)
         list.append(item)
         return list
         
     def _format_exception_only(self, etype, value):
         """Format the exception part of a traceback.
 
         The arguments are the exception type and value such as given by
         sys.exc_info()[:2]. The return value is a list of strings, each ending
         in a newline.  Normally, the list contains a single string; however,
         for SyntaxError exceptions, it contains several lines that (when
         printed) display detailed information about where the syntax error
         occurred.  The message indicating which exception occurred is the
         always last string in the list.
         
         Also lifted nearly verbatim from traceback.py
         """
         
+        have_filedata = False
         Colors = self.Colors
         list = []
         try:
             stype = Colors.excName + etype.__name__ + Colors.Normal
         except AttributeError:
             stype = etype  # String exceptions don't get special coloring
         if value is None:
             list.append( str(stype) + '\n')
         else:
             if etype is SyntaxError:
                 try:
                     msg, (filename, lineno, offset, line) = value
                 except:
                     have_filedata = False
                 else:
                     have_filedata = True
                     #print 'filename is',filename  # dbg
                     if not filename: filename = "<string>"
                     list.append('%s  File %s"%s"%s, line %s%d%s\n' % \
                             (Colors.normalEm,
                              Colors.filenameEm, filename, Colors.normalEm,
                              Colors.linenoEm, lineno, Colors.Normal  ))
                     if line is not None:
                         i = 0
                         while i < len(line) and line[i].isspace():
                             i = i+1
                         list.append('%s    %s%s\n' % (Colors.line,
                                                       line.strip(), 
                                                       Colors.Normal))
                         if offset is not None:
                             s = '    '
                             for c in line[i:offset-1]:
                                 if c.isspace():
                                     s = s + c
                                 else:
                                     s = s + ' '
                             list.append('%s%s^%s\n' % (Colors.caret, s,
                                                        Colors.Normal) )
                         value = msg
             s = self._some_str(value)
             if s:
                 list.append('%s%s:%s %s\n' % (str(stype), Colors.excName,
                                               Colors.Normal, s))
             else:
                 list.append('%s\n' % str(stype))
 
         # vds:>>
         if have_filedata:
             __IPYTHON__.hooks.synchronize_with_editor(filename, lineno, 0)
         # vds:<<
 
         return list
 
     def _some_str(self, value):
         # Lifted from traceback.py
         try:
             return str(value)
         except:
             return '<unprintable %s object>' % type(value).__name__
 
 #----------------------------------------------------------------------------
 class VerboseTB(TBTools):
     """A port of Ka-Ping Yee's cgitb.py module that outputs color text instead
     of HTML.  Requires inspect and pydoc.  Crazy, man.
 
     Modified version which optionally strips the topmost entries from the
     traceback, to be used with alternate interpreters (because their own code
     would appear in the traceback)."""
 
     def __init__(self,color_scheme = 'Linux',tb_offset=0,long_header=0,
                  call_pdb = 0, include_vars=1):
         """Specify traceback offset, headers and color scheme.
 
         Define how many frames to drop from the tracebacks. Calling it with
         tb_offset=1 allows use of this handler in interpreters which will have
         their own code at the top of the traceback (VerboseTB will first
         remove that frame before printing the traceback info)."""
         TBTools.__init__(self,color_scheme=color_scheme,call_pdb=call_pdb)
         self.tb_offset = tb_offset
         self.long_header = long_header
         self.include_vars = include_vars
 
     def text(self, etype, evalue, etb, context=5):
         """Return a nice text document describing the traceback."""
 
         # some locals
         try:
             etype = etype.__name__
         except AttributeError:
             pass
         Colors        = self.Colors   # just a shorthand + quicker name lookup
         ColorsNormal  = Colors.Normal  # used a lot
         col_scheme    = self.color_scheme_table.active_scheme_name
         indent        = ' '*INDENT_SIZE
         em_normal     = '%s\n%s%s' % (Colors.valEm, indent,ColorsNormal)
         undefined     = '%sundefined%s' % (Colors.em, ColorsNormal)
         exc = '%s%s%s' % (Colors.excName,etype,ColorsNormal)
 
         # some internal-use functions
         def text_repr(value):
             """Hopefully pretty robust repr equivalent."""
             # this is pretty horrible but should always return *something*
             try:
                 return pydoc.text.repr(value)
             except KeyboardInterrupt:
                 raise
             except:
                 try:
                     return repr(value)
                 except KeyboardInterrupt:
                     raise
                 except:
                     try:
                         # all still in an except block so we catch
                         # getattr raising
                         name = getattr(value, '__name__', None)
                         if name:
                             # ick, recursion
                             return text_repr(name)
                         klass = getattr(value, '__class__', None)
                         if klass:
                             return '%s instance' % text_repr(klass)
                     except KeyboardInterrupt:
                         raise
                     except:
                         return 'UNRECOVERABLE REPR FAILURE'
         def eqrepr(value, repr=text_repr): return '=%s' % repr(value)
         def nullrepr(value, repr=text_repr): return ''
 
         # meat of the code begins
         try:
             etype = etype.__name__
         except AttributeError:
             pass
 
         if self.long_header:
             # Header with the exception type, python version, and date
             pyver = 'Python ' + string.split(sys.version)[0] + ': ' + sys.executable
             date = time.ctime(time.time())
             
             head = '%s%s%s\n%s%s%s\n%s' % (Colors.topline, '-'*75, ColorsNormal,
                                            exc, ' '*(75-len(str(etype))-len(pyver)),
                                            pyver, string.rjust(date, 75) )
             head += "\nA problem occured executing Python code.  Here is the sequence of function"\
                     "\ncalls leading up to the error, with the most recent (innermost) call last."
         else:
             # Simplified header
             head = '%s%s%s\n%s%s' % (Colors.topline, '-'*75, ColorsNormal,exc,
                                      string.rjust('Traceback (most recent call last)',
                                                   75 - len(str(etype)) ) )
         frames = []
         # Flush cache before calling inspect.  This helps alleviate some of the
         # problems with python 2.3's inspect.py.
         linecache.checkcache()
         # Drop topmost frames if requested
         try:
             # Try the default getinnerframes and Alex's: Alex's fixes some
             # problems, but it generates empty tracebacks for console errors
             # (5 blanks lines) where none should be returned.
             #records = inspect.getinnerframes(etb, context)[self.tb_offset:]
             #print 'python records:', records # dbg
             records = _fixed_getinnerframes(etb, context,self.tb_offset)
             #print 'alex   records:', records # dbg
         except:
 
             # FIXME: I've been getting many crash reports from python 2.3
             # users, traceable to inspect.py.  If I can find a small test-case
             # to reproduce this, I should either write a better workaround or
             # file a bug report against inspect (if that's the real problem).
             # So far, I haven't been able to find an isolated example to
             # reproduce the problem.
             inspect_error()
             traceback.print_exc(file=Term.cerr)
             info('\nUnfortunately, your original traceback can not be constructed.\n')
             return ''
 
         # build some color string templates outside these nested loops
         tpl_link       = '%s%%s%s' % (Colors.filenameEm,ColorsNormal)
         tpl_call       = 'in %s%%s%s%%s%s' % (Colors.vName, Colors.valEm,
                                               ColorsNormal)
         tpl_call_fail  = 'in %s%%s%s(***failed resolving arguments***)%s' % \
                          (Colors.vName, Colors.valEm, ColorsNormal)
         tpl_local_var  = '%s%%s%s' % (Colors.vName, ColorsNormal)
         tpl_global_var = '%sglobal%s %s%%s%s' % (Colors.em, ColorsNormal,
                                                  Colors.vName, ColorsNormal)
         tpl_name_val   = '%%s %s= %%s%s' % (Colors.valEm, ColorsNormal)
         tpl_line       = '%s%%s%s %%s' % (Colors.lineno, ColorsNormal)
         tpl_line_em    = '%s%%s%s %%s%s' % (Colors.linenoEm,Colors.line,
                                             ColorsNormal)
 
         # now, loop over all records printing context and info
         abspath = os.path.abspath
         for frame, file, lnum, func, lines, index in records:
             #print '*** record:',file,lnum,func,lines,index  # dbg
             try:
                 file = file and abspath(file) or '?'
             except OSError:
                 # if file is '<console>' or something not in the filesystem,
                 # the abspath call will throw an OSError.  Just ignore it and
                 # keep the original file string.
                 pass
             link = tpl_link % file
             try:
                 args, varargs, varkw, locals = inspect.getargvalues(frame)
             except:
                 # This can happen due to a bug in python2.3.  We should be
                 # able to remove this try/except when 2.4 becomes a
                 # requirement.  Bug details at http://python.org/sf/1005466
                 inspect_error()
                 traceback.print_exc(file=Term.cerr)
                 info("\nIPython's exception reporting continues...\n")
                 
             if func == '?':
                 call = ''
             else:
                 # Decide whether to include variable details or not
                 var_repr = self.include_vars and eqrepr or nullrepr
                 try:
                     call = tpl_call % (func,inspect.formatargvalues(args,
                                                 varargs, varkw,
                                                 locals,formatvalue=var_repr))
                 except KeyError:
                     # Very odd crash from inspect.formatargvalues().  The
                     # scenario under which it appeared was a call to
                     # view(array,scale) in NumTut.view.view(), where scale had
                     # been defined as a scalar (it should be a tuple). Somehow
                     # inspect messes up resolving the argument list of view()
                     # and barfs out. At some point I should dig into this one
                     # and file a bug report about it.
                     inspect_error()
                     traceback.print_exc(file=Term.cerr)
                     info("\nIPython's exception reporting continues...\n")
                     call = tpl_call_fail % func
 
             # Initialize a list of names on the current line, which the
             # tokenizer below will populate.
             names = []
 
             def tokeneater(token_type, token, start, end, line):
                 """Stateful tokeneater which builds dotted names.
 
                 The list of names it appends to (from the enclosing scope) can
                 contain repeated composite names.  This is unavoidable, since
                 there is no way to disambguate partial dotted structures until
                 the full list is known.  The caller is responsible for pruning
                 the final list of duplicates before using it."""
                 
                 # build composite names
                 if token == '.':
                     try:
                         names[-1] += '.'
                         # store state so the next token is added for x.y.z names
                         tokeneater.name_cont = True
                         return
                     except IndexError:
                         pass
                 if token_type == tokenize.NAME and token not in keyword.kwlist:
                     if tokeneater.name_cont:
                         # Dotted names
                         names[-1] += token
                         tokeneater.name_cont = False
                     else:
                         # Regular new names.  We append everything, the caller
                         # will be responsible for pruning the list later.  It's
                         # very tricky to try to prune as we go, b/c composite
                         # names can fool us.  The pruning at the end is easy
                         # to do (or the caller can print a list with repeated
                         # names if so desired.
                         names.append(token)
                 elif token_type == tokenize.NEWLINE:
                     raise IndexError
             # we need to store a bit of state in the tokenizer to build
             # dotted names
             tokeneater.name_cont = False
 
             def linereader(file=file, lnum=[lnum], getline=linecache.getline):
                 line = getline(file, lnum[0])
                 lnum[0] += 1
                 return line
 
             # Build the list of names on this line of code where the exception
             # occurred.
             try:
                 # This builds the names list in-place by capturing it from the
                 # enclosing scope.
                 tokenize.tokenize(linereader, tokeneater)
             except IndexError:
                 # signals exit of tokenizer
                 pass
             except tokenize.TokenError,msg:
                 _m = ("An unexpected error occurred while tokenizing input\n"
                       "The following traceback may be corrupted or invalid\n"
                       "The error message is: %s\n" % msg)
                 error(_m)
             
             # prune names list of duplicates, but keep the right order
             unique_names = uniq_stable(names)
 
             # Start loop over vars
             lvals = []
             if self.include_vars:
                 for name_full in unique_names:
                     name_base = name_full.split('.',1)[0]
                     if name_base in frame.f_code.co_varnames:
                         if locals.has_key(name_base):
                             try:
                                 value = repr(eval(name_full,locals))
                             except:
                                 value = undefined
                         else:
                             value = undefined
                         name = tpl_local_var % name_full
                     else:
                         if frame.f_globals.has_key(name_base):
                             try:
                                 value = repr(eval(name_full,frame.f_globals))
                             except:
                                 value = undefined
                         else:
                             value = undefined
                         name = tpl_global_var % name_full
                     lvals.append(tpl_name_val % (name,value))
             if lvals:
                 lvals = '%s%s' % (indent,em_normal.join(lvals))
             else:
                 lvals = ''
 
             level = '%s %s\n' % (link,call)
 
             if index is None:
                 frames.append(level)
             else:
                 frames.append('%s%s' % (level,''.join(
                     _formatTracebackLines(lnum,index,lines,Colors,lvals,
                                           col_scheme))))
 
         # Get (safely) a string form of the exception info
         try:
             etype_str,evalue_str = map(str,(etype,evalue))
         except:
             # User exception is improperly defined.
             etype,evalue = str,sys.exc_info()[:2]
             etype_str,evalue_str = map(str,(etype,evalue))
         # ... and format it
         exception = ['%s%s%s: %s' % (Colors.excName, etype_str,
                                      ColorsNormal, evalue_str)]
         if type(evalue) is types.InstanceType:
             try:
                 names = [w for w in dir(evalue) if isinstance(w, basestring)]
             except:
                 # Every now and then, an object with funny inernals blows up
                 # when dir() is called on it.  We do the best we can to report
                 # the problem and continue
                 _m = '%sException reporting error (object with broken dir())%s:'
                 exception.append(_m % (Colors.excName,ColorsNormal))
                 etype_str,evalue_str = map(str,sys.exc_info()[:2])
                 exception.append('%s%s%s: %s' % (Colors.excName,etype_str,
                                      ColorsNormal, evalue_str))
                 names = []
             for name in names:
                 value = text_repr(getattr(evalue, name))
                 exception.append('\n%s%s = %s' % (indent, name, value))
 
         # vds: >>
         if records:
              filepath, lnum = records[-1][1:3]
              #print "file:", str(file), "linenb", str(lnum) # dbg
              filepath = os.path.abspath(filepath)
              __IPYTHON__.hooks.synchronize_with_editor(filepath, lnum, 0)
         # vds: <<
                 
         # return all our info assembled as a single string
         return '%s\n\n%s\n%s' % (head,'\n'.join(frames),''.join(exception[0]) )
 
     def debugger(self,force=False):
         """Call up the pdb debugger if desired, always clean up the tb
         reference.
 
         Keywords:
 
           - force(False): by default, this routine checks the instance call_pdb
           flag and does not actually invoke the debugger if the flag is false.
           The 'force' option forces the debugger to activate even if the flag
           is false.
 
         If the call_pdb flag is set, the pdb interactive debugger is
         invoked. In all cases, the self.tb reference to the current traceback
         is deleted to prevent lingering references which hamper memory
         management.
 
         Note that each call to pdb() does an 'import readline', so if your app
         requires a special setup for the readline completers, you'll have to
         fix that by hand after invoking the exception handler."""
 
         if force or self.call_pdb:
             if self.pdb is None:
                 self.pdb = Debugger.Pdb(
                     self.color_scheme_table.active_scheme_name)
             # the system displayhook may have changed, restore the original
             # for pdb
             dhook = sys.displayhook
             sys.displayhook = sys.__displayhook__
             self.pdb.reset()
             # Find the right frame so we don't pop up inside ipython itself
             if hasattr(self,'tb'):
                 etb = self.tb
             else:
                 etb = self.tb = sys.last_traceback
             while self.tb.tb_next is not None:
                 self.tb = self.tb.tb_next
             try:
                 if etb and etb.tb_next:
                     etb = etb.tb_next
                 self.pdb.botframe = etb.tb_frame
                 self.pdb.interaction(self.tb.tb_frame, self.tb)
             finally:
                 sys.displayhook = dhook
             
         if hasattr(self,'tb'):
             del self.tb
 
     def handler(self, info=None):
         (etype, evalue, etb) = info or sys.exc_info()
         self.tb = etb
         Term.cout.flush()
         print >> Term.cerr, self.text(etype, evalue, etb)
         Term.cerr.flush()
 
     # Changed so an instance can just be called as VerboseTB_inst() and print
     # out the right info on its own.
     def __call__(self, etype=None, evalue=None, etb=None):
         """This hook can replace sys.excepthook (for Python 2.1 or higher)."""
         if etb is None:
             self.handler()
         else:
             self.handler((etype, evalue, etb))
         try:
             self.debugger()
         except KeyboardInterrupt:
             print "\nKeyboardInterrupt"
 
 #----------------------------------------------------------------------------
 class FormattedTB(VerboseTB,ListTB):
     """Subclass ListTB but allow calling with a traceback.
 
     It can thus be used as a sys.excepthook for Python > 2.1.
 
     Also adds 'Context' and 'Verbose' modes, not available in ListTB.
 
     Allows a tb_offset to be specified. This is useful for situations where
     one needs to remove a number of topmost frames from the traceback (such as
     occurs with python programs that themselves execute other python code,
     like Python shells).  """
     
     def __init__(self, mode = 'Plain', color_scheme='Linux',
                  tb_offset = 0,long_header=0,call_pdb=0,include_vars=0):
 
         # NEVER change the order of this list. Put new modes at the end:
         self.valid_modes = ['Plain','Context','Verbose']
         self.verbose_modes = self.valid_modes[1:3]
 
         VerboseTB.__init__(self,color_scheme,tb_offset,long_header,
                            call_pdb=call_pdb,include_vars=include_vars)
         self.set_mode(mode)
         
     def _extract_tb(self,tb):
         if tb:
             return traceback.extract_tb(tb)
         else:
             return None
 
     def text(self, etype, value, tb,context=5,mode=None):
         """Return formatted traceback.
 
         If the optional mode parameter is given, it overrides the current
         mode."""
 
         if mode is None:
             mode = self.mode
         if mode in self.verbose_modes:
             # verbose modes need a full traceback
             return VerboseTB.text(self,etype, value, tb,context=5)
         else:
             # We must check the source cache because otherwise we can print
             # out-of-date source code.
             linecache.checkcache()
             # Now we can extract and format the exception
             elist = self._extract_tb(tb)
             if len(elist) > self.tb_offset:
                 del elist[:self.tb_offset]
             return ListTB.text(self,etype,value,elist)
 
     def set_mode(self,mode=None):
         """Switch to the desired mode.
 
         If mode is not specified, cycles through the available modes."""
 
         if not mode:
             new_idx = ( self.valid_modes.index(self.mode) + 1 ) % \
                       len(self.valid_modes)
             self.mode = self.valid_modes[new_idx]
         elif mode not in self.valid_modes:
             raise ValueError, 'Unrecognized mode in FormattedTB: <'+mode+'>\n'\
                   'Valid modes: '+str(self.valid_modes)
         else:
             self.mode = mode
         # include variable details only in 'Verbose' mode
         self.include_vars = (self.mode == self.valid_modes[2])
 
     # some convenient shorcuts
     def plain(self):
         self.set_mode(self.valid_modes[0])
 
     def context(self):
         self.set_mode(self.valid_modes[1])
 
     def verbose(self):
         self.set_mode(self.valid_modes[2])
 
 #----------------------------------------------------------------------------
 class AutoFormattedTB(FormattedTB):
     """A traceback printer which can be called on the fly.
 
     It will find out about exceptions by itself.
 
     A brief example:
     
     AutoTB = AutoFormattedTB(mode = 'Verbose',color_scheme='Linux')
     try:
       ...
     except:
       AutoTB()  # or AutoTB(out=logfile) where logfile is an open file object
     """
     def __call__(self,etype=None,evalue=None,etb=None,
                  out=None,tb_offset=None):
         """Print out a formatted exception traceback.
 
         Optional arguments:
           - out: an open file-like object to direct output to.
 
           - tb_offset: the number of frames to skip over in the stack, on a
           per-call basis (this overrides temporarily the instance's tb_offset
           given at initialization time.  """
         
         if out is None:
             out = Term.cerr
         Term.cout.flush()
         if tb_offset is not None:
             tb_offset, self.tb_offset = self.tb_offset, tb_offset
             print >> out, self.text(etype, evalue, etb)
             self.tb_offset = tb_offset
         else:
             print >> out, self.text(etype, evalue, etb)
         out.flush()
         try:
             self.debugger()
         except KeyboardInterrupt:
             print "\nKeyboardInterrupt"
 
     def text(self,etype=None,value=None,tb=None,context=5,mode=None):
         if etype is None:
             etype,value,tb = sys.exc_info()
         self.tb = tb
         return FormattedTB.text(self,etype,value,tb,context=5,mode=mode)
 
 #---------------------------------------------------------------------------
 # A simple class to preserve Nathan's original functionality.
 class ColorTB(FormattedTB):
     """Shorthand to initialize a FormattedTB in Linux colors mode."""
     def __init__(self,color_scheme='Linux',call_pdb=0):
         FormattedTB.__init__(self,color_scheme=color_scheme,
                              call_pdb=call_pdb)
 
 #----------------------------------------------------------------------------
 # module testing (minimal)
 if __name__ == "__main__":
     def spam(c, (d, e)):
         x = c + d
         y = c * d
         foo(x, y)
 
     def foo(a, b, bar=1):
         eggs(a, b + bar)
 
     def eggs(f, g, z=globals()):
         h = f + g
         i = f - g
         return h / i
 
     print ''
     print '*** Before ***'
     try:
         print spam(1, (2, 3))
     except:
         traceback.print_exc()
     print ''
     
     handler = ColorTB()
     print '*** ColorTB ***'
     try:
         print spam(1, (2, 3))
     except:
         apply(handler, sys.exc_info() )
     print ''
     
     handler = VerboseTB()
     print '*** VerboseTB ***'
     try:
         print spam(1, (2, 3))
     except:
         apply(handler, sys.exc_info() )
     print ''
     

setup.py

@@ -1,179 +1,177 b''
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 """Setup script for IPython.
 
 Under Posix environments it works like a typical setup.py script. 
 Under Windows, the command sdist is not supported, since IPython 
 requires utilities which are not available under Windows."""
 
 #-------------------------------------------------------------------------------
 #  Copyright (C) 2008  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-------------------------------------------------------------------------------
 
 #-------------------------------------------------------------------------------
 # Imports
 #-------------------------------------------------------------------------------
 
 # Stdlib imports
 import os
 import sys
 
 from glob import glob
 
 # BEFORE importing distutils, remove MANIFEST. distutils doesn't properly
 # update it when the contents of directories change.
 if os.path.exists('MANIFEST'): os.remove('MANIFEST')
 
 from distutils.core import setup
 
 # Local imports
 from IPython.genutils import target_update
 
 from setupbase import (
     setup_args, 
     find_packages, 
     find_package_data, 
     find_scripts,
     find_data_files,
     check_for_dependencies
 )
 
 isfile = os.path.isfile
 
 #-------------------------------------------------------------------------------
 # Handle OS specific things
 #-------------------------------------------------------------------------------
 
 if os.name == 'posix':
     os_name = 'posix'
 elif os.name in ['nt','dos']:
     os_name = 'windows'
 else:
     print 'Unsupported operating system:',os.name
     sys.exit(1)
 
 # Under Windows, 'sdist' has not been supported.  Now that the docs build with
 # Sphinx it might work, but let's not turn it on until someone confirms that it
 # actually works.
 if os_name == 'windows' and 'sdist' in sys.argv:
     print 'The sdist command is not available under Windows.  Exiting.'
     sys.exit(1)
 
 #-------------------------------------------------------------------------------
 # Things related to the IPython documentation
 #-------------------------------------------------------------------------------
 
 # update the manuals when building a source dist
 if len(sys.argv) >= 2 and sys.argv[1] in ('sdist','bdist_rpm'):
     import textwrap
 
     # List of things to be updated. Each entry is a triplet of args for
     # target_update()
     to_update = [
                   # FIXME - Disabled for now: we need to redo an automatic way
                   # of generating the magic info inside the rst.                
                   #('docs/magic.tex',
                   #['IPython/Magic.py'],
                   #"cd doc && ./update_magic.sh" ),
                  
                  ('docs/man/ipython.1.gz',
                   ['docs/man/ipython.1'],
                   "cd docs/man && gzip -9c ipython.1 > ipython.1.gz"),
 
                  ('docs/man/pycolor.1.gz',
                   ['docs/man/pycolor.1'],
                   "cd docs/man && gzip -9c pycolor.1 > pycolor.1.gz"),
                  ]
 
     # Only build the docs is sphinx is present
     try:
         import sphinx
     except ImportError:
         pass
     else:
         pass
         # BEG: This is disabled as I am not sure what to depend on.
         # I actually don't think we should be automatically building
         # the docs for people.
         # The do_sphinx scripts builds html and pdf, so just one
         # target is enough to cover all manual generation
         # to_update.append(
         #     ('docs/manual/ipython.pdf',
         #     ['IPython/Release.py','docs/source/ipython.rst'],
         #     "cd docs && python do_sphinx.py")
         # )
         
     [ target_update(*t) for t in to_update ]
 
     # Build the docs
     os.system('cd docs && make dist')
     
 #---------------------------------------------------------------------------
 # Find all the packages, package data, scripts and data_files
 #---------------------------------------------------------------------------
 
 packages = find_packages()
 package_data = find_package_data()
 scripts = find_scripts()
 data_files = find_data_files()
 
 #---------------------------------------------------------------------------
 # Handle dependencies and setuptools specific things
 #---------------------------------------------------------------------------
 
 # This dict is used for passing extra arguments that are setuptools 
 # specific to setup
 setuptools_extra_args = {}
 
 if 'setuptools' in sys.modules:
     setuptools_extra_args['zip_safe'] = False
     setuptools_extra_args['entry_points'] = {
         'console_scripts': [
             'ipython = IPython.ipapi:launch_new_instance',
             'pycolor = IPython.PyColorize:main',
             'ipcontroller = IPython.kernel.scripts.ipcontroller:main',
             'ipengine = IPython.kernel.scripts.ipengine:main',
             'ipcluster = IPython.kernel.scripts.ipcluster:main',
             'ipythonx = IPython.frontend.wx.ipythonx:main'
         ]
     }
     setup_args["extras_require"] = dict(
         kernel = [
             "zope.interface>=3.4.1",
             "Twisted>=8.0.1",
             "foolscap>=0.2.6"        
         ],
         doc=['Sphinx>=0.3','pygments'],
         test='nose>=0.10.1',
         security=["pyOpenSSL>=0.6"]
     )
     # Allow setuptools to handle the scripts
     scripts = []
-    # eggs will lack docs, examples
-    data_files = []
 else:
     # package_data of setuptools was introduced to distutils in 2.4
     cfgfiles = filter(isfile, glob('IPython/UserConfig/*'))
     if sys.version_info < (2,4):
         data_files.append(('lib', 'IPython/UserConfig', cfgfiles))
     # If we are running without setuptools, call this function which will
     # check for dependencies an inform the user what is needed.  This is
     # just to make life easy for users.
     check_for_dependencies()
 
 
 #---------------------------------------------------------------------------
 # Do the actual setup now
 #---------------------------------------------------------------------------
 
 setup_args['packages'] = packages
 setup_args['package_data'] = package_data
 setup_args['scripts'] = scripts
 setup_args['data_files'] = data_files
 setup_args.update(setuptools_extra_args)
 
 if __name__ == '__main__':
     setup(**setup_args)

setupbase.py

@@ -1,275 +1,276 b''
 # encoding: utf-8
 
 """
 This module defines the things that are used in setup.py for building IPython
 
 This includes:
 
     * The basic arguments to setup
     * Functions for finding things like packages, package data, etc.
     * A function for checking dependencies.
 """
 
 __docformat__ = "restructuredtext en"
 
 #-------------------------------------------------------------------------------
 #  Copyright (C) 2008  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-------------------------------------------------------------------------------
 
 #-------------------------------------------------------------------------------
 # Imports
 #-------------------------------------------------------------------------------
 
 import os, sys
 
 from glob import glob
 
 from setupext import install_data_ext
 
 #-------------------------------------------------------------------------------
 # Useful globals and utility functions
 #-------------------------------------------------------------------------------
 
 # A few handy globals
 isfile = os.path.isfile
 pjoin = os.path.join
 
 def oscmd(s):
     print ">", s
     os.system(s)
 
 # A little utility we'll need below, since glob() does NOT allow you to do
 # exclusion on multiple endings!
 def file_doesnt_endwith(test,endings):
     """Return true if test is a file and its name does NOT end with any
     of the strings listed in endings."""
     if not isfile(test):
         return False
     for e in endings:
         if test.endswith(e):
             return False
     return True
 
 #---------------------------------------------------------------------------
 # Basic project information
 #---------------------------------------------------------------------------
 
 # Release.py contains version, authors, license, url, keywords, etc.
 execfile(pjoin('IPython','Release.py'))
 
 # Create a dict with the basic information
 # This dict is eventually passed to setup after additional keys are added.
 setup_args = dict(
       name             = name,
       version          = version,
       description      = description,
       long_description = long_description,
       author           = author,
       author_email     = author_email,
       url              = url,
       download_url     = download_url,
       license          = license,
       platforms        = platforms,
       keywords         = keywords,
       cmdclass         = {'install_data': install_data_ext},
       )
 
 
 #---------------------------------------------------------------------------
 # Find packages
 #---------------------------------------------------------------------------
 
 def add_package(packages,pname,config=False,tests=False,scripts=False,
                 others=None):
     """
     Add a package to the list of packages, including certain subpackages.
     """
     packages.append('.'.join(['IPython',pname]))
     if config:
         packages.append('.'.join(['IPython',pname,'config']))
     if tests:
         packages.append('.'.join(['IPython',pname,'tests']))
     if scripts:
         packages.append('.'.join(['IPython',pname,'scripts']))
     if others is not None:
         for o in others:
             packages.append('.'.join(['IPython',pname,o]))
 
 def find_packages():
     """
     Find all of IPython's packages.
     """
     packages = ['IPython']
     add_package(packages, 'config', tests=True)
     add_package(packages , 'Extensions')
     add_package(packages, 'external')
     add_package(packages, 'gui')
     add_package(packages, 'gui.wx')
     add_package(packages, 'frontend', tests=True)
     add_package(packages, 'frontend._process')
     add_package(packages, 'frontend.wx')
     add_package(packages, 'frontend.cocoa', tests=True)
     add_package(packages, 'kernel', config=True, tests=True, scripts=True)
     add_package(packages, 'kernel.core', config=True, tests=True)
     add_package(packages, 'testing', tests=True)
     add_package(packages, 'tools', tests=True)
     add_package(packages, 'UserConfig')
     return packages
 
 #---------------------------------------------------------------------------
 # Find package data
 #---------------------------------------------------------------------------
 
 def find_package_data():
     """
     Find IPython's package_data.
     """
     # This is not enough for these things to appear in an sdist.
     # We need to muck with the MANIFEST to get this to work
     package_data = {
         'IPython.UserConfig' : ['*'],
         'IPython.tools.tests' : ['*.txt'],
         'IPython.testing' : ['*.txt']
     }
     return package_data
 
 
 #---------------------------------------------------------------------------
 # Find data files
 #---------------------------------------------------------------------------
 
 def make_dir_struct(tag,base,out_base):
     """Make the directory structure of all files below a starting dir.
 
     This is just a convenience routine to help build a nested directory
     hierarchy because distutils is too stupid to do this by itself.
 
     XXX - this needs a proper docstring!
     """
     
     # we'll use these a lot below
     lbase = len(base)
     pathsep = os.path.sep
     lpathsep = len(pathsep)
     
     out = []
     for (dirpath,dirnames,filenames) in os.walk(base):
         # we need to strip out the dirpath from the base to map it to the
         # output (installation) path.  This requires possibly stripping the
         # path separator, because otherwise pjoin will not work correctly
         # (pjoin('foo/','/bar') returns '/bar').
         
         dp_eff = dirpath[lbase:]
         if dp_eff.startswith(pathsep):
             dp_eff = dp_eff[lpathsep:]
         # The output path must be anchored at the out_base marker    
         out_path = pjoin(out_base,dp_eff)
         # Now we can generate the final filenames. Since os.walk only produces
         # filenames, we must join back with the dirpath to get full valid file
         # paths:
         pfiles = [pjoin(dirpath,f) for f in filenames]
         # Finally, generate the entry we need, which is a triple of (tag,output
         # path, files) for use as a data_files parameter in install_data.
         out.append((tag,out_path,pfiles))
 
     return out
     
 
 def find_data_files():
     """
     Find IPython's data_files.
 
     Most of these are docs.
     """
     
     docdirbase  = 'share/doc/ipython'
     manpagebase = 'share/man/man1'
 
     # Simple file lists can be made by hand
     manpages  = filter(isfile, glob('docs/man/*.1.gz'))
     igridhelpfiles = filter(isfile, glob('IPython/Extensions/igrid_help.*'))
 
     # For nested structures, use the utility above
     example_files = make_dir_struct('data','docs/examples',
                                     pjoin(docdirbase,'examples'))
     manual_files = make_dir_struct('data','docs/dist',pjoin(docdirbase,'manual'))
 
     # And assemble the entire output list
     data_files = [ ('data',manpagebase, manpages),
                    ('data',pjoin(docdirbase,'extensions'),igridhelpfiles),
                    ] + manual_files + example_files
                  
     ## import pprint  # dbg
     ## print '*'*80
     ## print 'data files'
     ## pprint.pprint(data_files)
     ## print '*'*80
     
     return data_files
 
 #---------------------------------------------------------------------------
 # Find scripts
 #---------------------------------------------------------------------------
 
 def find_scripts():
     """
     Find IPython's scripts.
     """
     scripts = ['IPython/kernel/scripts/ipengine',
                'IPython/kernel/scripts/ipcontroller',
                 'IPython/kernel/scripts/ipcluster',
                'scripts/ipython',
                'scripts/ipythonx',
                'scripts/pycolor',
                'scripts/irunner',
+               'scripts/iptest',
                ]
     
     # Script to be run by the windows binary installer after the default setup
     # routine, to add shortcuts and similar windows-only things.  Windows
     # post-install scripts MUST reside in the scripts/ dir, otherwise distutils
     # doesn't find them.
     if 'bdist_wininst' in sys.argv:
         if len(sys.argv) > 2 and ('sdist' in sys.argv or 'bdist_rpm' in sys.argv):
             print >> sys.stderr,"ERROR: bdist_wininst must be run alone. Exiting."
             sys.exit(1)
         scripts.append('scripts/ipython_win_post_install.py')
     
     return scripts
 
 #---------------------------------------------------------------------------
 # Verify all dependencies
 #---------------------------------------------------------------------------
 
 def check_for_dependencies():
     """Check for IPython's dependencies.
     
     This function should NOT be called if running under setuptools!
     """
     from setupext.setupext import (
         print_line, print_raw, print_status, print_message,
         check_for_zopeinterface, check_for_twisted,
         check_for_foolscap, check_for_pyopenssl,
         check_for_sphinx, check_for_pygments,
         check_for_nose, check_for_pexpect
     )
     print_line()
     print_raw("BUILDING IPYTHON")
     print_status('python', sys.version)
     print_status('platform', sys.platform)
     if sys.platform == 'win32':
         print_status('Windows version', sys.getwindowsversion())
     
     print_raw("")
     print_raw("OPTIONAL DEPENDENCIES")
 
     check_for_zopeinterface()
     check_for_twisted()
     check_for_foolscap()
     check_for_pyopenssl()
     check_for_sphinx()
     check_for_pygments()
     check_for_nose()
     check_for_pexpect()