Complete implementation of interactive traceback support.

Ever since IPython started, we've had no proper tracebacks for
interactively entered code.  The terminal version was fairly
uncomfortable for more than just a few lines of code, so this was
never too big of a deal.  But the new architecture allows clients with
complex multiline input, and having proper tracebacks becomes now
critical for real-world use.

Thanks to Robert Kern for key implementation ideas and original patch.

Closes gh-177.

IPython/core/compilerop.py

@@ -0,0 +1,119 b''
+"""Compiler tools with improved interactive support.
+
+Provides compilation machinery similar to codeop, but with caching support so
+we can provide interactive tracebacks.
+
+Authors
+-------
+* Robert Kern
+* Fernando Perez
+"""
+
+# Note: though it might be more natural to name this module 'compiler', that
+# name is in the stdlib and name collisions with the stdlib tend to produce
+# weird problems (often with third-party tools).
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2010 The IPython Development Team.
+#
+#  Distributed under the terms of the BSD License.
+#
+#  The full license is in the file COPYING.txt, distributed with this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+from __future__ import print_function
+
+# Stdlib imports
+import codeop
+import hashlib
+import linecache
+import time
+
+#-----------------------------------------------------------------------------
+# Local utilities
+#-----------------------------------------------------------------------------
+
+def code_name(code, number=0):
+    """ Compute a (probably) unique name for code for caching.
+    """
+    hash_digest = hashlib.md5(code).hexdigest()
+    # Include the number and 12 characters of the hash in the name.  It's
+    # pretty much impossible that in a single session we'll have collisions
+    # even with truncated hashes, and the full one makes tracebacks too long
+    return '<ipython-input-{0}-{1}>'.format(number, hash_digest[:12])
+
+#-----------------------------------------------------------------------------
+# Classes and functions
+#-----------------------------------------------------------------------------
+
+class CachingCompiler(object):
+    """A compiler that caches code compiled from interactive statements.
+    """
+
+    def __init__(self):
+        self._compiler = codeop.CommandCompiler()
+        
+        # This is ugly, but it must be done this way to allow multiple
+        # simultaneous ipython instances to coexist.  Since Python itself
+        # directly accesses the data structures in the linecache module, and
+        # the cache therein is global, we must work with that data structure.
+        # We must hold a reference to the original checkcache routine and call
+        # that in our own check_cache() below, but the special IPython cache
+        # must also be shared by all IPython instances.  If we were to hold
+        # separate caches (one in each CachingCompiler instance), any call made
+        # by Python itself to linecache.checkcache() would obliterate the
+        # cached data from the other IPython instances.
+        if not hasattr(linecache, '_ipython_cache'):
+            linecache._ipython_cache = {}
+        if not hasattr(linecache, '_checkcache_ori'):
+            linecache._checkcache_ori = linecache.checkcache
+        # Now, we must monkeypatch the linecache directly so that parts of the
+        # stdlib that call it outside our control go through our codepath
+        # (otherwise we'd lose our tracebacks).
+        linecache.checkcache = self.check_cache
+
+    @property
+    def compiler_flags(self):
+        """Flags currently active in the compilation process.
+        """
+        return self._compiler.compiler.flags
+        
+    def __call__(self, code, symbol, number=0):
+        """Compile some code while caching its contents such that the inspect
+        module can find it later.
+
+        Parameters
+        ----------
+        code : str
+          Source code to be compiled, one or more lines.
+
+        symbol : str
+          One of 'single', 'exec' or 'eval' (see the builtin ``compile``
+          documentation for further details on these fields).
+
+        number : int, optional
+          An integer argument identifying the code, useful for informational
+          purposes in tracebacks (typically it will be the IPython prompt
+          number).
+        """
+        name = code_name(code, number)
+        code_obj = self._compiler(code, name, symbol)
+        entry = (len(code), time.time(),
+                 [line+'\n' for line in code.splitlines()], name)
+        # Cache the info both in the linecache (a global cache used internally
+        # by most of Python's inspect/traceback machinery), and in our cache
+        linecache.cache[name] = entry
+        linecache._ipython_cache[name] = entry
+        return code_obj
+
+    def check_cache(self, *args):
+        """Call linecache.checkcache() safely protecting our cached values.
+        """
+        # First call the orignal checkcache as intended
+        linecache._checkcache_ori(*args)
+        # Then, update back the cache with our data, so that tracebacks related
+        # to our compiled codes can be produced.
+        linecache.cache.update(linecache._ipython_cache)

IPython/core/tests/test_compilerop.py

@@ -0,0 +1,62 b''
+"""Tests for the compilerop module.
+"""
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2010 The IPython Development Team.
+#
+#  Distributed under the terms of the BSD License.
+#
+#  The full license is in the file COPYING.txt, distributed with this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+from __future__ import print_function
+
+# Stdlib imports
+import linecache
+
+# Third-party imports
+import nose.tools as nt
+
+# Our own imports
+from IPython.core import compilerop
+
+#-----------------------------------------------------------------------------
+# Test functions
+#-----------------------------------------------------------------------------
+
+def test_code_name():
+    code = 'x=1'
+    name = compilerop.code_name(code)
+    nt.assert_true(name.startswith('<ipython-input-0'))
+
+
+def test_code_name2():
+    code = 'x=1'
+    name = compilerop.code_name(code, 9)
+    nt.assert_true(name.startswith('<ipython-input-9'))
+
+
+def test_compiler():
+    """Test the compiler correctly compiles and caches inputs
+    """
+    cp = compilerop.CachingCompiler()
+    ncache = len(linecache.cache)
+    cp('x=1', 'single')
+    nt.assert_true(len(linecache.cache) > ncache)
+
+
+def test_compiler_check_cache():
+    """Test the compiler properly manages the cache.
+    """
+    # Rather simple-minded tests that just exercise the API
+    cp = compilerop.CachingCompiler()
+    cp('x=1', 'single', 99)
+    # Ensure now that after clearing the cache, our entries survive
+    cp.check_cache()
+    for k in linecache.cache:
+        if k.startswith('<ipython-input-99'):
+            break
+    else:
+        raise AssertionError('Entry for input-99 missing from linecache')

IPython/core/interactiveshell.py

@@ -40,6 +40,7 b' from IPython.core import shadowns'
 from IPython.core import ultratb
 from IPython.core.alias import AliasManager
 from IPython.core.builtin_trap import BuiltinTrap
+from IPython.core.compilerop import CachingCompiler
 from IPython.core.display_trap import DisplayTrap
 from IPython.core.displayhook import DisplayHook
 from IPython.core.error import TryNext, UsageError
@@ -136,39 +137,6 b' class MultipleInstanceError(Exception):'
 # Main IPython class
 #-----------------------------------------------------------------------------
 
-
-######## Code to be moved later if it works, meant to try to get proper
-######## tracebacks
-
-import hashlib
-import linecache
-import time
-import types
-
-def code_name(code):
-    """ Compute a (probably) unique name for code for caching.
-    """
-    hash_digest = hashlib.md5(code).hexdigest()
-    return '<code %s>' % hash_digest
-
-
-class CachingCompiler(codeop.CommandCompiler):
-        
-    def __call__(self, code, filename, symbol):
-        """ Compile some code while caching its contents such that the inspect
-        module can find it later.
-        """
-        #code += '\n'
-        name = code_name(code)
-        code_obj = codeop.CommandCompiler.__call__(self, code, name, symbol)
-        linecache.cache[name] = (len(code),
-                                 time.time(),
-                                 [line+'\n' for line in  code.splitlines()],
-                                 name) 
-        return code_obj
-
-#############
-
 class InteractiveShell(Configurable, Magic):
     """An enhanced, interactive shell for Python."""
 
@@ -401,7 +369,6 b' class InteractiveShell(Configurable, Magic):'
         self.more = False
 
         # command compiler
-        #self.compile = codeop.CommandCompiler()
         self.compile = CachingCompiler()
         
         # User input buffers
@@ -1134,7 +1101,7 b' class InteractiveShell(Configurable, Magic):'
         # We need to special-case 'print', which as of python2.6 registers as a
         # function but should only be treated as one if print_function was
         # loaded with a future import.  In this case, just bail.
-        if (oname == 'print' and not (self.compile.compiler.flags &
+        if (oname == 'print' and not (self.compile.compiler_flags &
                                       __future__.CO_FUTURE_PRINT_FUNCTION)):
             return {'found':found, 'obj':obj, 'namespace':ospace,
                     'ismagic':ismagic, 'isalias':isalias, 'parent':parent}
@@ -1293,7 +1260,8 b' class InteractiveShell(Configurable, Magic):'
         # internal code. Valid modes: ['Plain','Context','Verbose']
         self.InteractiveTB = ultratb.AutoFormattedTB(mode = 'Plain',
                                                      color_scheme='NoColor',
-                                                     tb_offset = 1)
+                                                     tb_offset = 1,
+                                   check_cache=self.compile.check_cache)
 
         # The instance will store a pointer to the system-wide exception hook,
         # so that runtime code (such as magics) can access it.  This is because
@@ -2159,14 +2127,15 b' class InteractiveShell(Configurable, Magic):'
 
                 # Get the main body to run as a cell
                 ipy_body = ''.join(blocks[:-1])
-                retcode = self.run_code(ipy_body, post_execute=False)
+                retcode = self.run_source(ipy_body, symbol='exec',
+                                          post_execute=False)
                 if retcode==0:
                     # And the last expression via runlines so it produces output
                     self.run_one_block(last)
             else:
                 # Run the whole cell as one entity, storing both raw and
                 # processed input in history
-                self.run_code(ipy_cell)
+                self.run_source(ipy_cell, symbol='exec')
 
         # Each cell is a *single* input, regardless of how many lines it has
         self.execution_count += 1
@@ -2242,7 +2211,8 b' class InteractiveShell(Configurable, Magic):'
             if more:
                 self.push_line('\n')
 
-    def run_source(self, source, filename='<ipython console>', symbol='single'):
+    def run_source(self, source, filename=None,
+                   symbol='single', post_execute=True):
         """Compile and run some source in the interpreter.
 
         Arguments are as for compile_command().
@@ -2284,7 +2254,7 b' class InteractiveShell(Configurable, Magic):'
             print 'encoding', self.stdin_encoding  # dbg
         
         try:
-            code = self.compile(usource,filename,symbol)
+            code = self.compile(usource, symbol, self.execution_count)
         except (OverflowError, SyntaxError, ValueError, TypeError, MemoryError):
             # Case 1
             self.showsyntaxerror(filename)
@@ -2301,7 +2271,7 b' class InteractiveShell(Configurable, Magic):'
         # buffer attribute as '\n'.join(self.buffer).
         self.code_to_run = code
         # now actually execute the code object
-        if self.run_code(code) == 0:
+        if self.run_code(code, post_execute) == 0:
             return False
         else:
             return None
@@ -2480,7 +2450,7 b' class InteractiveShell(Configurable, Magic):'
                           sys._getframe(depth+1).f_locals # locals
                           ))
 
-    def mktempfile(self,data=None):
+    def mktempfile(self, data=None, prefix='ipython_edit_'):
         """Make a new tempfile and return its filename.
 
         This makes a call to tempfile.mktemp, but it registers the created
@@ -2491,7 +2461,7 b' class InteractiveShell(Configurable, Magic):'
           - data(None): if data is given, it gets written out to the temp file
           immediately, and the file is closed again."""
 
-        filename = tempfile.mktemp('.py','ipython_edit_')
+        filename = tempfile.mktemp('.py', prefix)
         self.tempfiles.append(filename)
         
         if data:

IPython/core/tests/test_iplib.py

@@ -92,8 +92,6 b' ZeroDivisionError Traceback (most recent call last)'
      30         mode = 'div'
      31 
 ---> 32     bar(mode)
-     33 
-     34 
 <BLANKLINE>
 ... in bar(mode)
      14     "bar"
@@ -128,8 +126,6 b' ZeroDivisionError Traceback (most recent call last)'
 ---> 32     bar(mode)
         global bar = <function bar at ...>
         global mode = 'div'
-     33 
-     34 
 <BLANKLINE>
 ... in bar(mode='div')
      14     "bar"
@@ -186,8 +182,6 b' SystemExit Traceback (most recent call last)'
      30         mode = 'div'
      31 
 ---> 32     bar(mode)
-     33 
-     34 
 <BLANKLINE>
 ...bar(mode)
      20         except:
@@ -218,8 +212,6 b' SystemExit Traceback (most recent call last)'
 ---> 32     bar(mode)
         global bar = <function bar at ...>
         global mode = 'exit'
-     33 
-     34 
 <BLANKLINE>
 ... in bar(mode='exit')
      20         except:

IPython/core/ultratb.py

@@ -244,11 +244,6 b' def _fixed_getinnerframes(etb, context=1,tb_offset=0):'
         start =  max(maybeStart, 0)
         end   = start + context
         lines = linecache.getlines(file)[start:end]
-        # pad with empty lines if necessary
-        if maybeStart < 0:
-            lines = (['\n'] * -maybeStart) + lines
-        if len(lines) < context:
-            lines += ['\n'] * (context - len(lines))
         buf = list(records[i])
         buf[LNUM_POS] = lnum
         buf[INDEX_POS] = lnum - 1 - start
@@ -279,7 +274,15 b' def _format_traceback_lines(lnum, index, lines, Colors, lvals=None,scheme=None):'
     _line_format = _parser.format2
 
     for line in lines:
-        new_line, err = _line_format(line,'str',scheme)
+        # FIXME: we need to ensure the source is a pure string at this point,
+        # else the coloring code makes a  royal mess.  This is in need of a
+        # serious refactoring, so that all of the ultratb and PyColorize code
+        # is unicode-safe.  So for now this is rather an ugly hack, but
+        # necessary to at least have readable tracebacks. Improvements welcome!
+        if type(line)==unicode:
+            line = line.encode('utf-8', 'replace')
+            
+        new_line, err = _line_format(line, 'str', scheme)
         if not err: line = new_line
         
         if i == lnum:
@@ -636,7 +639,8 b' class VerboseTB(TBTools):'
     would appear in the traceback)."""
 
     def __init__(self,color_scheme = 'Linux', call_pdb=False, ostream=None,
-                 tb_offset=0, long_header=False, include_vars=True):
+                 tb_offset=0, long_header=False, include_vars=True,
+                 check_cache=None):
         """Specify traceback offset, headers and color scheme.
 
         Define how many frames to drop from the tracebacks. Calling it with
@@ -648,6 +652,14 b' class VerboseTB(TBTools):'
         self.tb_offset = tb_offset
         self.long_header = long_header
         self.include_vars = include_vars
+        # By default we use linecache.checkcache, but the user can provide a
+        # different check_cache implementation.  This is used by the IPython
+        # kernel to provide tracebacks for interactive code that is cached,
+        # by a compiler instance that flushes the linecache but preserves its
+        # own code cache.
+        if check_cache is None:
+            check_cache = linecache.checkcache
+        self.check_cache = check_cache
 
     def structured_traceback(self, etype, evalue, etb, tb_offset=None,
                              context=5):
@@ -723,7 +735,7 b' class VerboseTB(TBTools):'
         frames = []
         # Flush cache before calling inspect.  This helps alleviate some of the
         # problems with python 2.3's inspect.py.
-        linecache.checkcache()
+        ##self.check_cache()
         # Drop topmost frames if requested
         try:
             # Try the default getinnerframes and Alex's: Alex's fixes some
@@ -1034,7 +1046,8 b' class FormattedTB(VerboseTB, ListTB):'
     
     def __init__(self, mode='Plain', color_scheme='Linux', call_pdb=False,
                  ostream=None, 
-                 tb_offset=0, long_header=False, include_vars=False):
+                 tb_offset=0, long_header=False, include_vars=False,
+                 check_cache=None):
 
         # NEVER change the order of this list. Put new modes at the end:
         self.valid_modes = ['Plain','Context','Verbose']
@@ -1042,7 +1055,8 b' class FormattedTB(VerboseTB, ListTB):'
 
         VerboseTB.__init__(self, color_scheme=color_scheme, call_pdb=call_pdb,
                            ostream=ostream, tb_offset=tb_offset,
-                           long_header=long_header, include_vars=include_vars)
+                           long_header=long_header, include_vars=include_vars,
+                           check_cache=check_cache)
 
         # Different types of tracebacks are joined with different separators to
         # form a single string.  They are taken from this dict
@@ -1067,7 +1081,7 b' class FormattedTB(VerboseTB, ListTB):'
         else:
             # We must check the source cache because otherwise we can print
             # out-of-date source code.
-            linecache.checkcache()
+            self.check_cache()
             # Now we can extract and format the exception
             elist = self._extract_tb(tb)
             return ListTB.structured_traceback(

IPython/lib/tests/test_irunner.py

@@ -97,7 +97,7 b' In [7]: autocall 0'
 Automatic calling is: OFF
 
 In [8]: cos pi
-   File "<ipython console>", line 1
+   File "<ipython-input-8-6bd7313dd9a9>", line 1
      cos pi
           ^
 SyntaxError: invalid syntax