From c94126c20b9b761f78a574eb4dcacc04f164b398 2010-10-22 07:14:51 From: Fernando Perez Date: 2010-10-22 07:14:51 Subject: [PATCH] Complete implementation of interactive traceback support. Ever since IPython started, we've had no proper tracebacks for interactively entered code. The terminal version was fairly uncomfortable for more than just a few lines of code, so this was never too big of a deal. But the new architecture allows clients with complex multiline input, and having proper tracebacks becomes now critical for real-world use. Thanks to Robert Kern for key implementation ideas and original patch. Closes gh-177. --- diff --git a/IPython/core/compilerop.py b/IPython/core/compilerop.py new file mode 100644 index 0000000..29c9f68 --- /dev/null +++ b/IPython/core/compilerop.py @@ -0,0 +1,119 @@ +"""Compiler tools with improved interactive support. + +Provides compilation machinery similar to codeop, but with caching support so +we can provide interactive tracebacks. + +Authors +------- +* Robert Kern +* Fernando Perez +""" + +# Note: though it might be more natural to name this module 'compiler', that +# name is in the stdlib and name collisions with the stdlib tend to produce +# weird problems (often with third-party tools). + +#----------------------------------------------------------------------------- +# Copyright (C) 2010 The IPython Development Team. +# +# Distributed under the terms of the BSD License. +# +# The full license is in the file COPYING.txt, distributed with this software. +#----------------------------------------------------------------------------- + +#----------------------------------------------------------------------------- +# Imports +#----------------------------------------------------------------------------- +from __future__ import print_function + +# Stdlib imports +import codeop +import hashlib +import linecache +import time + +#----------------------------------------------------------------------------- +# Local utilities +#----------------------------------------------------------------------------- + +def code_name(code, number=0): + """ Compute a (probably) unique name for code for caching. + """ + hash_digest = hashlib.md5(code).hexdigest() + # Include the number and 12 characters of the hash in the name. It's + # pretty much impossible that in a single session we'll have collisions + # even with truncated hashes, and the full one makes tracebacks too long + return ''.format(number, hash_digest[:12]) + +#----------------------------------------------------------------------------- +# Classes and functions +#----------------------------------------------------------------------------- + +class CachingCompiler(object): + """A compiler that caches code compiled from interactive statements. + """ + + def __init__(self): + self._compiler = codeop.CommandCompiler() + + # This is ugly, but it must be done this way to allow multiple + # simultaneous ipython instances to coexist. Since Python itself + # directly accesses the data structures in the linecache module, and + # the cache therein is global, we must work with that data structure. + # We must hold a reference to the original checkcache routine and call + # that in our own check_cache() below, but the special IPython cache + # must also be shared by all IPython instances. If we were to hold + # separate caches (one in each CachingCompiler instance), any call made + # by Python itself to linecache.checkcache() would obliterate the + # cached data from the other IPython instances. + if not hasattr(linecache, '_ipython_cache'): + linecache._ipython_cache = {} + if not hasattr(linecache, '_checkcache_ori'): + linecache._checkcache_ori = linecache.checkcache + # Now, we must monkeypatch the linecache directly so that parts of the + # stdlib that call it outside our control go through our codepath + # (otherwise we'd lose our tracebacks). + linecache.checkcache = self.check_cache + + @property + def compiler_flags(self): + """Flags currently active in the compilation process. + """ + return self._compiler.compiler.flags + + def __call__(self, code, symbol, number=0): + """Compile some code while caching its contents such that the inspect + module can find it later. + + Parameters + ---------- + code : str + Source code to be compiled, one or more lines. + + symbol : str + One of 'single', 'exec' or 'eval' (see the builtin ``compile`` + documentation for further details on these fields). + + number : int, optional + An integer argument identifying the code, useful for informational + purposes in tracebacks (typically it will be the IPython prompt + number). + """ + name = code_name(code, number) + code_obj = self._compiler(code, name, symbol) + entry = (len(code), time.time(), + [line+'\n' for line in code.splitlines()], name) + # Cache the info both in the linecache (a global cache used internally + # by most of Python's inspect/traceback machinery), and in our cache + linecache.cache[name] = entry + linecache._ipython_cache[name] = entry + return code_obj + + def check_cache(self, *args): + """Call linecache.checkcache() safely protecting our cached values. + """ + # First call the orignal checkcache as intended + linecache._checkcache_ori(*args) + # Then, update back the cache with our data, so that tracebacks related + # to our compiled codes can be produced. + linecache.cache.update(linecache._ipython_cache) diff --git a/IPython/core/interactiveshell.py b/IPython/core/interactiveshell.py index 081c5ee..7f192dd 100644 --- a/IPython/core/interactiveshell.py +++ b/IPython/core/interactiveshell.py @@ -40,6 +40,7 @@ from IPython.core import shadowns from IPython.core import ultratb from IPython.core.alias import AliasManager from IPython.core.builtin_trap import BuiltinTrap +from IPython.core.compilerop import CachingCompiler from IPython.core.display_trap import DisplayTrap from IPython.core.displayhook import DisplayHook from IPython.core.error import TryNext, UsageError @@ -136,39 +137,6 @@ class MultipleInstanceError(Exception): # Main IPython class #----------------------------------------------------------------------------- - -######## Code to be moved later if it works, meant to try to get proper -######## tracebacks - -import hashlib -import linecache -import time -import types - -def code_name(code): - """ Compute a (probably) unique name for code for caching. - """ - hash_digest = hashlib.md5(code).hexdigest() - return '' % hash_digest - - -class CachingCompiler(codeop.CommandCompiler): - - def __call__(self, code, filename, symbol): - """ Compile some code while caching its contents such that the inspect - module can find it later. - """ - #code += '\n' - name = code_name(code) - code_obj = codeop.CommandCompiler.__call__(self, code, name, symbol) - linecache.cache[name] = (len(code), - time.time(), - [line+'\n' for line in code.splitlines()], - name) - return code_obj - -############# - class InteractiveShell(Configurable, Magic): """An enhanced, interactive shell for Python.""" @@ -401,7 +369,6 @@ class InteractiveShell(Configurable, Magic): self.more = False # command compiler - #self.compile = codeop.CommandCompiler() self.compile = CachingCompiler() # User input buffers @@ -1134,7 +1101,7 @@ class InteractiveShell(Configurable, Magic): # We need to special-case 'print', which as of python2.6 registers as a # function but should only be treated as one if print_function was # loaded with a future import. In this case, just bail. - if (oname == 'print' and not (self.compile.compiler.flags & + if (oname == 'print' and not (self.compile.compiler_flags & __future__.CO_FUTURE_PRINT_FUNCTION)): return {'found':found, 'obj':obj, 'namespace':ospace, 'ismagic':ismagic, 'isalias':isalias, 'parent':parent} @@ -1293,7 +1260,8 @@ class InteractiveShell(Configurable, Magic): # internal code. Valid modes: ['Plain','Context','Verbose'] self.InteractiveTB = ultratb.AutoFormattedTB(mode = 'Plain', color_scheme='NoColor', - tb_offset = 1) + tb_offset = 1, + check_cache=self.compile.check_cache) # The instance will store a pointer to the system-wide exception hook, # so that runtime code (such as magics) can access it. This is because @@ -2159,14 +2127,15 @@ class InteractiveShell(Configurable, Magic): # Get the main body to run as a cell ipy_body = ''.join(blocks[:-1]) - retcode = self.run_code(ipy_body, post_execute=False) + retcode = self.run_source(ipy_body, symbol='exec', + post_execute=False) if retcode==0: # And the last expression via runlines so it produces output self.run_one_block(last) else: # Run the whole cell as one entity, storing both raw and # processed input in history - self.run_code(ipy_cell) + self.run_source(ipy_cell, symbol='exec') # Each cell is a *single* input, regardless of how many lines it has self.execution_count += 1 @@ -2242,7 +2211,8 @@ class InteractiveShell(Configurable, Magic): if more: self.push_line('\n') - def run_source(self, source, filename='', symbol='single'): + def run_source(self, source, filename=None, + symbol='single', post_execute=True): """Compile and run some source in the interpreter. Arguments are as for compile_command(). @@ -2284,7 +2254,7 @@ class InteractiveShell(Configurable, Magic): print 'encoding', self.stdin_encoding # dbg try: - code = self.compile(usource,filename,symbol) + code = self.compile(usource, symbol, self.execution_count) except (OverflowError, SyntaxError, ValueError, TypeError, MemoryError): # Case 1 self.showsyntaxerror(filename) @@ -2301,7 +2271,7 @@ class InteractiveShell(Configurable, Magic): # buffer attribute as '\n'.join(self.buffer). self.code_to_run = code # now actually execute the code object - if self.run_code(code) == 0: + if self.run_code(code, post_execute) == 0: return False else: return None @@ -2480,7 +2450,7 @@ class InteractiveShell(Configurable, Magic): sys._getframe(depth+1).f_locals # locals )) - def mktempfile(self,data=None): + def mktempfile(self, data=None, prefix='ipython_edit_'): """Make a new tempfile and return its filename. This makes a call to tempfile.mktemp, but it registers the created @@ -2491,7 +2461,7 @@ class InteractiveShell(Configurable, Magic): - data(None): if data is given, it gets written out to the temp file immediately, and the file is closed again.""" - filename = tempfile.mktemp('.py','ipython_edit_') + filename = tempfile.mktemp('.py', prefix) self.tempfiles.append(filename) if data: diff --git a/IPython/core/tests/test_compilerop.py b/IPython/core/tests/test_compilerop.py new file mode 100644 index 0000000..3d133ae --- /dev/null +++ b/IPython/core/tests/test_compilerop.py @@ -0,0 +1,62 @@ +"""Tests for the compilerop module. +""" +#----------------------------------------------------------------------------- +# Copyright (C) 2010 The IPython Development Team. +# +# Distributed under the terms of the BSD License. +# +# The full license is in the file COPYING.txt, distributed with this software. +#----------------------------------------------------------------------------- + +#----------------------------------------------------------------------------- +# Imports +#----------------------------------------------------------------------------- +from __future__ import print_function + +# Stdlib imports +import linecache + +# Third-party imports +import nose.tools as nt + +# Our own imports +from IPython.core import compilerop + +#----------------------------------------------------------------------------- +# Test functions +#----------------------------------------------------------------------------- + +def test_code_name(): + code = 'x=1' + name = compilerop.code_name(code) + nt.assert_true(name.startswith(' ncache) + + +def test_compiler_check_cache(): + """Test the compiler properly manages the cache. + """ + # Rather simple-minded tests that just exercise the API + cp = compilerop.CachingCompiler() + cp('x=1', 'single', 99) + # Ensure now that after clearing the cache, our entries survive + cp.check_cache() + for k in linecache.cache: + if k.startswith(' 32 bar(mode) - 33 - 34 ... in bar(mode) 14 "bar" @@ -128,8 +126,6 @@ ZeroDivisionError Traceback (most recent call last) ---> 32 bar(mode) global bar = global mode = 'div' - 33 - 34 ... in bar(mode='div') 14 "bar" @@ -186,8 +182,6 @@ SystemExit Traceback (most recent call last) 30 mode = 'div' 31 ---> 32 bar(mode) - 33 - 34 ...bar(mode) 20 except: @@ -218,8 +212,6 @@ SystemExit Traceback (most recent call last) ---> 32 bar(mode) global bar = global mode = 'exit' - 33 - 34 ... in bar(mode='exit') 20 except: diff --git a/IPython/core/ultratb.py b/IPython/core/ultratb.py index 9a1bef2..cfb7b2b 100644 --- a/IPython/core/ultratb.py +++ b/IPython/core/ultratb.py @@ -244,11 +244,6 @@ def _fixed_getinnerframes(etb, context=1,tb_offset=0): start = max(maybeStart, 0) end = start + context lines = linecache.getlines(file)[start:end] - # pad with empty lines if necessary - if maybeStart < 0: - lines = (['\n'] * -maybeStart) + lines - if len(lines) < context: - lines += ['\n'] * (context - len(lines)) buf = list(records[i]) buf[LNUM_POS] = lnum buf[INDEX_POS] = lnum - 1 - start @@ -279,7 +274,15 @@ def _format_traceback_lines(lnum, index, lines, Colors, lvals=None,scheme=None): _line_format = _parser.format2 for line in lines: - new_line, err = _line_format(line,'str',scheme) + # FIXME: we need to ensure the source is a pure string at this point, + # else the coloring code makes a royal mess. This is in need of a + # serious refactoring, so that all of the ultratb and PyColorize code + # is unicode-safe. So for now this is rather an ugly hack, but + # necessary to at least have readable tracebacks. Improvements welcome! + if type(line)==unicode: + line = line.encode('utf-8', 'replace') + + new_line, err = _line_format(line, 'str', scheme) if not err: line = new_line if i == lnum: @@ -636,7 +639,8 @@ class VerboseTB(TBTools): would appear in the traceback).""" def __init__(self,color_scheme = 'Linux', call_pdb=False, ostream=None, - tb_offset=0, long_header=False, include_vars=True): + tb_offset=0, long_header=False, include_vars=True, + check_cache=None): """Specify traceback offset, headers and color scheme. Define how many frames to drop from the tracebacks. Calling it with @@ -648,6 +652,14 @@ class VerboseTB(TBTools): self.tb_offset = tb_offset self.long_header = long_header self.include_vars = include_vars + # By default we use linecache.checkcache, but the user can provide a + # different check_cache implementation. This is used by the IPython + # kernel to provide tracebacks for interactive code that is cached, + # by a compiler instance that flushes the linecache but preserves its + # own code cache. + if check_cache is None: + check_cache = linecache.checkcache + self.check_cache = check_cache def structured_traceback(self, etype, evalue, etb, tb_offset=None, context=5): @@ -723,7 +735,7 @@ class VerboseTB(TBTools): frames = [] # Flush cache before calling inspect. This helps alleviate some of the # problems with python 2.3's inspect.py. - linecache.checkcache() + ##self.check_cache() # Drop topmost frames if requested try: # Try the default getinnerframes and Alex's: Alex's fixes some @@ -1034,7 +1046,8 @@ class FormattedTB(VerboseTB, ListTB): def __init__(self, mode='Plain', color_scheme='Linux', call_pdb=False, ostream=None, - tb_offset=0, long_header=False, include_vars=False): + tb_offset=0, long_header=False, include_vars=False, + check_cache=None): # NEVER change the order of this list. Put new modes at the end: self.valid_modes = ['Plain','Context','Verbose'] @@ -1042,7 +1055,8 @@ class FormattedTB(VerboseTB, ListTB): VerboseTB.__init__(self, color_scheme=color_scheme, call_pdb=call_pdb, ostream=ostream, tb_offset=tb_offset, - long_header=long_header, include_vars=include_vars) + long_header=long_header, include_vars=include_vars, + check_cache=check_cache) # Different types of tracebacks are joined with different separators to # form a single string. They are taken from this dict @@ -1067,7 +1081,7 @@ class FormattedTB(VerboseTB, ListTB): else: # We must check the source cache because otherwise we can print # out-of-date source code. - linecache.checkcache() + self.check_cache() # Now we can extract and format the exception elist = self._extract_tb(tb) return ListTB.structured_traceback( diff --git a/IPython/lib/tests/test_irunner.py b/IPython/lib/tests/test_irunner.py index 320654c..dd8cfd3 100755 --- a/IPython/lib/tests/test_irunner.py +++ b/IPython/lib/tests/test_irunner.py @@ -97,7 +97,7 @@ In [7]: autocall 0 Automatic calling is: OFF In [8]: cos pi - File "", line 1 + File "", line 1 cos pi ^ SyntaxError: invalid syntax