Show More
@@ -0,0 +1,119 b'' | |||
|
1 | """Compiler tools with improved interactive support. | |
|
2 | ||
|
3 | Provides compilation machinery similar to codeop, but with caching support so | |
|
4 | we can provide interactive tracebacks. | |
|
5 | ||
|
6 | Authors | |
|
7 | ------- | |
|
8 | * Robert Kern | |
|
9 | * Fernando Perez | |
|
10 | """ | |
|
11 | ||
|
12 | # Note: though it might be more natural to name this module 'compiler', that | |
|
13 | # name is in the stdlib and name collisions with the stdlib tend to produce | |
|
14 | # weird problems (often with third-party tools). | |
|
15 | ||
|
16 | #----------------------------------------------------------------------------- | |
|
17 | # Copyright (C) 2010 The IPython Development Team. | |
|
18 | # | |
|
19 | # Distributed under the terms of the BSD License. | |
|
20 | # | |
|
21 | # The full license is in the file COPYING.txt, distributed with this software. | |
|
22 | #----------------------------------------------------------------------------- | |
|
23 | ||
|
24 | #----------------------------------------------------------------------------- | |
|
25 | # Imports | |
|
26 | #----------------------------------------------------------------------------- | |
|
27 | from __future__ import print_function | |
|
28 | ||
|
29 | # Stdlib imports | |
|
30 | import codeop | |
|
31 | import hashlib | |
|
32 | import linecache | |
|
33 | import time | |
|
34 | ||
|
35 | #----------------------------------------------------------------------------- | |
|
36 | # Local utilities | |
|
37 | #----------------------------------------------------------------------------- | |
|
38 | ||
|
39 | def code_name(code, number=0): | |
|
40 | """ Compute a (probably) unique name for code for caching. | |
|
41 | """ | |
|
42 | hash_digest = hashlib.md5(code).hexdigest() | |
|
43 | # Include the number and 12 characters of the hash in the name. It's | |
|
44 | # pretty much impossible that in a single session we'll have collisions | |
|
45 | # even with truncated hashes, and the full one makes tracebacks too long | |
|
46 | return '<ipython-input-{0}-{1}>'.format(number, hash_digest[:12]) | |
|
47 | ||
|
48 | #----------------------------------------------------------------------------- | |
|
49 | # Classes and functions | |
|
50 | #----------------------------------------------------------------------------- | |
|
51 | ||
|
52 | class CachingCompiler(object): | |
|
53 | """A compiler that caches code compiled from interactive statements. | |
|
54 | """ | |
|
55 | ||
|
56 | def __init__(self): | |
|
57 | self._compiler = codeop.CommandCompiler() | |
|
58 | ||
|
59 | # This is ugly, but it must be done this way to allow multiple | |
|
60 | # simultaneous ipython instances to coexist. Since Python itself | |
|
61 | # directly accesses the data structures in the linecache module, and | |
|
62 | # the cache therein is global, we must work with that data structure. | |
|
63 | # We must hold a reference to the original checkcache routine and call | |
|
64 | # that in our own check_cache() below, but the special IPython cache | |
|
65 | # must also be shared by all IPython instances. If we were to hold | |
|
66 | # separate caches (one in each CachingCompiler instance), any call made | |
|
67 | # by Python itself to linecache.checkcache() would obliterate the | |
|
68 | # cached data from the other IPython instances. | |
|
69 | if not hasattr(linecache, '_ipython_cache'): | |
|
70 | linecache._ipython_cache = {} | |
|
71 | if not hasattr(linecache, '_checkcache_ori'): | |
|
72 | linecache._checkcache_ori = linecache.checkcache | |
|
73 | # Now, we must monkeypatch the linecache directly so that parts of the | |
|
74 | # stdlib that call it outside our control go through our codepath | |
|
75 | # (otherwise we'd lose our tracebacks). | |
|
76 | linecache.checkcache = self.check_cache | |
|
77 | ||
|
78 | @property | |
|
79 | def compiler_flags(self): | |
|
80 | """Flags currently active in the compilation process. | |
|
81 | """ | |
|
82 | return self._compiler.compiler.flags | |
|
83 | ||
|
84 | def __call__(self, code, symbol, number=0): | |
|
85 | """Compile some code while caching its contents such that the inspect | |
|
86 | module can find it later. | |
|
87 | ||
|
88 | Parameters | |
|
89 | ---------- | |
|
90 | code : str | |
|
91 | Source code to be compiled, one or more lines. | |
|
92 | ||
|
93 | symbol : str | |
|
94 | One of 'single', 'exec' or 'eval' (see the builtin ``compile`` | |
|
95 | documentation for further details on these fields). | |
|
96 | ||
|
97 | number : int, optional | |
|
98 | An integer argument identifying the code, useful for informational | |
|
99 | purposes in tracebacks (typically it will be the IPython prompt | |
|
100 | number). | |
|
101 | """ | |
|
102 | name = code_name(code, number) | |
|
103 | code_obj = self._compiler(code, name, symbol) | |
|
104 | entry = (len(code), time.time(), | |
|
105 | [line+'\n' for line in code.splitlines()], name) | |
|
106 | # Cache the info both in the linecache (a global cache used internally | |
|
107 | # by most of Python's inspect/traceback machinery), and in our cache | |
|
108 | linecache.cache[name] = entry | |
|
109 | linecache._ipython_cache[name] = entry | |
|
110 | return code_obj | |
|
111 | ||
|
112 | def check_cache(self, *args): | |
|
113 | """Call linecache.checkcache() safely protecting our cached values. | |
|
114 | """ | |
|
115 | # First call the orignal checkcache as intended | |
|
116 | linecache._checkcache_ori(*args) | |
|
117 | # Then, update back the cache with our data, so that tracebacks related | |
|
118 | # to our compiled codes can be produced. | |
|
119 | linecache.cache.update(linecache._ipython_cache) |
@@ -0,0 +1,62 b'' | |||
|
1 | """Tests for the compilerop module. | |
|
2 | """ | |
|
3 | #----------------------------------------------------------------------------- | |
|
4 | # Copyright (C) 2010 The IPython Development Team. | |
|
5 | # | |
|
6 | # Distributed under the terms of the BSD License. | |
|
7 | # | |
|
8 | # The full license is in the file COPYING.txt, distributed with this software. | |
|
9 | #----------------------------------------------------------------------------- | |
|
10 | ||
|
11 | #----------------------------------------------------------------------------- | |
|
12 | # Imports | |
|
13 | #----------------------------------------------------------------------------- | |
|
14 | from __future__ import print_function | |
|
15 | ||
|
16 | # Stdlib imports | |
|
17 | import linecache | |
|
18 | ||
|
19 | # Third-party imports | |
|
20 | import nose.tools as nt | |
|
21 | ||
|
22 | # Our own imports | |
|
23 | from IPython.core import compilerop | |
|
24 | ||
|
25 | #----------------------------------------------------------------------------- | |
|
26 | # Test functions | |
|
27 | #----------------------------------------------------------------------------- | |
|
28 | ||
|
29 | def test_code_name(): | |
|
30 | code = 'x=1' | |
|
31 | name = compilerop.code_name(code) | |
|
32 | nt.assert_true(name.startswith('<ipython-input-0')) | |
|
33 | ||
|
34 | ||
|
35 | def test_code_name2(): | |
|
36 | code = 'x=1' | |
|
37 | name = compilerop.code_name(code, 9) | |
|
38 | nt.assert_true(name.startswith('<ipython-input-9')) | |
|
39 | ||
|
40 | ||
|
41 | def test_compiler(): | |
|
42 | """Test the compiler correctly compiles and caches inputs | |
|
43 | """ | |
|
44 | cp = compilerop.CachingCompiler() | |
|
45 | ncache = len(linecache.cache) | |
|
46 | cp('x=1', 'single') | |
|
47 | nt.assert_true(len(linecache.cache) > ncache) | |
|
48 | ||
|
49 | ||
|
50 | def test_compiler_check_cache(): | |
|
51 | """Test the compiler properly manages the cache. | |
|
52 | """ | |
|
53 | # Rather simple-minded tests that just exercise the API | |
|
54 | cp = compilerop.CachingCompiler() | |
|
55 | cp('x=1', 'single', 99) | |
|
56 | # Ensure now that after clearing the cache, our entries survive | |
|
57 | cp.check_cache() | |
|
58 | for k in linecache.cache: | |
|
59 | if k.startswith('<ipython-input-99'): | |
|
60 | break | |
|
61 | else: | |
|
62 | raise AssertionError('Entry for input-99 missing from linecache') |
@@ -40,6 +40,7 b' from IPython.core import shadowns' | |||
|
40 | 40 | from IPython.core import ultratb |
|
41 | 41 | from IPython.core.alias import AliasManager |
|
42 | 42 | from IPython.core.builtin_trap import BuiltinTrap |
|
43 | from IPython.core.compilerop import CachingCompiler | |
|
43 | 44 | from IPython.core.display_trap import DisplayTrap |
|
44 | 45 | from IPython.core.displayhook import DisplayHook |
|
45 | 46 | from IPython.core.error import TryNext, UsageError |
@@ -136,39 +137,6 b' class MultipleInstanceError(Exception):' | |||
|
136 | 137 | # Main IPython class |
|
137 | 138 | #----------------------------------------------------------------------------- |
|
138 | 139 | |
|
139 | ||
|
140 | ######## Code to be moved later if it works, meant to try to get proper | |
|
141 | ######## tracebacks | |
|
142 | ||
|
143 | import hashlib | |
|
144 | import linecache | |
|
145 | import time | |
|
146 | import types | |
|
147 | ||
|
148 | def code_name(code): | |
|
149 | """ Compute a (probably) unique name for code for caching. | |
|
150 | """ | |
|
151 | hash_digest = hashlib.md5(code).hexdigest() | |
|
152 | return '<code %s>' % hash_digest | |
|
153 | ||
|
154 | ||
|
155 | class CachingCompiler(codeop.CommandCompiler): | |
|
156 | ||
|
157 | def __call__(self, code, filename, symbol): | |
|
158 | """ Compile some code while caching its contents such that the inspect | |
|
159 | module can find it later. | |
|
160 | """ | |
|
161 | #code += '\n' | |
|
162 | name = code_name(code) | |
|
163 | code_obj = codeop.CommandCompiler.__call__(self, code, name, symbol) | |
|
164 | linecache.cache[name] = (len(code), | |
|
165 | time.time(), | |
|
166 | [line+'\n' for line in code.splitlines()], | |
|
167 | name) | |
|
168 | return code_obj | |
|
169 | ||
|
170 | ############# | |
|
171 | ||
|
172 | 140 | class InteractiveShell(Configurable, Magic): |
|
173 | 141 | """An enhanced, interactive shell for Python.""" |
|
174 | 142 | |
@@ -401,7 +369,6 b' class InteractiveShell(Configurable, Magic):' | |||
|
401 | 369 | self.more = False |
|
402 | 370 | |
|
403 | 371 | # command compiler |
|
404 | #self.compile = codeop.CommandCompiler() | |
|
405 | 372 | self.compile = CachingCompiler() |
|
406 | 373 | |
|
407 | 374 | # User input buffers |
@@ -1134,7 +1101,7 b' class InteractiveShell(Configurable, Magic):' | |||
|
1134 | 1101 | # We need to special-case 'print', which as of python2.6 registers as a |
|
1135 | 1102 | # function but should only be treated as one if print_function was |
|
1136 | 1103 | # loaded with a future import. In this case, just bail. |
|
1137 |
if (oname == 'print' and not (self.compile.compiler |
|
|
1104 | if (oname == 'print' and not (self.compile.compiler_flags & | |
|
1138 | 1105 | __future__.CO_FUTURE_PRINT_FUNCTION)): |
|
1139 | 1106 | return {'found':found, 'obj':obj, 'namespace':ospace, |
|
1140 | 1107 | 'ismagic':ismagic, 'isalias':isalias, 'parent':parent} |
@@ -1293,7 +1260,8 b' class InteractiveShell(Configurable, Magic):' | |||
|
1293 | 1260 | # internal code. Valid modes: ['Plain','Context','Verbose'] |
|
1294 | 1261 | self.InteractiveTB = ultratb.AutoFormattedTB(mode = 'Plain', |
|
1295 | 1262 | color_scheme='NoColor', |
|
1296 |
tb_offset = 1 |
|
|
1263 | tb_offset = 1, | |
|
1264 | check_cache=self.compile.check_cache) | |
|
1297 | 1265 | |
|
1298 | 1266 | # The instance will store a pointer to the system-wide exception hook, |
|
1299 | 1267 | # so that runtime code (such as magics) can access it. This is because |
@@ -2159,14 +2127,15 b' class InteractiveShell(Configurable, Magic):' | |||
|
2159 | 2127 | |
|
2160 | 2128 | # Get the main body to run as a cell |
|
2161 | 2129 | ipy_body = ''.join(blocks[:-1]) |
|
2162 |
retcode = self.run_ |
|
|
2130 | retcode = self.run_source(ipy_body, symbol='exec', | |
|
2131 | post_execute=False) | |
|
2163 | 2132 | if retcode==0: |
|
2164 | 2133 | # And the last expression via runlines so it produces output |
|
2165 | 2134 | self.run_one_block(last) |
|
2166 | 2135 | else: |
|
2167 | 2136 | # Run the whole cell as one entity, storing both raw and |
|
2168 | 2137 | # processed input in history |
|
2169 |
self.run_ |
|
|
2138 | self.run_source(ipy_cell, symbol='exec') | |
|
2170 | 2139 | |
|
2171 | 2140 | # Each cell is a *single* input, regardless of how many lines it has |
|
2172 | 2141 | self.execution_count += 1 |
@@ -2242,7 +2211,8 b' class InteractiveShell(Configurable, Magic):' | |||
|
2242 | 2211 | if more: |
|
2243 | 2212 | self.push_line('\n') |
|
2244 | 2213 | |
|
2245 |
def run_source(self, source, filename= |
|
|
2214 | def run_source(self, source, filename=None, | |
|
2215 | symbol='single', post_execute=True): | |
|
2246 | 2216 | """Compile and run some source in the interpreter. |
|
2247 | 2217 | |
|
2248 | 2218 | Arguments are as for compile_command(). |
@@ -2284,7 +2254,7 b' class InteractiveShell(Configurable, Magic):' | |||
|
2284 | 2254 | print 'encoding', self.stdin_encoding # dbg |
|
2285 | 2255 | |
|
2286 | 2256 | try: |
|
2287 |
code = self.compile(usource, |
|
|
2257 | code = self.compile(usource, symbol, self.execution_count) | |
|
2288 | 2258 | except (OverflowError, SyntaxError, ValueError, TypeError, MemoryError): |
|
2289 | 2259 | # Case 1 |
|
2290 | 2260 | self.showsyntaxerror(filename) |
@@ -2301,7 +2271,7 b' class InteractiveShell(Configurable, Magic):' | |||
|
2301 | 2271 | # buffer attribute as '\n'.join(self.buffer). |
|
2302 | 2272 | self.code_to_run = code |
|
2303 | 2273 | # now actually execute the code object |
|
2304 | if self.run_code(code) == 0: | |
|
2274 | if self.run_code(code, post_execute) == 0: | |
|
2305 | 2275 | return False |
|
2306 | 2276 | else: |
|
2307 | 2277 | return None |
@@ -2480,7 +2450,7 b' class InteractiveShell(Configurable, Magic):' | |||
|
2480 | 2450 | sys._getframe(depth+1).f_locals # locals |
|
2481 | 2451 | )) |
|
2482 | 2452 | |
|
2483 | def mktempfile(self,data=None): | |
|
2453 | def mktempfile(self, data=None, prefix='ipython_edit_'): | |
|
2484 | 2454 | """Make a new tempfile and return its filename. |
|
2485 | 2455 | |
|
2486 | 2456 | This makes a call to tempfile.mktemp, but it registers the created |
@@ -2491,7 +2461,7 b' class InteractiveShell(Configurable, Magic):' | |||
|
2491 | 2461 | - data(None): if data is given, it gets written out to the temp file |
|
2492 | 2462 | immediately, and the file is closed again.""" |
|
2493 | 2463 | |
|
2494 |
filename = tempfile.mktemp('.py', |
|
|
2464 | filename = tempfile.mktemp('.py', prefix) | |
|
2495 | 2465 | self.tempfiles.append(filename) |
|
2496 | 2466 | |
|
2497 | 2467 | if data: |
@@ -92,8 +92,6 b' ZeroDivisionError Traceback (most recent call last)' | |||
|
92 | 92 | 30 mode = 'div' |
|
93 | 93 | 31 |
|
94 | 94 | ---> 32 bar(mode) |
|
95 | 33 | |
|
96 | 34 | |
|
97 | 95 | <BLANKLINE> |
|
98 | 96 | ... in bar(mode) |
|
99 | 97 | 14 "bar" |
@@ -128,8 +126,6 b' ZeroDivisionError Traceback (most recent call last)' | |||
|
128 | 126 | ---> 32 bar(mode) |
|
129 | 127 | global bar = <function bar at ...> |
|
130 | 128 | global mode = 'div' |
|
131 | 33 | |
|
132 | 34 | |
|
133 | 129 | <BLANKLINE> |
|
134 | 130 | ... in bar(mode='div') |
|
135 | 131 | 14 "bar" |
@@ -186,8 +182,6 b' SystemExit Traceback (most recent call last)' | |||
|
186 | 182 | 30 mode = 'div' |
|
187 | 183 | 31 |
|
188 | 184 | ---> 32 bar(mode) |
|
189 | 33 | |
|
190 | 34 | |
|
191 | 185 | <BLANKLINE> |
|
192 | 186 | ...bar(mode) |
|
193 | 187 | 20 except: |
@@ -218,8 +212,6 b' SystemExit Traceback (most recent call last)' | |||
|
218 | 212 | ---> 32 bar(mode) |
|
219 | 213 | global bar = <function bar at ...> |
|
220 | 214 | global mode = 'exit' |
|
221 | 33 | |
|
222 | 34 | |
|
223 | 215 | <BLANKLINE> |
|
224 | 216 | ... in bar(mode='exit') |
|
225 | 217 | 20 except: |
@@ -244,11 +244,6 b' def _fixed_getinnerframes(etb, context=1,tb_offset=0):' | |||
|
244 | 244 | start = max(maybeStart, 0) |
|
245 | 245 | end = start + context |
|
246 | 246 | lines = linecache.getlines(file)[start:end] |
|
247 | # pad with empty lines if necessary | |
|
248 | if maybeStart < 0: | |
|
249 | lines = (['\n'] * -maybeStart) + lines | |
|
250 | if len(lines) < context: | |
|
251 | lines += ['\n'] * (context - len(lines)) | |
|
252 | 247 | buf = list(records[i]) |
|
253 | 248 | buf[LNUM_POS] = lnum |
|
254 | 249 | buf[INDEX_POS] = lnum - 1 - start |
@@ -279,7 +274,15 b' def _format_traceback_lines(lnum, index, lines, Colors, lvals=None,scheme=None):' | |||
|
279 | 274 | _line_format = _parser.format2 |
|
280 | 275 | |
|
281 | 276 | for line in lines: |
|
282 | new_line, err = _line_format(line,'str',scheme) | |
|
277 | # FIXME: we need to ensure the source is a pure string at this point, | |
|
278 | # else the coloring code makes a royal mess. This is in need of a | |
|
279 | # serious refactoring, so that all of the ultratb and PyColorize code | |
|
280 | # is unicode-safe. So for now this is rather an ugly hack, but | |
|
281 | # necessary to at least have readable tracebacks. Improvements welcome! | |
|
282 | if type(line)==unicode: | |
|
283 | line = line.encode('utf-8', 'replace') | |
|
284 | ||
|
285 | new_line, err = _line_format(line, 'str', scheme) | |
|
283 | 286 | if not err: line = new_line |
|
284 | 287 | |
|
285 | 288 | if i == lnum: |
@@ -636,7 +639,8 b' class VerboseTB(TBTools):' | |||
|
636 | 639 | would appear in the traceback).""" |
|
637 | 640 | |
|
638 | 641 | def __init__(self,color_scheme = 'Linux', call_pdb=False, ostream=None, |
|
639 |
tb_offset=0, long_header=False, include_vars=True |
|
|
642 | tb_offset=0, long_header=False, include_vars=True, | |
|
643 | check_cache=None): | |
|
640 | 644 | """Specify traceback offset, headers and color scheme. |
|
641 | 645 | |
|
642 | 646 | Define how many frames to drop from the tracebacks. Calling it with |
@@ -648,6 +652,14 b' class VerboseTB(TBTools):' | |||
|
648 | 652 | self.tb_offset = tb_offset |
|
649 | 653 | self.long_header = long_header |
|
650 | 654 | self.include_vars = include_vars |
|
655 | # By default we use linecache.checkcache, but the user can provide a | |
|
656 | # different check_cache implementation. This is used by the IPython | |
|
657 | # kernel to provide tracebacks for interactive code that is cached, | |
|
658 | # by a compiler instance that flushes the linecache but preserves its | |
|
659 | # own code cache. | |
|
660 | if check_cache is None: | |
|
661 | check_cache = linecache.checkcache | |
|
662 | self.check_cache = check_cache | |
|
651 | 663 | |
|
652 | 664 | def structured_traceback(self, etype, evalue, etb, tb_offset=None, |
|
653 | 665 | context=5): |
@@ -723,7 +735,7 b' class VerboseTB(TBTools):' | |||
|
723 | 735 | frames = [] |
|
724 | 736 | # Flush cache before calling inspect. This helps alleviate some of the |
|
725 | 737 | # problems with python 2.3's inspect.py. |
|
726 |
|
|
|
738 | ##self.check_cache() | |
|
727 | 739 | # Drop topmost frames if requested |
|
728 | 740 | try: |
|
729 | 741 | # Try the default getinnerframes and Alex's: Alex's fixes some |
@@ -1034,7 +1046,8 b' class FormattedTB(VerboseTB, ListTB):' | |||
|
1034 | 1046 | |
|
1035 | 1047 | def __init__(self, mode='Plain', color_scheme='Linux', call_pdb=False, |
|
1036 | 1048 | ostream=None, |
|
1037 |
tb_offset=0, long_header=False, include_vars=False |
|
|
1049 | tb_offset=0, long_header=False, include_vars=False, | |
|
1050 | check_cache=None): | |
|
1038 | 1051 | |
|
1039 | 1052 | # NEVER change the order of this list. Put new modes at the end: |
|
1040 | 1053 | self.valid_modes = ['Plain','Context','Verbose'] |
@@ -1042,7 +1055,8 b' class FormattedTB(VerboseTB, ListTB):' | |||
|
1042 | 1055 | |
|
1043 | 1056 | VerboseTB.__init__(self, color_scheme=color_scheme, call_pdb=call_pdb, |
|
1044 | 1057 | ostream=ostream, tb_offset=tb_offset, |
|
1045 |
long_header=long_header, include_vars=include_vars |
|
|
1058 | long_header=long_header, include_vars=include_vars, | |
|
1059 | check_cache=check_cache) | |
|
1046 | 1060 | |
|
1047 | 1061 | # Different types of tracebacks are joined with different separators to |
|
1048 | 1062 | # form a single string. They are taken from this dict |
@@ -1067,7 +1081,7 b' class FormattedTB(VerboseTB, ListTB):' | |||
|
1067 | 1081 | else: |
|
1068 | 1082 | # We must check the source cache because otherwise we can print |
|
1069 | 1083 | # out-of-date source code. |
|
1070 |
|
|
|
1084 | self.check_cache() | |
|
1071 | 1085 | # Now we can extract and format the exception |
|
1072 | 1086 | elist = self._extract_tb(tb) |
|
1073 | 1087 | return ListTB.structured_traceback( |
General Comments 0
You need to be logged in to leave comments.
Login now