##// END OF EJS Templates
Initial integration of stack_data
Alex Hall -
Show More
This diff has been collapsed as it changes many lines, (514 lines changed) Show them Hide them
@@ -89,23 +89,20 b' Inheritance diagram:'
89 89 #*****************************************************************************
90 90
91 91
92 import dis
93 92 import inspect
94 import keyword
95 93 import linecache
96 import os
97 94 import pydoc
98 import re
99 95 import sys
100 96 import time
101 97 import tokenize
102 98 import traceback
103 99
104 from tokenize import generate_tokens
100 import stack_data
105 101
106 # For purposes of monkeypatching inspect to fix a bug in it.
107 from inspect import getsourcefile, getfile, getmodule, \
108 ismodule, isclass, ismethod, isfunction, istraceback, isframe, iscode
102 try: # Python 2
103 generate_tokens = tokenize.generate_tokens
104 except AttributeError: # Python 3
105 generate_tokens = tokenize.tokenize
109 106
110 107 # IPython's own modules
111 108 from IPython import get_ipython
@@ -115,13 +112,8 b' from IPython.core.excolors import exception_colors'
115 112 from IPython.utils import PyColorize
116 113 from IPython.utils import path as util_path
117 114 from IPython.utils import py3compat
118 from IPython.utils.data import uniq_stable
119 115 from IPython.utils.terminal import get_terminal_size
120 116
121 from logging import info, error, debug
122
123 from importlib.util import source_from_cache
124
125 117 import IPython.utils.colorable as colorable
126 118
127 119 # Globals
@@ -134,264 +126,26 b' INDENT_SIZE = 8'
134 126 # to users of ultratb who are NOT running inside ipython.
135 127 DEFAULT_SCHEME = 'NoColor'
136 128
137
138 # Number of frame above which we are likely to have a recursion and will
139 # **attempt** to detect it. Made modifiable mostly to speedup test suite
140 # as detecting recursion is one of our slowest test
141 _FRAME_RECURSION_LIMIT = 500
142
143 129 # ---------------------------------------------------------------------------
144 130 # Code begins
145 131
146 # Utility functions
147 def inspect_error():
148 """Print a message about internal inspect errors.
149
150 These are unfortunately quite common."""
151
152 error('Internal Python error in the inspect module.\n'
153 'Below is the traceback from this internal error.\n')
154
155
156 # This function is a monkeypatch we apply to the Python inspect module. We have
157 # now found when it's needed (see discussion on issue gh-1456), and we have a
158 # test case (IPython.core.tests.test_ultratb.ChangedPyFileTest) that fails if
159 # the monkeypatch is not applied. TK, Aug 2012.
160 def findsource(object):
161 """Return the entire source file and starting line number for an object.
162
163 The argument may be a module, class, method, function, traceback, frame,
164 or code object. The source code is returned as a list of all the lines
165 in the file and the line number indexes a line in that list. An IOError
166 is raised if the source code cannot be retrieved.
167
168 FIXED version with which we monkeypatch the stdlib to work around a bug."""
169
170 file = getsourcefile(object) or getfile(object)
171 # If the object is a frame, then trying to get the globals dict from its
172 # module won't work. Instead, the frame object itself has the globals
173 # dictionary.
174 globals_dict = None
175 if inspect.isframe(object):
176 # XXX: can this ever be false?
177 globals_dict = object.f_globals
178 else:
179 module = getmodule(object, file)
180 if module:
181 globals_dict = module.__dict__
182 lines = linecache.getlines(file, globals_dict)
183 if not lines:
184 raise IOError('could not get source code')
185
186 if ismodule(object):
187 return lines, 0
188
189 if isclass(object):
190 name = object.__name__
191 pat = re.compile(r'^(\s*)class\s*' + name + r'\b')
192 # make some effort to find the best matching class definition:
193 # use the one with the least indentation, which is the one
194 # that's most probably not inside a function definition.
195 candidates = []
196 for i, line in enumerate(lines):
197 match = pat.match(line)
198 if match:
199 # if it's at toplevel, it's already the best one
200 if line[0] == 'c':
201 return lines, i
202 # else add whitespace to candidate list
203 candidates.append((match.group(1), i))
204 if candidates:
205 # this will sort by whitespace, and by line number,
206 # less whitespace first
207 candidates.sort()
208 return lines, candidates[0][1]
209 else:
210 raise IOError('could not find class definition')
211
212 if ismethod(object):
213 object = object.__func__
214 if isfunction(object):
215 object = object.__code__
216 if istraceback(object):
217 object = object.tb_frame
218 if isframe(object):
219 object = object.f_code
220 if iscode(object):
221 if not hasattr(object, 'co_firstlineno'):
222 raise IOError('could not find function definition')
223 pat = re.compile(r'^(\s*def\s)|(.*(?<!\w)lambda(:|\s))|^(\s*@)')
224 pmatch = pat.match
225 # fperez - fix: sometimes, co_firstlineno can give a number larger than
226 # the length of lines, which causes an error. Safeguard against that.
227 lnum = min(object.co_firstlineno, len(lines)) - 1
228 while lnum > 0:
229 if pmatch(lines[lnum]):
230 break
231 lnum -= 1
232
233 return lines, lnum
234 raise IOError('could not find code object')
235
236
237 # This is a patched version of inspect.getargs that applies the (unmerged)
238 # patch for http://bugs.python.org/issue14611 by Stefano Taschini. This fixes
239 # https://github.com/ipython/ipython/issues/8205 and
240 # https://github.com/ipython/ipython/issues/8293
241 def getargs(co):
242 """Get information about the arguments accepted by a code object.
243
244 Three things are returned: (args, varargs, varkw), where 'args' is
245 a list of argument names (possibly containing nested lists), and
246 'varargs' and 'varkw' are the names of the * and ** arguments or None."""
247 if not iscode(co):
248 raise TypeError('{!r} is not a code object'.format(co))
249
250 nargs = co.co_argcount
251 names = co.co_varnames
252 args = list(names[:nargs])
253 step = 0
254
255 # The following acrobatics are for anonymous (tuple) arguments.
256 for i in range(nargs):
257 if args[i][:1] in ('', '.'):
258 stack, remain, count = [], [], []
259 while step < len(co.co_code):
260 op = ord(co.co_code[step])
261 step = step + 1
262 if op >= dis.HAVE_ARGUMENT:
263 opname = dis.opname[op]
264 value = ord(co.co_code[step]) + ord(co.co_code[step+1])*256
265 step = step + 2
266 if opname in ('UNPACK_TUPLE', 'UNPACK_SEQUENCE'):
267 remain.append(value)
268 count.append(value)
269 elif opname in ('STORE_FAST', 'STORE_DEREF'):
270 if op in dis.haslocal:
271 stack.append(co.co_varnames[value])
272 elif op in dis.hasfree:
273 stack.append((co.co_cellvars + co.co_freevars)[value])
274 # Special case for sublists of length 1: def foo((bar))
275 # doesn't generate the UNPACK_TUPLE bytecode, so if
276 # `remain` is empty here, we have such a sublist.
277 if not remain:
278 stack[0] = [stack[0]]
279 break
280 else:
281 remain[-1] = remain[-1] - 1
282 while remain[-1] == 0:
283 remain.pop()
284 size = count.pop()
285 stack[-size:] = [stack[-size:]]
286 if not remain:
287 break
288 remain[-1] = remain[-1] - 1
289 if not remain:
290 break
291 args[i] = stack[0]
292
293 varargs = None
294 if co.co_flags & inspect.CO_VARARGS:
295 varargs = co.co_varnames[nargs]
296 nargs = nargs + 1
297 varkw = None
298 if co.co_flags & inspect.CO_VARKEYWORDS:
299 varkw = co.co_varnames[nargs]
300 return inspect.Arguments(args, varargs, varkw)
301
302
303 # Monkeypatch inspect to apply our bugfix.
304 def with_patch_inspect(f):
305 """
306 Deprecated since IPython 6.0
307 decorator for monkeypatching inspect.findsource
308 """
309
310 def wrapped(*args, **kwargs):
311 save_findsource = inspect.findsource
312 save_getargs = inspect.getargs
313 inspect.findsource = findsource
314 inspect.getargs = getargs
315 try:
316 return f(*args, **kwargs)
317 finally:
318 inspect.findsource = save_findsource
319 inspect.getargs = save_getargs
320
321 return wrapped
322
323
324 def fix_frame_records_filenames(records):
325 """Try to fix the filenames in each record from inspect.getinnerframes().
326
327 Particularly, modules loaded from within zip files have useless filenames
328 attached to their code object, and inspect.getinnerframes() just uses it.
329 """
330 fixed_records = []
331 for frame, filename, line_no, func_name, lines, index in records:
332 # Look inside the frame's globals dictionary for __file__,
333 # which should be better. However, keep Cython filenames since
334 # we prefer the source filenames over the compiled .so file.
335 if not filename.endswith(('.pyx', '.pxd', '.pxi')):
336 better_fn = frame.f_globals.get('__file__', None)
337 if isinstance(better_fn, str):
338 # Check the type just in case someone did something weird with
339 # __file__. It might also be None if the error occurred during
340 # import.
341 filename = better_fn
342 fixed_records.append((frame, filename, line_no, func_name, lines, index))
343 return fixed_records
344
345
346 @with_patch_inspect
347 def _fixed_getinnerframes(etb, context=1, tb_offset=0):
348 LNUM_POS, LINES_POS, INDEX_POS = 2, 4, 5
349
350 records = fix_frame_records_filenames(inspect.getinnerframes(etb, context))
351 # If the error is at the console, don't build any context, since it would
352 # otherwise produce 5 blank lines printed out (there is no file at the
353 # console)
354 rec_check = records[tb_offset:]
355 try:
356 rname = rec_check[0][1]
357 if rname == '<ipython console>' or rname.endswith('<string>'):
358 return rec_check
359 except IndexError:
360 pass
361
362 aux = traceback.extract_tb(etb)
363 assert len(records) == len(aux)
364 for i, (file, lnum, _, _) in enumerate(aux):
365 maybeStart = lnum - 1 - context // 2
366 start = max(maybeStart, 0)
367 end = start + context
368 lines = linecache.getlines(file)[start:end]
369 buf = list(records[i])
370 buf[LNUM_POS] = lnum
371 buf[INDEX_POS] = lnum - 1 - start
372 buf[LINES_POS] = lines
373 records[i] = tuple(buf)
374 return records[tb_offset:]
375
376 132 # Helper function -- largely belongs to VerboseTB, but we need the same
377 133 # functionality to produce a pseudo verbose TB for SyntaxErrors, so that they
378 134 # can be recognized properly by ipython.el's py-traceback-line-re
379 135 # (SyntaxErrors have to be treated specially because they have no traceback)
380 136
381 137
382 def _format_traceback_lines(lnum, index, lines, Colors, lvals, _line_format):
138 def _format_traceback_lines(lines, Colors, lvals, _line_format):
383 139 """
384 140 Format tracebacks lines with pointing arrow, leading numbers...
385 141
386 142 Parameters
387 143 ==========
388 144
389 lnum: int
390 index: int
391 lines: list[string]
145 lines: list[Line]
392 146 Colors:
393 147 ColorScheme used.
394 lvals: bytes
148 lvals: str
395 149 Values of local variables, already colored, to inject just after the error line.
396 150 _line_format: f (str) -> (str, bool)
397 151 return (colorized version of str, failure to do so)
@@ -399,80 +153,30 b' def _format_traceback_lines(lnum, index, lines, Colors, lvals, _line_format):'
399 153 numbers_width = INDENT_SIZE - 1
400 154 res = []
401 155
402 for i,line in enumerate(lines, lnum-index):
403 line = py3compat.cast_unicode(line)
156 for stack_line in lines:
157 line = stack_line.text.rstrip('\n') + '\n'
404 158
405 159 new_line, err = _line_format(line, 'str')
406 160 if not err:
407 161 line = new_line
408 162
409 if i == lnum:
163 lineno = stack_line.lineno
164 if stack_line.is_current:
410 165 # This is the line with the error
411 pad = numbers_width - len(str(i))
412 num = '%s%s' % (debugger.make_arrow(pad), str(lnum))
166 pad = numbers_width - len(str(lineno))
167 num = '%s%s' % (debugger.make_arrow(pad), str(lineno))
413 168 line = '%s%s%s %s%s' % (Colors.linenoEm, num,
414 169 Colors.line, line, Colors.Normal)
415 170 else:
416 num = '%*s' % (numbers_width, i)
171 num = '%*s' % (numbers_width, lineno)
417 172 line = '%s%s%s %s' % (Colors.lineno, num,
418 173 Colors.Normal, line)
419 174
420 175 res.append(line)
421 if lvals and i == lnum:
176 if lvals and stack_line.is_current:
422 177 res.append(lvals + '\n')
423 178 return res
424 179
425 def is_recursion_error(etype, value, records):
426 try:
427 # RecursionError is new in Python 3.5
428 recursion_error_type = RecursionError
429 except NameError:
430 recursion_error_type = RuntimeError
431
432 # The default recursion limit is 1000, but some of that will be taken up
433 # by stack frames in IPython itself. >500 frames probably indicates
434 # a recursion error.
435 return (etype is recursion_error_type) \
436 and "recursion" in str(value).lower() \
437 and len(records) > _FRAME_RECURSION_LIMIT
438
439 def find_recursion(etype, value, records):
440 """Identify the repeating stack frames from a RecursionError traceback
441
442 'records' is a list as returned by VerboseTB.get_records()
443
444 Returns (last_unique, repeat_length)
445 """
446 # This involves a bit of guesswork - we want to show enough of the traceback
447 # to indicate where the recursion is occurring. We guess that the innermost
448 # quarter of the traceback (250 frames by default) is repeats, and find the
449 # first frame (from in to out) that looks different.
450 if not is_recursion_error(etype, value, records):
451 return len(records), 0
452
453 # Select filename, lineno, func_name to track frames with
454 records = [r[1:4] for r in records]
455 inner_frames = records[-(len(records)//4):]
456 frames_repeated = set(inner_frames)
457
458 last_seen_at = {}
459 longest_repeat = 0
460 i = len(records)
461 for frame in reversed(records):
462 i -= 1
463 if frame not in frames_repeated:
464 last_unique = i
465 break
466
467 if frame in last_seen_at:
468 distance = last_seen_at[frame] - i
469 longest_repeat = max(longest_repeat, distance)
470
471 last_seen_at[frame] = i
472 else:
473 last_unique = 0 # The whole traceback was recursion
474
475 return last_unique, longest_repeat
476 180
477 181 #---------------------------------------------------------------------------
478 182 # Module classes
@@ -880,63 +584,28 b' class VerboseTB(TBTools):'
880 584
881 585 self.debugger_cls = debugger_cls or debugger.Pdb
882 586
883 def format_records(self, records, last_unique, recursion_repeat):
884 """Format the stack frames of the traceback"""
885 frames = []
886 for r in records[:last_unique+recursion_repeat+1]:
887 #print '*** record:',file,lnum,func,lines,index # dbg
888 frames.append(self.format_record(*r))
889
890 if recursion_repeat:
891 frames.append('... last %d frames repeated, from the frame below ...\n' % recursion_repeat)
892 frames.append(self.format_record(*records[last_unique+recursion_repeat+1]))
893
894 return frames
895
896 def format_record(self, frame, file, lnum, func, lines, index):
587 def format_record(self, frame_info):
897 588 """Format a single stack frame"""
898 589 Colors = self.Colors # just a shorthand + quicker name lookup
899 590 ColorsNormal = Colors.Normal # used a lot
900 591 col_scheme = self.color_scheme_table.active_scheme_name
901 592 indent = ' ' * INDENT_SIZE
902 593 em_normal = '%s\n%s%s' % (Colors.valEm, indent, ColorsNormal)
903 undefined = '%sundefined%s' % (Colors.em, ColorsNormal)
904 594 tpl_link = '%s%%s%s' % (Colors.filenameEm, ColorsNormal)
905 595 tpl_call = 'in %s%%s%s%%s%s' % (Colors.vName, Colors.valEm,
906 596 ColorsNormal)
907 597 tpl_call_fail = 'in %s%%s%s(***failed resolving arguments***)%s' % \
908 598 (Colors.vName, Colors.valEm, ColorsNormal)
909 599 tpl_local_var = '%s%%s%s' % (Colors.vName, ColorsNormal)
910 tpl_global_var = '%sglobal%s %s%%s%s' % (Colors.em, ColorsNormal,
911 Colors.vName, ColorsNormal)
912 600 tpl_name_val = '%%s %s= %%s%s' % (Colors.valEm, ColorsNormal)
913 601
914 if not file:
915 file = '?'
916 elif file.startswith(str("<")) and file.endswith(str(">")):
917 # Not a real filename, no problem...
918 pass
919 elif not os.path.isabs(file):
920 # Try to make the filename absolute by trying all
921 # sys.path entries (which is also what linecache does)
922 for dirname in sys.path:
923 try:
924 fullname = os.path.join(dirname, file)
925 if os.path.isfile(fullname):
926 file = os.path.abspath(fullname)
927 break
928 except Exception:
929 # Just in case that sys.path contains very
930 # strange entries...
931 pass
932
602 file = frame_info.filename
933 603 file = py3compat.cast_unicode(file, util_path.fs_encoding)
934 604 link = tpl_link % util_path.compress_user(file)
935 args, varargs, varkw, locals_ = inspect.getargvalues(frame)
605 args, varargs, varkw, locals_ = inspect.getargvalues(frame_info.frame)
936 606
937 if func == '?':
938 call = ''
939 elif func == '<module>':
607 func = frame_info.executing.code_qualname()
608 if func == '<module>':
940 609 call = tpl_call % (func, '')
941 610 else:
942 611 # Decide whether to include variable details or not
@@ -964,111 +633,19 b' class VerboseTB(TBTools):'
964 633 # disabled.
965 634 call = tpl_call_fail % func
966 635
967 # Don't attempt to tokenize binary files.
968 if file.endswith(('.so', '.pyd', '.dll')):
969 return '%s %s\n' % (link, call)
970
971 elif file.endswith(('.pyc', '.pyo')):
972 # Look up the corresponding source file.
973 try:
974 file = source_from_cache(file)
975 except ValueError:
976 # Failed to get the source file for some reason
977 # E.g. https://github.com/ipython/ipython/issues/9486
978 return '%s %s\n' % (link, call)
979
980 def linereader(file=file, lnum=[lnum], getline=linecache.getline):
981 line = getline(file, lnum[0])
982 lnum[0] += 1
983 return line
984
985 # Build the list of names on this line of code where the exception
986 # occurred.
987 try:
988 names = []
989 name_cont = False
990
991 for token_type, token, start, end, line in generate_tokens(linereader):
992 # build composite names
993 if token_type == tokenize.NAME and token not in keyword.kwlist:
994 if name_cont:
995 # Continuation of a dotted name
996 try:
997 names[-1].append(token)
998 except IndexError:
999 names.append([token])
1000 name_cont = False
1001 else:
1002 # Regular new names. We append everything, the caller
1003 # will be responsible for pruning the list later. It's
1004 # very tricky to try to prune as we go, b/c composite
1005 # names can fool us. The pruning at the end is easy
1006 # to do (or the caller can print a list with repeated
1007 # names if so desired.
1008 names.append([token])
1009 elif token == '.':
1010 name_cont = True
1011 elif token_type == tokenize.NEWLINE:
1012 break
1013
1014 except (IndexError, UnicodeDecodeError, SyntaxError):
1015 # signals exit of tokenizer
1016 # SyntaxError can occur if the file is not actually Python
1017 # - see gh-6300
1018 pass
1019 except tokenize.TokenError as msg:
1020 # Tokenizing may fail for various reasons, many of which are
1021 # harmless. (A good example is when the line in question is the
1022 # close of a triple-quoted string, cf gh-6864). We don't want to
1023 # show this to users, but want make it available for debugging
1024 # purposes.
1025 _m = ("An unexpected error occurred while tokenizing input\n"
1026 "The following traceback may be corrupted or invalid\n"
1027 "The error message is: %s\n" % msg)
1028 debug(_m)
1029
1030 # Join composite names (e.g. "dict.fromkeys")
1031 names = ['.'.join(n) for n in names]
1032 # prune names list of duplicates, but keep the right order
1033 unique_names = uniq_stable(names)
1034
1035 # Start loop over vars
1036 636 lvals = ''
1037 637 lvals_list = []
1038 638 if self.include_vars:
1039 for name_full in unique_names:
1040 name_base = name_full.split('.', 1)[0]
1041 if name_base in frame.f_code.co_varnames:
1042 if name_base in locals_:
1043 try:
1044 value = repr(eval(name_full, locals_))
1045 except:
1046 value = undefined
1047 else:
1048 value = undefined
1049 name = tpl_local_var % name_full
1050 else:
1051 if name_base in frame.f_globals:
1052 try:
1053 value = repr(eval(name_full, frame.f_globals))
1054 except:
1055 value = undefined
1056 else:
1057 value = undefined
1058 name = tpl_global_var % name_full
1059 lvals_list.append(tpl_name_val % (name, value))
639 for var in frame_info.variables_in_executing_piece:
640 lvals_list.append(tpl_name_val % (var.name, var.value))
1060 641 if lvals_list:
1061 642 lvals = '%s%s' % (indent, em_normal.join(lvals_list))
1062 643
1063 level = '%s %s\n' % (link, call)
644 result = '%s %s\n' % (link, call)
1064 645
1065 if index is None:
1066 return level
1067 else:
1068 646 _line_format = PyColorize.Parser(style=col_scheme, parent=self).format2
1069 return '%s%s' % (level, ''.join(
1070 _format_traceback_lines(lnum, index, lines, Colors, lvals,
1071 _line_format)))
647 result += ''.join(_format_traceback_lines(frame_info.lines, Colors, lvals, _line_format))
648 return result
1072 649
1073 650 def prepare_header(self, etype, long_version=False):
1074 651 colors = self.Colors # just a shorthand + quicker name lookup
@@ -1123,46 +700,23 b' class VerboseTB(TBTools):'
1123 700 head = self.prepare_header(etype, self.long_header)
1124 701 records = self.get_records(etb, number_of_lines_of_context, tb_offset)
1125 702
1126 if records is None:
1127 return ""
1128
1129 last_unique, recursion_repeat = find_recursion(orig_etype, evalue, records)
1130
1131 frames = self.format_records(records, last_unique, recursion_repeat)
703 frames = list(map(self.format_record, records))
1132 704
1133 705 formatted_exception = self.format_exception(etype, evalue)
1134 706 if records:
1135 filepath, lnum = records[-1][1:3]
1136 filepath = os.path.abspath(filepath)
707 frame_info = records[-1]
1137 708 ipinst = get_ipython()
1138 709 if ipinst is not None:
1139 ipinst.hooks.synchronize_with_editor(filepath, lnum, 0)
710 ipinst.hooks.synchronize_with_editor(frame_info.filename, frame_info.lineno, 0)
1140 711
1141 712 return [[head] + frames + [''.join(formatted_exception[0])]]
1142 713
1143 714 def get_records(self, etb, number_of_lines_of_context, tb_offset):
1144 try:
1145 # Try the default getinnerframes and Alex's: Alex's fixes some
1146 # problems, but it generates empty tracebacks for console errors
1147 # (5 blanks lines) where none should be returned.
1148 return _fixed_getinnerframes(etb, number_of_lines_of_context, tb_offset)
1149 except UnicodeDecodeError:
1150 # This can occur if a file's encoding magic comment is wrong.
1151 # I can't see a way to recover without duplicating a bunch of code
1152 # from the stdlib traceback module. --TK
1153 error('\nUnicodeDecodeError while processing traceback.\n')
1154 return None
1155 except:
1156 # FIXME: I've been getting many crash reports from python 2.3
1157 # users, traceable to inspect.py. If I can find a small test-case
1158 # to reproduce this, I should either write a better workaround or
1159 # file a bug report against inspect (if that's the real problem).
1160 # So far, I haven't been able to find an isolated example to
1161 # reproduce the problem.
1162 inspect_error()
1163 traceback.print_exc(file=self.ostream)
1164 info('\nUnfortunately, your original traceback can not be constructed.\n')
1165 return None
715 context = number_of_lines_of_context - 1
716 after = context // 2
717 before = context - after
718 options = stack_data.Options(before=before, after=after)
719 return list(stack_data.FrameInfo.stack_data(etb, options=options))[tb_offset:]
1166 720
1167 721 def structured_traceback(self, etype, evalue, etb, tb_offset=None,
1168 722 number_of_lines_of_context=5):
General Comments 0
You need to be logged in to leave comments. Login now