From 3362771d2dddb2429993ba7f0aa67cc2c6a98fd1 2023-02-21 09:14:54 From: Matthias Bussonnier Date: 2023-02-21 09:14:54 Subject: [PATCH] FEAT: Fast TB. Try to detect when one of the file we are trying to highlight is too big, and fallback on pre-8.0 traceback code, that avoids stack_data. You can configure the limit with: >>> from IPython.core import ultratb >>> ultratb.FAST_THRESHOLD = 200 We are trying to _guess_ whether the traceback we will render is in a file that will require to parse more than FAST_THRESHOLD lines, though it is actually difficult to get this value correctly, so in some case, the slow path may be used, even if files are longer than FAST_THRESHOLD, especially if the crashing code is at the beginning of the file. --- diff --git a/IPython/core/ultratb.py b/IPython/core/ultratb.py index 18eff27..23b4eae 100644 --- a/IPython/core/ultratb.py +++ b/IPython/core/ultratb.py @@ -99,6 +99,7 @@ from types import TracebackType from typing import Tuple, List, Any, Optional import stack_data +from stack_data import FrameInfo as SDFrameInfo from pygments.formatters.terminal256 import Terminal256Formatter from pygments.styles import get_style_by_name @@ -107,6 +108,7 @@ from IPython import get_ipython from IPython.core import debugger from IPython.core.display_trap import DisplayTrap from IPython.core.excolors import exception_colors +from IPython.utils import PyColorize from IPython.utils import path as util_path from IPython.utils import py3compat from IPython.utils.terminal import get_terminal_size @@ -122,6 +124,7 @@ INDENT_SIZE = 8 # value is used, but having a module global makes this functionality available # to users of ultratb who are NOT running inside ipython. DEFAULT_SCHEME = 'NoColor' +FAST_THRESHOLD = 10_000 # --------------------------------------------------------------------------- # Code begins @@ -170,6 +173,50 @@ def _format_traceback_lines(lines, Colors, has_colors: bool, lvals): res.append(lvals + '\n') return res +def _simple_format_traceback_lines(lnum, index, lines, Colors, lvals, _line_format): + """ + Format tracebacks lines with pointing arrow, leading numbers... + + Parameters + ========== + + lnum: int + number of the target line of code. + index: int + which line in the list should be highlighted. + lines: list[string] + Colors: + ColorScheme used. + lvals: bytes + Values of local variables, already colored, to inject just after the error line. + _line_format: f (str) -> (str, bool) + return (colorized version of str, failure to do so) + """ + numbers_width = INDENT_SIZE - 1 + res = [] + + for i,line in enumerate(lines, lnum-index): + line = py3compat.cast_unicode(line) + + new_line, err = _line_format(line, 'str') + if not err: + line = new_line + + if i == lnum: + # This is the line with the error + pad = numbers_width - len(str(i)) + num = '%s%s' % (debugger.make_arrow(pad), str(lnum)) + line = '%s%s%s %s%s' % (Colors.linenoEm, num, + Colors.line, line, Colors.Normal) + else: + num = '%*s' % (numbers_width, i) + line = '%s%s%s %s' % (Colors.lineno, num, + Colors.Normal, line) + + res.append(line) + if lvals and i == lnum: + res.append(lvals + '\n') + return res def _format_filename(file, ColorFilename, ColorNormal, *, lineno=None): """ @@ -607,6 +654,61 @@ class ListTB(TBTools): return u'' % type(value).__name__ +class FrameInfo: + """ + Mirror of stack data's FrameInfo, but so that we can bypass highlighting on + really long frames. + """ + + description : Optional[str] + filename : str + lineno : int + + @classmethod + def _from_stack_data_FrameInfo(cls, frame_info): + return cls( + getattr(frame_info, "description", None), + getattr(frame_info, "filename", None), + getattr(frame_info, "lineno", None), + getattr(frame_info, "frame", None), + getattr(frame_info, "code", None), + sd=frame_info, + ) + + def __init__(self, description, filename, lineno, frame, code, sd=None): + self.description = description + self.filename = filename + self.lineno = lineno + self.frame = frame + self.code = code + self._sd = sd + + # self.lines = [] + if sd is None: + ix = inspect.getsourcelines(frame) + self.raw_lines = ix[0] + + @property + def variables_in_executing_piece(self): + if self._sd: + return self._sd.variables_in_executing_piece + else: + return [] + + @property + def lines(self): + return self._sd.lines + + @property + def executing(self): + if self._sd: + return self._sd.executing + else: + return None + + + + #---------------------------------------------------------------------------- class VerboseTB(TBTools): """A port of Ka-Ping Yee's cgitb.py module that outputs color text instead @@ -658,12 +760,13 @@ class VerboseTB(TBTools): self.skip_hidden = True - def format_record(self, frame_info): + def format_record(self, frame_info:FrameInfo): """Format a single stack frame""" + assert isinstance(frame_info, FrameInfo) Colors = self.Colors # just a shorthand + quicker name lookup ColorsNormal = Colors.Normal # used a lot - if isinstance(frame_info, stack_data.RepeatedFrames): + if isinstance(frame_info._sd, stack_data.RepeatedFrames): return ' %s[... skipping similar frames: %s]%s\n' % ( Colors.excName, frame_info.description, ColorsNormal) @@ -684,8 +787,10 @@ class VerboseTB(TBTools): lineno=frame_info.lineno, ) args, varargs, varkw, locals_ = inspect.getargvalues(frame_info.frame) - - func = frame_info.executing.code_qualname() + if frame_info.executing is not None: + func = frame_info.executing.code_qualname() + else: + func = '?' if func == "": call = "" else: @@ -732,8 +837,25 @@ class VerboseTB(TBTools): lvals = '%s%s' % (indent, em_normal.join(lvals_list)) result = f'{link}{", " if call else ""}{call}\n' - - result += ''.join(_format_traceback_lines(frame_info.lines, Colors, self.has_colors, lvals)) + if frame_info._sd is None: + assert False + # fast fallback if file is too long + tpl_link = '%s%%s%s' % (Colors.filenameEm, ColorsNormal) + link = tpl_link % util_path.compress_user(frame_info.filename) + level = '%s %s\n' % (link, call) + _line_format = PyColorize.Parser(style=self.color_scheme_table.active_scheme_name, parent=self).format2 + first_line = frame_info.code.co_firstlineno + current_line = frame_info.lineno[0] + return '%s%s' % (level, ''.join( + _simple_format_traceback_lines(current_line, current_line-first_line, frame_info.raw_lines, Colors, lvals, + _line_format))) + #result += "\n".join(frame_info.raw_lines) + else: + result += "".join( + _format_traceback_lines( + frame_info.lines, Colors, self.has_colors, lvals + ) + ) return result def prepare_header(self, etype, long_version=False): @@ -802,9 +924,9 @@ class VerboseTB(TBTools): frames = [] skipped = 0 lastrecord = len(records) - 1 - for i, r in enumerate(records): - if not isinstance(r, stack_data.RepeatedFrames) and self.skip_hidden: - if r.frame.f_locals.get("__tracebackhide__", 0) and i != lastrecord: + for i, record in enumerate(records): + if not isinstance(record._sd, stack_data.RepeatedFrames) and self.skip_hidden: + if record.frame.f_locals.get("__tracebackhide__", 0) and i != lastrecord: skipped += 1 continue if skipped: @@ -815,7 +937,7 @@ class VerboseTB(TBTools): % (Colors.excName, skipped, ColorsNormal) ) skipped = 0 - frames.append(self.format_record(r)) + frames.append(self.format_record(record)) if skipped: Colors = self.Colors # just a shorthand + quicker name lookup ColorsNormal = Colors.Normal # used a lot @@ -851,7 +973,32 @@ class VerboseTB(TBTools): after=after, pygments_formatter=formatter, ) - return list(stack_data.FrameInfo.stack_data(etb, options=options))[tb_offset:] + + # let's estimate the amount of code we eill have to parse/highlight. + cf = etb + max_len = 0 + tbs = [] + while cf is not None: + source_file = inspect.getsourcefile(etb.tb_frame) + lines, first = inspect.getsourcelines(etb.tb_frame) + max_len = max(max_len, first+len(lines)) + tbs.append(cf) + cf = cf.tb_next + + + + if max_len > FAST_THRESHOLD: + FIs = [] + for tb in tbs: + frame = tb.tb_frame + lineno = frame.f_lineno, + code = frame.f_code + filename = code.co_filename + FIs.append( FrameInfo("Raw frame", filename, lineno, frame, code)) + return FIs + res = list(stack_data.FrameInfo.stack_data(etb, options=options))[tb_offset:] + res = [FrameInfo._from_stack_data_FrameInfo(r) for r in res] + return res def structured_traceback( self,