From 03e38b6f4b913bafbdd2c62c68f64408a71d1e52 2012-01-12 22:20:08
From: Thomas Kluyver <takowl@gmail.com>
Date: 2012-01-12 22:20:08
Subject: [PATCH] Simplify tokenizing code for VerboseTB.
---
diff --git a/IPython/core/ultratb.py b/IPython/core/ultratb.py
index 2fbe859..5556d55 100644
--- a/IPython/core/ultratb.py
+++ b/IPython/core/ultratb.py
@@ -832,47 +832,6 @@ class VerboseTB(TBTools):
# disabled.
call = tpl_call_fail % func
- # Initialize a list of names on the current line, which the
- # tokenizer below will populate.
- names = []
-
- def tokeneater(token_type, token, start, end, line):
- """Stateful tokeneater which builds dotted names.
-
- The list of names it appends to (from the enclosing scope) can
- contain repeated composite names. This is unavoidable, since
- there is no way to disambguate partial dotted structures until
- the full list is known. The caller is responsible for pruning
- the final list of duplicates before using it."""
-
- # build composite names
- if token == '.':
- try:
- names[-1] += '.'
- # store state so the next token is added for x.y.z names
- tokeneater.name_cont = True
- return
- except IndexError:
- pass
- if token_type == tokenize.NAME and token not in keyword.kwlist:
- if tokeneater.name_cont:
- # Dotted names
- names[-1] += token
- tokeneater.name_cont = False
- else:
- # Regular new names. We append everything, the caller
- # will be responsible for pruning the list later. It's
- # very tricky to try to prune as we go, b/c composite
- # names can fool us. The pruning at the end is easy
- # to do (or the caller can print a list with repeated
- # names if so desired.
- names.append(token)
- elif token_type == tokenize.NEWLINE:
- raise IndexError
- # we need to store a bit of state in the tokenizer to build
- # dotted names
- tokeneater.name_cont = False
-
def linereader(file=file, lnum=[lnum], getline=linecache.getline):
if file.endswith(('.pyc','.pyo')):
file = pyfile.source_from_cache(file)
@@ -883,10 +842,32 @@ class VerboseTB(TBTools):
# Build the list of names on this line of code where the exception
# occurred.
try:
- # This builds the names list in-place by capturing it from the
- # enclosing scope.
- for token in generate_tokens(linereader):
- tokeneater(*token)
+ names = []
+ name_cont = False
+
+ for token_type, token, start, end, line in generate_tokens(linereader):
+ # build composite names
+ if token_type == tokenize.NAME and token not in keyword.kwlist:
+ if name_cont:
+ # Continuation of a dotted name
+ try:
+ names[-1].append(token)
+ except IndexError:
+ names.append([token])
+ name_cont = False
+ else:
+ # Regular new names. We append everything, the caller
+ # will be responsible for pruning the list later. It's
+ # very tricky to try to prune as we go, b/c composite
+ # names can fool us. The pruning at the end is easy
+ # to do (or the caller can print a list with repeated
+ # names if so desired.
+ names.append([token])
+ elif token == '.':
+ name_cont = True
+ elif token_type == tokenize.NEWLINE:
+ break
+
except (IndexError, UnicodeDecodeError):
# signals exit of tokenizer
pass
@@ -896,6 +877,8 @@ class VerboseTB(TBTools):
"The error message is: %s\n" % msg)
error(_m)
+ # Join composite names (e.g. "dict.fromkeys")
+ names = ['.'.join(n) for n in names]
# prune names list of duplicates, but keep the right order
unique_names = uniq_stable(names)