Fixes for UltraTB and PyColorize with Python 3

IPython/core/ultratb.py

@@ -83,6 +83,11 b' import tokenize'
 import traceback
 import types
 
+try:                           # Python 2
+    generate_tokens = tokenize.generate_tokens
+except AttributeError:         # Python 3
+    generate_tokens = tokenize.tokenize
+
 # For purposes of monkeypatching inspect to fix a bug in it.
 from inspect import getsourcefile, getfile, getmodule,\
      ismodule, isclass, ismethod, isfunction, istraceback, isframe, iscode
@@ -94,6 +99,7 b' from IPython.core.display_trap import DisplayTrap'
 from IPython.core.excolors import exception_colors
 from IPython.utils import PyColorize
 from IPython.utils import io
+from IPython.utils import py3compat
 from IPython.utils.data import uniq_stable
 from IPython.utils.warn import info, error
 
@@ -278,8 +284,7 b' def _format_traceback_lines(lnum, index, lines, Colors, lvals=None,scheme=None):'
         # serious refactoring, so that all of the ultratb and PyColorize code
         # is unicode-safe.  So for now this is rather an ugly hack, but
         # necessary to at least have readable tracebacks. Improvements welcome!
-        if type(line)==unicode:
-            line = line.encode('utf-8', 'replace')
+        line = py3compat.cast_bytes_py2(line, 'utf-8')
             
         new_line, err = _line_format(line, 'str', scheme)
         if not err: line = new_line
@@ -872,7 +877,8 b' class VerboseTB(TBTools):'
             try:
                 # This builds the names list in-place by capturing it from the
                 # enclosing scope.
-                tokenize.tokenize(linereader, tokeneater)
+                for token in generate_tokens(linereader):
+                    tokeneater(*token)
             except IndexError:
                 # signals exit of tokenizer
                 pass
@@ -933,7 +939,7 b' class VerboseTB(TBTools):'
         # ... and format it
         exception = ['%s%s%s: %s' % (Colors.excName, etype_str,
                                      ColorsNormal, evalue_str)]
-        if type(evalue) is types.InstanceType:
+        if (not py3compat.PY3) and type(evalue) is types.InstanceType:
             try:
                 names = [w for w in dir(evalue) if isinstance(w, basestring)]
             except:

IPython/utils/PyColorize.py

@@ -42,6 +42,13 b' import sys'
 import token
 import tokenize
 
+try:
+    generate_tokens = tokenize.generate_tokens
+except AttributeError:
+    # Python 3. Note that we use the undocumented _tokenize because it expects
+    # strings, not bytes. See also Python issue #9969.
+    generate_tokens = tokenize._tokenize
+
 from IPython.utils.coloransi import *
 
 #############################################################################
@@ -177,7 +184,8 b' class Parser:'
 
         error = False
         try:
-            tokenize.tokenize(text.readline, self)
+            for token in generate_tokens(text.readline):
+                self(*token)
         except tokenize.TokenError, ex:
             msg = ex[0]
             line = ex[1][0]

IPython/utils/py3compat.py

@@ -35,6 +35,7 b' if sys.version_info[0] >= 3:'
     unicode_to_str = no_code
     str_to_bytes = encode
     bytes_to_str = decode
+    cast_bytes_py2 = no_code
     
     def isidentifier(s, dotted=False):
         if dotted:
@@ -53,6 +54,7 b' else:'
     unicode_to_str = encode
     str_to_bytes = no_code
     bytes_to_str = no_code
+    cast_bytes_py2 = cast_bytes
     
     import re
     _name_re = re.compile(r"[a-zA-Z_][a-zA-Z0-9_]*$")