Drop bundled, outdated copy of the tokenize module

IPython/core/inputtransformer.py

@@ -8,11 +8,11 b' This includes the machinery to recognise and transform ``%magic`` commands,'
 import abc
 import functools
 import re
+import tokenize
+from tokenize import generate_tokens, untokenize, TokenError
 from io import StringIO
 
 from IPython.core.splitinput import LineInfo
-from IPython.utils import tokenize2
-from IPython.utils.tokenize2 import generate_tokens, untokenize, TokenError
 
 #-----------------------------------------------------------------------------
 # Globals
@@ -140,10 +140,10 b' class TokenInputTransformer(InputTransformer):'
             for intok in self.tokenizer:
                 tokens.append(intok)
                 t = intok[0]
-                if t == tokenize2.NEWLINE or (stop_at_NL and t == tokenize2.NL):
+                if t == tokenize.NEWLINE or (stop_at_NL and t == tokenize.NL):
                     # Stop before we try to pull a line we don't have yet
                     break
-                elif t == tokenize2.ERRORTOKEN:
+                elif t == tokenize.ERRORTOKEN:
                     stop_at_NL = True
         except TokenError:
             # Multi-line statement - stop and try again with the next line
@@ -319,7 +319,7 b' def has_comment(src):'
     comment : bool
         True if source has a comment.
     """
-    return (tokenize2.COMMENT in _line_tokens(src))
+    return (tokenize.COMMENT in _line_tokens(src))
 
 def ends_in_comment_or_string(src):
     """Indicates whether or not an input line ends in a comment or within
@@ -336,7 +336,7 b' def ends_in_comment_or_string(src):'
         True if source ends in a comment or multiline string.
     """
     toktypes = _line_tokens(src)
-    return (tokenize2.COMMENT in toktypes) or (_MULTILINE_STRING in toktypes)
+    return (tokenize.COMMENT in toktypes) or (_MULTILINE_STRING in toktypes)
         
 
 @StatelessInputTransformer.wrap

IPython/utils/tokenutil.py

@@ -7,7 +7,7 b' from collections import namedtuple'
 from io import StringIO
 from keyword import iskeyword
 
-from . import tokenize2
+import tokenize
 
 
 Token = namedtuple('Token', ['token', 'text', 'start', 'end', 'line'])
@@ -15,9 +15,9 b" Token = namedtuple('Token', ['token', 'text', 'start', 'end', 'line'])"
 def generate_tokens(readline):
     """wrap generate_tokens to catch EOF errors"""
     try:
-        for token in tokenize2.generate_tokens(readline):
+        for token in tokenize.generate_tokens(readline):
             yield token
-    except tokenize2.TokenError:
+    except tokenize.TokenError:
         # catch EOF error
         return
 
@@ -99,12 +99,12 b' def token_at_cursor(cell, cursor_pos=0):'
             # don't consume it
             break
         
-        if tok.token == tokenize2.NAME and not iskeyword(tok.text):
-            if names and tokens and tokens[-1].token == tokenize2.OP and tokens[-1].text == '.':
+        if tok.token == tokenize.NAME and not iskeyword(tok.text):
+            if names and tokens and tokens[-1].token == tokenize.OP and tokens[-1].text == '.':
                 names[-1] = "%s.%s" % (names[-1], tok.text)
             else:
                 names.append(tok.text)
-        elif tok.token == tokenize2.OP:
+        elif tok.token == tokenize.OP:
             if tok.text == '=' and names:
                 # don't inspect the lhs of an assignment
                 names.pop(-1)