Silence tokenization errors in ultratb.

Currently, the ultratb module attempts to tokenize the code it finds in
tracebacks, and issues a scary-sounding warnings about potentially invalid
tracebacks if it fails. However, this tokenization is known to fail in some
cases, notably if the reported error line is the terminating line of a
multiline, triple-quoted string literal.

This change fixes the issue by switching this to a `logging.debug` message,
which silences it by default without completely removing it for debugging
purposes.

Fixes  #6864.

IPython/core/tests/test_ultratb.py

@@ -2,6 +2,7 b''
 """Tests for IPython.core.ultratb
 """
 import io
+import logging
 import sys
 import os.path
 from textwrap import dedent
@@ -372,3 +373,28 b' def test_handlers():'
         handler(*sys.exc_info())
     buff.write('')
 
+
+class TokenizeFailureTest(unittest.TestCase):
+    """Tests related to https://github.com/ipython/ipython/issues/6864."""
+
+    def testLogging(self):
+        message = "An unexpected error occurred while tokenizing input"
+        cell = 'raise ValueError("""a\nb""")'
+
+        stream = io.StringIO()
+        handler = logging.StreamHandler(stream)
+        logger = logging.getLogger()
+        loglevel = logger.level
+        logger.addHandler(handler)
+        self.addCleanup(lambda: logger.removeHandler(handler))
+        self.addCleanup(lambda: logger.setLevel(loglevel))
+
+        logger.setLevel(logging.INFO)
+        with tt.AssertNotPrints(message):
+            ip.run_cell(cell)
+        self.assertNotIn(message, stream.getvalue())
+
+        logger.setLevel(logging.DEBUG)
+        with tt.AssertNotPrints(message):
+            ip.run_cell(cell)
+        self.assertIn(message, stream.getvalue())

IPython/core/ultratb.py

@@ -121,7 +121,7 b' from IPython.utils import path as util_path'
 from IPython.utils import py3compat
 from IPython.utils.data import uniq_stable
 from IPython.utils.terminal import get_terminal_size
-from logging import info, error
+from logging import info, error, debug
 
 import IPython.utils.colorable as colorable
 
@@ -952,10 +952,15 b' class VerboseTB(TBTools):'
             #  - see gh-6300
             pass
         except tokenize.TokenError as msg:
+            # Tokenizing may fail for various reasons, many of which are
+            # harmless. (A good example is when the line in question is the
+            # close of a triple-quoted string, cf gh-6864). We don't want to
+            # show this to users, but want make it available for debugging
+            # purposes.
             _m = ("An unexpected error occurred while tokenizing input\n"
                   "The following traceback may be corrupted or invalid\n"
                   "The error message is: %s\n" % msg)
-            error(_m)
+            debug(_m)
 
         # Join composite names (e.g. "dict.fromkeys")
         names = ['.'.join(n) for n in names]