Merge pull request #11307 from Carreau/safe-input-transformer

Safe input transformer

IPython/core/inputtransformer2.py

@@ -418,6 +418,8 b' class HelpEnd(TokenTransformBase):'
         lines_after = lines[self.q_line + 1:]
 
         m = _help_end_re.search(content)
+        if not m:
+            raise SyntaxError(content)
         assert m is not None, content
         target = m.group(1)
         esc = m.group(3)
@@ -460,6 +462,8 b' def make_tokens_by_line(lines):'
     except tokenize.TokenError:
         # Input ended in a multiline string or expression. That's OK for us.
         pass
+    if not tokens_by_line[-1]:
+        tokens_by_line.pop()
     
     return tokens_by_line
 
@@ -522,9 +526,13 b' class TransformerManager:'
         if not candidates:
             # Nothing to transform
             return False, lines
-
-        transformer = min(candidates, key=TokenTransformBase.sortby)
-        return True, transformer.transform(lines)
+        ordered_transformers = sorted(candidates, key=TokenTransformBase.sortby)
+        for transformer in ordered_transformers:
+            try:
+                return True, transformer.transform(lines)
+            except SyntaxError:
+                pass
+        return False, lines
 
     def do_token_transforms(self, lines):
         for _ in range(TRANSFORM_LOOP_LIMIT):
@@ -591,10 +599,13 b' class TransformerManager:'
             return 'invalid', None
 
         tokens_by_line = make_tokens_by_line(lines)
+        if not tokens_by_line:
+            return 'incomplete', find_last_indent(lines)
         if tokens_by_line[-1][-1].type != tokenize.ENDMARKER:
             # We're in a multiline string or expression
             return 'incomplete', find_last_indent(lines)
-
+        if len(tokens_by_line) == 1:
+            return 'incomplete', find_last_indent(lines)
         # Find the last token on the previous line that's not NEWLINE or COMMENT
         toks_last_line = tokens_by_line[-2]
         ix = len(toks_last_line) - 1

IPython/core/tests/test_inputtransformer2.py

@@ -5,6 +5,7 b' more complex. See test_inputtransformer2_line for tests for line-based'
 transformations.
 """
 import nose.tools as nt
+import string
 
 from IPython.core import inputtransformer2 as ipt2
 from IPython.core.inputtransformer2 import make_tokens_by_line
@@ -100,6 +101,16 b' b) = zip?'
 [r"get_ipython().set_next_input('(a,\nb) = zip');get_ipython().run_line_magic('pinfo', 'zip')" + "\n"]
 )
 
+def check_make_token_by_line_never_ends_empty():
+    """
+    Check that not sequence of single or double characters ends up leading to en empty list of tokens
+    """
+    from string import printable
+    for c in printable:
+        nt.assert_not_equal(make_tokens_by_line(c)[-1], [])
+        for k in printable:
+            nt.assert_not_equal(make_tokens_by_line(c+k)[-1], [])
+
 def check_find(transformer, case, match=True):
     sample, expected_start, _  = case
     tbl = make_tokens_by_line(sample)
@@ -190,6 +201,17 b' def test_check_complete():'
     nt.assert_equal(cc("for a in range(5):"), ('incomplete', 4))
     nt.assert_equal(cc("raise = 2"), ('invalid', None))
     nt.assert_equal(cc("a = [1,\n2,"), ('incomplete', 0))
+    nt.assert_equal(cc(")"), ('incomplete', 0))
+    nt.assert_equal(cc("\\\r\n"), ('incomplete', 0))
     nt.assert_equal(cc("a = '''\n   hi"), ('incomplete', 3))
     nt.assert_equal(cc("def a():\n x=1\n global x"), ('invalid', None))
     nt.assert_equal(cc("a \\ "), ('invalid', None))  # Nothing allowed after backslash
+
+    # no need to loop on all the letters/numbers.
+    short = '12abAB'+string.printable[62:]
+    for c in short:
+        # test does not raise:
+        cc(c)
+        for k in short:
+            cc(c+k)
+