Backport PR #10157: Fix detection of complete input with line continuations inside a multiline string

I thought that tokenize accepted multiple lines of input as a single 'line' from the readline callback, and it does in most cases, but in this case the behaviour makes a difference.

Closes  9841.

Alternative to  9899.

IPython/core/inputtransformer.py

@@ -126,25 +126,18 b' class TokenInputTransformer(InputTransformer):'
     """
     def __init__(self, func):
         self.func = func
-        self.current_line = ""
-        self.line_used = False
+        self.buf = []
         self.reset_tokenizer()
-    
+
     def reset_tokenizer(self):
-        self.tokenizer = generate_tokens(self.get_line)
-    
-    def get_line(self):
-        if self.line_used:
-            raise TokenError
-        self.line_used = True
-        return self.current_line
-    
+        it = iter(self.buf)
+        self.tokenizer = generate_tokens(it.__next__)
+
     def push(self, line):
-        self.current_line += line + "\n"
-        if self.current_line.isspace():
+        self.buf.append(line + '\n')
+        if all(l.isspace() for l in self.buf):
             return self.reset()
-        
-        self.line_used = False
+
         tokens = []
         stop_at_NL = False
         try:
@@ -164,13 +157,13 b' class TokenInputTransformer(InputTransformer):'
         return self.output(tokens)
     
     def output(self, tokens):
-        self.current_line = ""
+        self.buf.clear()
         self.reset_tokenizer()
         return untokenize(self.func(tokens)).rstrip('\n')
     
     def reset(self):
-        l = self.current_line
-        self.current_line = ""
+        l = ''.join(self.buf)
+        self.buf.clear()
         self.reset_tokenizer()
         if l:
             return l.rstrip('\n')

IPython/core/tests/test_inputtransformer.py

@@ -415,6 +415,11 b' def test_assemble_python_lines():'
        (u"2,", None),
        (None, u"a = [1,\n2,"),
       ],
+      [(u"a = '''", None),  # Test line continuation within a multi-line string
+       (u"abc\\", None),
+       (u"def", None),
+       (u"'''", u"a = '''\nabc\\\ndef\n'''"),
+      ],
     ] + syntax_ml['multiline_datastructure']
     for example in tests:
         transform_checker(example, ipt.assemble_python_lines)