Start integrating new input transformation machinery into InteractiveShell

IPython/core/inputtransformer2.py

@@ -66,13 +66,6 def cell_magic(lines):
     return ['get_ipython().run_cell_magic(%r, %r, %r)\n'
             % (magic_name, first_line, body)]
 
-line_transforms = [
-    leading_indent,
-    classic_prompt,
-    ipython_prompt,
-    cell_magic,
-]
-
 # -----
 
 def _find_assign_op(token_line):
@@ -378,16 +371,22 def show_linewise_tokens(s: str):
         for tokinfo in line:
             print(" ", tokinfo)
 
-class TokenTransformers:
+class TransformerManager:
     def __init__(self):
-        self.transformers = [
+        self.line_transforms = [
+            leading_indent,
+            classic_prompt,
+            ipython_prompt,
+            cell_magic,
+        ]
+        self.token_transformers = [
             MagicAssign,
             SystemAssign,
             EscapedCommand,
             HelpEnd,
         ]
     
-    def do_one_transform(self, lines):
+    def do_one_token_transform(self, lines):
         """Find and run the transform earliest in the code.
         
         Returns (changed, lines).
@@ -399,11 +398,11 class TokenTransformers:
         the transformed code is retokenised every time to identify the next
         piece of special syntax. Hopefully long code cells are mostly valid
         Python, not using lots of IPython special syntax, so this shouldn't be
-        a performance issue. 
+        a performance issue.
         """
         tokens_by_line = make_tokens_by_line(lines)
         candidates = []
-        for transformer_cls in self.transformers:
+        for transformer_cls in self.token_transformers:
             transformer = transformer_cls.find(tokens_by_line)
             if transformer:
                 candidates.append(transformer)
@@ -415,20 +414,19 class TokenTransformers:
         transformer = min(candidates, key=TokenTransformBase.sortby)
         return True, transformer.transform(lines)
 
-    def __call__(self, lines):
+    def do_token_transforms(self, lines):
         while True:
-            changed, lines = self.do_one_transform(lines)
+            changed, lines = self.do_one_token_transform(lines)
             if not changed:
                 return lines
 
+    def transform_cell(self, cell: str):
+        if not cell.endswith('\n'):
+            cell += '\n'  # Ensure every line has a newline
+        lines = cell.splitlines(keepends=True)
+        for transform in self.line_transforms:
+            #print(transform, lines)
+            lines = transform(lines)
 
-def transform_cell(cell):
-    if not cell.endswith('\n'):
-        cell += '\n'  # Ensure every line has a newline
-    lines = cell.splitlines(keepends=True)
-    for transform in line_transforms:
-        #print(transform, lines)
-        lines = transform(lines)
-    
-    lines = TokenTransformers()(lines)
-    return ''.join(lines)
+        lines = self.do_token_transforms(lines)
+        return ''.join(lines)

IPython/core/interactiveshell.py

@@ -339,10 +339,9 class InteractiveShell(SingletonConfigurable):
     input_splitter = Instance('IPython.core.inputsplitter.IPythonInputSplitter',
                               (), {'line_input_checker': True})
 
-    # This InputSplitter instance is used to transform completed cells before
-    # running them. It allows cell magics to contain blank lines.
-    input_transformer_manager = Instance('IPython.core.inputsplitter.IPythonInputSplitter',
-                                         (), {'line_input_checker': False})
+    # Used to transform cells before running them.
+    input_transformer_manager = Instance('IPython.core.inputtransformer2.TransformerManager',
+                                         ())
 
     logstart = Bool(False, help=
         """

IPython/core/tests/test_interactiveshell.py

@@ -832,28 +832,22 def test_user_expression():
 class TestSyntaxErrorTransformer(unittest.TestCase):
     """Check that SyntaxError raised by an input transformer is handled by run_cell()"""
 
-    class SyntaxErrorTransformer(InputTransformer):
-
-        def push(self, line):
+    @staticmethod
+    def transformer(lines):
+        for line in lines:
             pos = line.find('syntaxerror')
             if pos >= 0:
                 e = SyntaxError('input contains "syntaxerror"')
                 e.text = line
                 e.offset = pos + 1
                 raise e
-            return line
-
-        def reset(self):
-            pass
+        return lines
 
     def setUp(self):
-        self.transformer = TestSyntaxErrorTransformer.SyntaxErrorTransformer()
-        ip.input_splitter.python_line_transforms.append(self.transformer)
-        ip.input_transformer_manager.python_line_transforms.append(self.transformer)
+        ip.input_transformer_manager.line_transforms.append(self.transformer)
 
     def tearDown(self):
-        ip.input_splitter.python_line_transforms.remove(self.transformer)
-        ip.input_transformer_manager.python_line_transforms.remove(self.transformer)
+        ip.input_transformer_manager.line_transforms.remove(self.transformer)
 
     def test_syntaxerror_input_transformer(self):
         with tt.AssertPrints('1234'):