From f55a4a583ad79f5be3692d0acce0f18b1ec25e12 2018-03-10 18:55:49
From: Thomas Kluyver <thomas@kluyver.me.uk>
Date: 2018-03-10 18:55:49
Subject: [PATCH] Start integrating new input transformation machinery into InteractiveShell

---

diff --git a/IPython/core/inputtransformer2.py b/IPython/core/inputtransformer2.py
index 0027013..f843ce4 100644
--- a/IPython/core/inputtransformer2.py
+++ b/IPython/core/inputtransformer2.py
@@ -66,13 +66,6 @@ def cell_magic(lines):
     return ['get_ipython().run_cell_magic(%r, %r, %r)\n'
             % (magic_name, first_line, body)]
 
-line_transforms = [
-    leading_indent,
-    classic_prompt,
-    ipython_prompt,
-    cell_magic,
-]
-
 # -----
 
 def _find_assign_op(token_line):
@@ -378,16 +371,22 @@ def show_linewise_tokens(s: str):
         for tokinfo in line:
             print(" ", tokinfo)
 
-class TokenTransformers:
+class TransformerManager:
     def __init__(self):
-        self.transformers = [
+        self.line_transforms = [
+            leading_indent,
+            classic_prompt,
+            ipython_prompt,
+            cell_magic,
+        ]
+        self.token_transformers = [
             MagicAssign,
             SystemAssign,
             EscapedCommand,
             HelpEnd,
         ]
     
-    def do_one_transform(self, lines):
+    def do_one_token_transform(self, lines):
         """Find and run the transform earliest in the code.
         
         Returns (changed, lines).
@@ -399,11 +398,11 @@ class TokenTransformers:
         the transformed code is retokenised every time to identify the next
         piece of special syntax. Hopefully long code cells are mostly valid
         Python, not using lots of IPython special syntax, so this shouldn't be
-        a performance issue. 
+        a performance issue.
         """
         tokens_by_line = make_tokens_by_line(lines)
         candidates = []
-        for transformer_cls in self.transformers:
+        for transformer_cls in self.token_transformers:
             transformer = transformer_cls.find(tokens_by_line)
             if transformer:
                 candidates.append(transformer)
@@ -415,20 +414,19 @@ class TokenTransformers:
         transformer = min(candidates, key=TokenTransformBase.sortby)
         return True, transformer.transform(lines)
 
-    def __call__(self, lines):
+    def do_token_transforms(self, lines):
         while True:
-            changed, lines = self.do_one_transform(lines)
+            changed, lines = self.do_one_token_transform(lines)
             if not changed:
                 return lines
 
+    def transform_cell(self, cell: str):
+        if not cell.endswith('\n'):
+            cell += '\n'  # Ensure every line has a newline
+        lines = cell.splitlines(keepends=True)
+        for transform in self.line_transforms:
+            #print(transform, lines)
+            lines = transform(lines)
 
-def transform_cell(cell):
-    if not cell.endswith('\n'):
-        cell += '\n'  # Ensure every line has a newline
-    lines = cell.splitlines(keepends=True)
-    for transform in line_transforms:
-        #print(transform, lines)
-        lines = transform(lines)
-    
-    lines = TokenTransformers()(lines)
-    return ''.join(lines)
+        lines = self.do_token_transforms(lines)
+        return ''.join(lines)
diff --git a/IPython/core/interactiveshell.py b/IPython/core/interactiveshell.py
index c874ff9..ac1c585 100644
--- a/IPython/core/interactiveshell.py
+++ b/IPython/core/interactiveshell.py
@@ -339,10 +339,9 @@ class InteractiveShell(SingletonConfigurable):
     input_splitter = Instance('IPython.core.inputsplitter.IPythonInputSplitter',
                               (), {'line_input_checker': True})
 
-    # This InputSplitter instance is used to transform completed cells before
-    # running them. It allows cell magics to contain blank lines.
-    input_transformer_manager = Instance('IPython.core.inputsplitter.IPythonInputSplitter',
-                                         (), {'line_input_checker': False})
+    # Used to transform cells before running them.
+    input_transformer_manager = Instance('IPython.core.inputtransformer2.TransformerManager',
+                                         ())
 
     logstart = Bool(False, help=
         """
diff --git a/IPython/core/tests/test_interactiveshell.py b/IPython/core/tests/test_interactiveshell.py
index 12bd380..d6c647f 100644
--- a/IPython/core/tests/test_interactiveshell.py
+++ b/IPython/core/tests/test_interactiveshell.py
@@ -832,28 +832,22 @@ def test_user_expression():
 class TestSyntaxErrorTransformer(unittest.TestCase):
     """Check that SyntaxError raised by an input transformer is handled by run_cell()"""
 
-    class SyntaxErrorTransformer(InputTransformer):
-
-        def push(self, line):
+    @staticmethod
+    def transformer(lines):
+        for line in lines:
             pos = line.find('syntaxerror')
             if pos >= 0:
                 e = SyntaxError('input contains "syntaxerror"')
                 e.text = line
                 e.offset = pos + 1
                 raise e
-            return line
-
-        def reset(self):
-            pass
+        return lines
 
     def setUp(self):
-        self.transformer = TestSyntaxErrorTransformer.SyntaxErrorTransformer()
-        ip.input_splitter.python_line_transforms.append(self.transformer)
-        ip.input_transformer_manager.python_line_transforms.append(self.transformer)
+        ip.input_transformer_manager.line_transforms.append(self.transformer)
 
     def tearDown(self):
-        ip.input_splitter.python_line_transforms.remove(self.transformer)
-        ip.input_transformer_manager.python_line_transforms.remove(self.transformer)
+        ip.input_transformer_manager.line_transforms.remove(self.transformer)
 
     def test_syntaxerror_input_transformer(self):
         with tt.AssertPrints('1234'):