Revised input transformation framework.

IPython/core/inputsplitter.py

@@ -82,6 +82,7 b' from IPython.core.inputtransformer import (leading_indent,'
                                            escaped_transformer,
                                            assign_from_magic,
                                            assign_from_system,
+                                           assemble_python_lines,
                                            )
 
 # Temporary!
@@ -499,29 +500,55 b' class IPythonInputSplitter(InputSplitter):'
     # Flag to track when a transformer has stored input that it hasn't given
     # back yet.
     transformer_accumulating = False
+    
+    # Flag to track when assemble_python_lines has stored input that it hasn't
+    # given back yet.
+    within_python_line = False
 
     # Private attributes
 
     # List with lines of raw input accumulated so far.
     _buffer_raw = None
 
-    def __init__(self, input_mode=None, transforms=None):
+    def __init__(self, input_mode=None, physical_line_transforms=None,
+                    logical_line_transforms=None, python_line_transforms=None):
         super(IPythonInputSplitter, self).__init__(input_mode)
         self._buffer_raw = []
         self._validate = True
-        if transforms is not None:
-            self.transforms = transforms
-        else:
-            self.transforms = [leading_indent(),
-                               classic_prompt(),
-                               ipy_prompt(),
-                               cellmagic(),
-                               assemble_logical_lines(),
-                               help_end(),
-                               escaped_transformer(),
-                               assign_from_magic(),
-                               assign_from_system(),
-                              ]
+        
+        self.physical_line_transforms = physical_line_transforms or \
+                [leading_indent(),
+                 classic_prompt(),
+                 ipy_prompt(),
+                ]
+        
+        self.assemble_logical_lines = assemble_logical_lines()
+        self.logical_line_transforms = logical_line_transforms or \
+                [cellmagic(),
+                 help_end(),
+                 escaped_transformer(),
+                 assign_from_magic(),
+                 assign_from_system(),
+                ]
+        
+        self.assemble_python_lines = assemble_python_lines()
+        self.python_line_transforms = python_line_transforms or []
+    
+    @property
+    def transforms(self):
+        "Quick access to all transformers."
+        return self.physical_line_transforms + \
+            [self.assemble_logical_lines] + self.logical_line_transforms + \
+            [self.assemble_python_lines]  + self.python_line_transforms
+    
+    @property
+    def transforms_in_use(self):
+        """Transformers, excluding logical line transformers if we're in a
+        Python line."""
+        t = self.physical_line_transforms + [self.assemble_logical_lines]
+        if not self.within_python_line:
+            t += self.logical_line_transforms
+        return t + [self.assemble_python_lines] + self.python_line_transforms
 
     def reset(self):
         """Reset the input buffer and associated state."""
@@ -533,12 +560,18 b' class IPythonInputSplitter(InputSplitter):'
             t.reset()
     
     def flush_transformers(self):
+        def _flush(transform, out):
+            if out is not None:
+                tmp = transform.push(out)
+                return tmp or transform.reset() or None
+            else:
+                return transform.reset() or None
+        
         out = None
-        for t in self.transforms:
-            tmp = t.reset()
-            if tmp:
-                out = tmp
-        if out:
+        for t in self.transforms_in_use:
+            out = _flush(t, out)
+        
+        if out is not None:
             self._store(out)
 
     def source_raw_reset(self):
@@ -641,11 +674,39 b' class IPythonInputSplitter(InputSplitter):'
     
     def push_line(self, line):
         buf = self._buffer
-        for transformer in self.transforms:
+        
+        def _accumulating(dbg):
+            #print(dbg)
+            self.transformer_accumulating = True
+            return False
+        
+        for transformer in self.physical_line_transforms:
+            line = transformer.push(line)
+            if line is None:
+                return _accumulating(transformer)
+        
+        line = self.assemble_logical_lines.push(line)
+        if line is None:
+            return _accumulating('acc logical line')
+        
+        if not self.within_python_line:
+            for transformer in self.logical_line_transforms:
+                line = transformer.push(line)
+                if line is None:
+                    return _accumulating(transformer)
+        
+        line = self.assemble_python_lines.push(line)
+        if line is None:
+            self.within_python_line = True
+            return _accumulating('acc python line')
+        else:
+            self.within_python_line = False
+        
+        for transformer in self.python_line_transforms:
             line = transformer.push(line)
             if line is None:
-                self.transformer_accumulating = True
-                return False                
+                return _accumulating(transformer)
 
+        #print("transformers clear") #debug
         self.transformer_accumulating = False
         return super(IPythonInputSplitter, self).push(line)

IPython/core/inputtransformer.py

@@ -127,7 +127,7 b' class TokenInputTransformer(InputTransformer):'
     def __init__(self, func):
         self.func = func
         self.current_line = ""
-        self.line_used= False
+        self.line_used = False
         self.reset_tokenizer()
     
     def reset_tokenizer(self):
@@ -141,19 +141,29 b' class TokenInputTransformer(InputTransformer):'
     
     def push(self, line):
         self.current_line += line + "\n"
+        if self.current_line.isspace():
+            return self.reset()
+        
         self.line_used = False
         tokens = []
+        stop_at_NL = False
         try:
             for intok in self.tokenizer:
                 tokens.append(intok)
-                if intok[0] in (tokenize.NEWLINE, tokenize.NL):
+                t = intok[0]
+                if t == tokenize.NEWLINE or (stop_at_NL and t == tokenize.NL):
                     # Stop before we try to pull a line we don't have yet
                     break
+                elif t in (tokenize.COMMENT, tokenize.ERRORTOKEN):
+                    stop_at_NL = True
         except tokenize.TokenError:
             # Multi-line statement - stop and try again with the next line
             self.reset_tokenizer()
             return None
         
+        return self.output(tokens)
+    
+    def output(self, tokens):
         self.current_line = ""
         self.reset_tokenizer()
         return untokenize(self.func(tokens)).rstrip('\n')
@@ -161,12 +171,35 b' class TokenInputTransformer(InputTransformer):'
     def reset(self):
         l = self.current_line
         self.current_line = ""
+        self.reset_tokenizer()
         if l:
             return l.rstrip('\n')
 
[e-mail]
-def assemble_logical_lines(tokens):
-    return tokens
+class assemble_python_lines(TokenInputTransformer):
+    def __init__(self):
+        super(assemble_python_lines, self).__init__(None)
+    
+    def output(self, tokens):
+        return self.reset()
+
[e-mail]
+def assemble_logical_lines():
+    """Join lines following explicit line continuations (\)"""
+    line = ''
+    while True:
+        line = (yield line)
+        if not line or line.isspace():
+            continue
+        
+        parts = []
+        while line is not None:
+            parts.append(line.rstrip('\\'))
+            if not line.endswith('\\'):
+                break
+            line = (yield None)
+        
+        # Output
+        line = ' '.join(parts)
 
 # Utilities
 def _make_help_call(target, esc, lspace, next_input=None):

IPython/core/tests/test_inputsplitter.py

@@ -270,6 +270,7 b' class InputSplitterTestCase(unittest.TestCase):'
         isp = self.isp
         self.assertFalse(isp.push('if 1:'))
         for line in ['  x=1', '# a comment', '  y=2']:
+            print(line)
             self.assertTrue(isp.push(line))
 
     def test_push3(self):

IPython/core/tests/test_inputtransformer.py

@@ -356,8 +356,8 b' def test_token_input_transformer():'
     [ [(u"a = 1.2; b = '''x", None),
        (u"y'''", u_fmt(u"a =Decimal ({u}'1.2');b ='''x\ny'''")),
       ],
-      [(u"a = [1.2,", u_fmt(u"a =[Decimal ({u}'1.2'),")),
-       (u"3]", u"3 ]"),
+      [(u"a = [1.2,", None),
+       (u"3]", u_fmt(u"a =[Decimal ({u}'1.2'),\n3 ]")),
       ],
       [(u"a = '''foo", None),  # Test resetting when within a multi-line string
        (u"bar", None),