Simplify input transformers

More can now be implemented as stateless transformers

IPython/core/inputsplitter.py

@@ -79,7 +79,7 b' from IPython.core.inputtransformer import (leading_indent,'
                                            cellmagic,
                                            assemble_logical_lines,
                                            help_end,
-                                           escaped_transformer,
+                                           escaped_commands,
                                            assign_from_magic,
                                            assign_from_system,
                                            assemble_python_lines,
@@ -526,7 +526,7 b' class IPythonInputSplitter(InputSplitter):'
         self.logical_line_transforms = logical_line_transforms or \
                 [cellmagic(),
                  help_end(),
-                 escaped_transformer(),
+                 escaped_commands(),
                  assign_from_magic(),
                  assign_from_system(),
                 ]

IPython/core/inputtransformer.py

@@ -61,9 +61,6 b' class InputTransformer(object):'
         """
         pass
     
-    # Set this to True to allow the transformer to act on lines inside strings.
-    look_in_string = False
-    
     @classmethod
     def wrap(cls, func):
         """Can be used by subclasses as a decorator, to return a factory that
@@ -71,10 +68,7 b' class InputTransformer(object):'
         """
         @functools.wraps(func)
         def transformer_factory():
-            transformer = cls(func)
-            if getattr(transformer_factory, 'look_in_string', False):
-                transformer.look_in_string = True
-            return transformer
+            return cls(func)
         
         return transformer_factory
 
@@ -215,14 +209,6 b' def _make_help_call(target, esc, lspace, next_input=None):'
         return '%sget_ipython().set_next_input(%r);get_ipython().magic(%r)' % \
            (lspace, next_input, arg)
 
[e-mail]
-def escaped_transformer():
-    """Translate lines beginning with one of IPython's escape characters.
-    
-    This is stateful to allow magic commands etc. to be continued over several
-    lines using explicit line continuations (\ at the end of a line).
-    """
-    
     # These define the transformations for the different escape characters.
     def _tr_system(line_info):
         "Translate lines escaped with: !"
@@ -272,25 +258,17 b' def escaped_transformer():'
            ESC_QUOTE2 : _tr_quote2,
            ESC_PAREN  : _tr_paren }
     
-    line = ''
-    while True:
-        line = (yield line)
[e-mail]
+def escaped_commands(line):
+    """Transform escaped commands - %magic, !system, ?help + various autocalls.
+    """
         if not line or line.isspace():
-            continue
+        return line
         lineinf = LineInfo(line)
         if lineinf.esc not in tr:
-            continue
-        
-        parts = []
-        while line is not None:
-            parts.append(line.rstrip('\\'))
-            if not line.endswith('\\'):
-                break
-            line = (yield None)
+        return line
         
-        # Output
-        lineinf = LineInfo(' '.join(parts))
-        line = tr[lineinf.esc](lineinf)
+    return tr[lineinf.esc](lineinf)
 
 _initial_space_re = re.compile(r'\s*')
 
@@ -401,8 +379,6 b' def classic_prompt():'
     prompt2_re = re.compile(r'^(>>> |^\.\.\. )')
     return _strip_prompts(prompt1_re, prompt2_re)
 
-classic_prompt.look_in_string = True
-
 @CoroutineInputTransformer.wrap
 def ipy_prompt():
     """Strip IPython's In [1]:/...: prompts."""
@@ -410,8 +386,6 b' def ipy_prompt():'
     prompt2_re = re.compile(r'^(In \[\d+\]: |^\ \ \ \.\.\.+: )')
     return _strip_prompts(prompt1_re, prompt2_re)
 
-ipy_prompt.look_in_string = True
-
 
 @CoroutineInputTransformer.wrap
 def leading_indent():
@@ -440,48 +414,27 b' def leading_indent():'
             while line is not None:
                 line = (yield line)
 
-leading_indent.look_in_string = True
-
-
-def _special_assignment(assignment_re, template):
-    """Transform assignment from system & magic commands.
-    
-    This is stateful so that it can handle magic commands continued on several
-    lines.
-    """
-    line = ''
-    while True:
-        line = (yield line)
-        if not line or line.isspace():
-            continue
-        
-        m = assignment_re.match(line)
-        if not m:
-            continue
-        
-        parts = []
-        while line is not None:
-            parts.append(line.rstrip('\\'))
-            if not line.endswith('\\'):
-                break
-            line = (yield None)
-        
-        # Output
-        whole = assignment_re.match(' '.join(parts))
-        line = template % (whole.group('lhs'), whole.group('cmd'))
 
[e-mail]
-def assign_from_system():
-    """Transform assignment from system commands (e.g. files = !ls)"""
-    assignment_re = re.compile(r'(?P<lhs>(\s*)([\w\.]+)((\s*,\s*[\w\.]+)*))'
+assign_system_re = re.compile(r'(?P<lhs>(\s*)([\w\.]+)((\s*,\s*[\w\.]+)*))'
                                r'\s*=\s*!\s*(?P<cmd>.*)')
-    template = '%s = get_ipython().getoutput(%r)'
-    return _special_assignment(assignment_re, template)
+assign_system_template = '%s = get_ipython().getoutput(%r)'
[e-mail]
+def assign_from_system(line):
+    """Transform assignment from system commands (e.g. files = !ls)"""
+    m = assign_system_re.match(line)
+    if m is None:
+        return line
 
[e-mail]
-def assign_from_magic():
-    """Transform assignment from magic commands (e.g. a = %who_ls)"""
-    assignment_re = re.compile(r'(?P<lhs>(\s*)([\w\.]+)((\s*,\s*[\w\.]+)*))'
+    return assign_system_template % m.group('lhs', 'cmd')
+
+assign_magic_re = re.compile(r'(?P<lhs>(\s*)([\w\.]+)((\s*,\s*[\w\.]+)*))'
                                r'\s*=\s*%\s*(?P<cmd>.*)')
-    template = '%s = get_ipython().magic(%r)'
-    return _special_assignment(assignment_re, template)
+assign_magic_template = '%s = get_ipython().magic(%r)'
[e-mail]
+def assign_from_magic(line):
+    """Transform assignment from magic commands (e.g. a = %who_ls)"""
+    m = assign_magic_re.match(line)
+    if m is None:
+        return line
+    
+    return assign_magic_template % m.group('lhs', 'cmd')

IPython/core/tests/test_inputtransformer.py

@@ -254,14 +254,9 b' syntax_ml = \\'
 
 def test_assign_system():
     tt.check_pairs(transform_and_reset(ipt.assign_from_system), syntax['assign_system'])
-    for example in syntax_ml['assign_system']:
-        transform_checker(example, ipt.assign_from_system)
 
 def test_assign_magic():
     tt.check_pairs(transform_and_reset(ipt.assign_from_magic), syntax['assign_magic'])
-    for example in syntax_ml['assign_magic']:
-        transform_checker(example, ipt.assign_from_magic)
-
 
 def test_classic_prompt():
     tt.check_pairs(transform_and_reset(ipt.classic_prompt), syntax['classic_prompt'])
@@ -276,39 +271,70 b' def test_ipy_prompt():'
     for example in syntax_ml['ipy_prompt']:
         transform_checker(example, ipt.ipy_prompt)
 
+def test_assemble_logical_lines():
+    tests = \
+    [ [(u"a = \\", None),
+       (u"123", u"a =  123"),
+      ],
+      [(u"a = \\", None),  # Test resetting when within a multi-line string
+       (u"12 *\\", None),
+       (None, u"a =  12 *"),
+      ],
+    ]
+    for example in tests:
+        transform_checker(example, ipt.assemble_logical_lines)
+
+def test_assemble_python_lines():
+    tests = \
+    [ [(u"a = '''", None),
+       (u"abc'''", u"a = '''\nabc'''"),
+      ],
+      [(u"a = '''", None),  # Test resetting when within a multi-line string
+       (u"def", None),
+       (None, u"a = '''\ndef"),
+      ],
+      [(u"a = [1,", None),
+       (u"2]", u"a = [1,\n2]"),
+      ],
+      [(u"a = [1,", None),  # Test resetting when within a multi-line string
+       (u"2,", None),
+       (None, u"a = [1,\n2,"),
+      ],
+    ]
+    for example in tests:
+        transform_checker(example, ipt.assemble_python_lines)
+
+
 def test_help_end():
     tt.check_pairs(transform_and_reset(ipt.help_end), syntax['end_help'])
 
 def test_escaped_noesc():
-    tt.check_pairs(transform_and_reset(ipt.escaped_transformer), syntax['escaped_noesc'])
+    tt.check_pairs(transform_and_reset(ipt.escaped_commands), syntax['escaped_noesc'])
 
 
 def test_escaped_shell():
-    tt.check_pairs(transform_and_reset(ipt.escaped_transformer), syntax['escaped_shell'])
+    tt.check_pairs(transform_and_reset(ipt.escaped_commands), syntax['escaped_shell'])
 
 
 def test_escaped_help():
-    tt.check_pairs(transform_and_reset(ipt.escaped_transformer), syntax['escaped_help'])
+    tt.check_pairs(transform_and_reset(ipt.escaped_commands), syntax['escaped_help'])
 
 
 def test_escaped_magic():
-    tt.check_pairs(transform_and_reset(ipt.escaped_transformer), syntax['escaped_magic'])
+    tt.check_pairs(transform_and_reset(ipt.escaped_commands), syntax['escaped_magic'])
 
 
 def test_escaped_quote():
-    tt.check_pairs(transform_and_reset(ipt.escaped_transformer), syntax['escaped_quote'])
+    tt.check_pairs(transform_and_reset(ipt.escaped_commands), syntax['escaped_quote'])
 
 
 def test_escaped_quote2():
-    tt.check_pairs(transform_and_reset(ipt.escaped_transformer), syntax['escaped_quote2'])
+    tt.check_pairs(transform_and_reset(ipt.escaped_commands), syntax['escaped_quote2'])
 
 
 def test_escaped_paren():
-    tt.check_pairs(transform_and_reset(ipt.escaped_transformer), syntax['escaped_paren'])
+    tt.check_pairs(transform_and_reset(ipt.escaped_commands), syntax['escaped_paren'])
 
-def test_escaped_multiline():
-    for example in syntax_ml['escaped']:
-        transform_checker(example, ipt.escaped_transformer)
 
 def test_cellmagic():
     for example in syntax_ml['cellmagic']: