Simplify input transformers

More can now be implemented as stateless transformers

IPython/core/inputsplitter.py

@@ -79,7 +79,7 b' from IPython.core.inputtransformer import (leading_indent,'
                                            cellmagic,
                                            assemble_logical_lines,
                                            help_end,
-                                           escaped_transformer,
+                                           escaped_commands,
                                            assign_from_magic,
                                            assign_from_system,
                                            assemble_python_lines,
@@ -526,7 +526,7 b' class IPythonInputSplitter(InputSplitter):'
         self.logical_line_transforms = logical_line_transforms or \
                 [cellmagic(),
                  help_end(),
-                 escaped_transformer(),
+                 escaped_commands(),
                  assign_from_magic(),
                  assign_from_system(),
                 ]

IPython/core/inputtransformer.py

@@ -61,9 +61,6 b' class InputTransformer(object):'
         """
         pass
     
-    # Set this to True to allow the transformer to act on lines inside strings.
-    look_in_string = False
-    
     @classmethod
     def wrap(cls, func):
         """Can be used by subclasses as a decorator, to return a factory that
@@ -71,10 +68,7 b' class InputTransformer(object):'
         """
         @functools.wraps(func)
         def transformer_factory():
-            transformer = cls(func)
-            if getattr(transformer_factory, 'look_in_string', False):
-                transformer.look_in_string = True
-            return transformer
+            return cls(func)
         
         return transformer_factory
 
@@ -214,83 +208,67 b' def _make_help_call(target, esc, lspace, next_input=None):'
     else:
         return '%sget_ipython().set_next_input(%r);get_ipython().magic(%r)' % \
            (lspace, next_input, arg)
-
[e-mail]
-def escaped_transformer():
-    """Translate lines beginning with one of IPython's escape characters.
     
-    This is stateful to allow magic commands etc. to be continued over several
-    lines using explicit line continuations (\ at the end of a line).
+# These define the transformations for the different escape characters.
+def _tr_system(line_info):
+    "Translate lines escaped with: !"
+    cmd = line_info.line.lstrip().lstrip(ESC_SHELL)
+    return '%sget_ipython().system(%r)' % (line_info.pre, cmd)
+
+def _tr_system2(line_info):
+    "Translate lines escaped with: !!"
+    cmd = line_info.line.lstrip()[2:]
+    return '%sget_ipython().getoutput(%r)' % (line_info.pre, cmd)
+
+def _tr_help(line_info):
+    "Translate lines escaped with: ?/??"
+    # A naked help line should just fire the intro help screen
+    if not line_info.line[1:]:
+        return 'get_ipython().show_usage()'
+
+    return _make_help_call(line_info.ifun, line_info.esc, line_info.pre)
+
+def _tr_magic(line_info):
+    "Translate lines escaped with: %"
+    tpl = '%sget_ipython().magic(%r)'
+    cmd = ' '.join([line_info.ifun, line_info.the_rest]).strip()
+    return tpl % (line_info.pre, cmd)
+
+def _tr_quote(line_info):
+    "Translate lines escaped with: ,"
+    return '%s%s("%s")' % (line_info.pre, line_info.ifun,
+                         '", "'.join(line_info.the_rest.split()) )
+
+def _tr_quote2(line_info):
+    "Translate lines escaped with: ;"
+    return '%s%s("%s")' % (line_info.pre, line_info.ifun,
+                           line_info.the_rest)
+
+def _tr_paren(line_info):
+    "Translate lines escaped with: /"
+    return '%s%s(%s)' % (line_info.pre, line_info.ifun,
+                         ", ".join(line_info.the_rest.split()))
+
+tr = { ESC_SHELL  : _tr_system,
+       ESC_SH_CAP : _tr_system2,
+       ESC_HELP   : _tr_help,
+       ESC_HELP2  : _tr_help,
+       ESC_MAGIC  : _tr_magic,
+       ESC_QUOTE  : _tr_quote,
+       ESC_QUOTE2 : _tr_quote2,
+       ESC_PAREN  : _tr_paren }
+
[e-mail]
+def escaped_commands(line):
+    """Transform escaped commands - %magic, !system, ?help + various autocalls.
     """
+    if not line or line.isspace():
+        return line
+    lineinf = LineInfo(line)
+    if lineinf.esc not in tr:
+        return line
     
-    # These define the transformations for the different escape characters.
-    def _tr_system(line_info):
-        "Translate lines escaped with: !"
-        cmd = line_info.line.lstrip().lstrip(ESC_SHELL)
-        return '%sget_ipython().system(%r)' % (line_info.pre, cmd)
-
-    def _tr_system2(line_info):
-        "Translate lines escaped with: !!"
-        cmd = line_info.line.lstrip()[2:]
-        return '%sget_ipython().getoutput(%r)' % (line_info.pre, cmd)
-
-    def _tr_help(line_info):
-        "Translate lines escaped with: ?/??"
-        # A naked help line should just fire the intro help screen
-        if not line_info.line[1:]:
-            return 'get_ipython().show_usage()'
-
-        return _make_help_call(line_info.ifun, line_info.esc, line_info.pre)
-
-    def _tr_magic(line_info):
-        "Translate lines escaped with: %"
-        tpl = '%sget_ipython().magic(%r)'
-        cmd = ' '.join([line_info.ifun, line_info.the_rest]).strip()
-        return tpl % (line_info.pre, cmd)
-
-    def _tr_quote(line_info):
-        "Translate lines escaped with: ,"
-        return '%s%s("%s")' % (line_info.pre, line_info.ifun,
-                             '", "'.join(line_info.the_rest.split()) )
-
-    def _tr_quote2(line_info):
-        "Translate lines escaped with: ;"
-        return '%s%s("%s")' % (line_info.pre, line_info.ifun,
-                               line_info.the_rest)
-
-    def _tr_paren(line_info):
-        "Translate lines escaped with: /"
-        return '%s%s(%s)' % (line_info.pre, line_info.ifun,
-                             ", ".join(line_info.the_rest.split()))
-    
-    tr = { ESC_SHELL  : _tr_system,
-           ESC_SH_CAP : _tr_system2,
-           ESC_HELP   : _tr_help,
-           ESC_HELP2  : _tr_help,
-           ESC_MAGIC  : _tr_magic,
-           ESC_QUOTE  : _tr_quote,
-           ESC_QUOTE2 : _tr_quote2,
-           ESC_PAREN  : _tr_paren }
-    
-    line = ''
-    while True:
-        line = (yield line)
-        if not line or line.isspace():
-            continue
-        lineinf = LineInfo(line)
-        if lineinf.esc not in tr:
-            continue
-        
-        parts = []
-        while line is not None:
-            parts.append(line.rstrip('\\'))
-            if not line.endswith('\\'):
-                break
-            line = (yield None)
-        
-        # Output
-        lineinf = LineInfo(' '.join(parts))
-        line = tr[lineinf.esc](lineinf)
+    return tr[lineinf.esc](lineinf)
 
 _initial_space_re = re.compile(r'\s*')
 
@@ -401,8 +379,6 b' def classic_prompt():'
     prompt2_re = re.compile(r'^(>>> |^\.\.\. )')
     return _strip_prompts(prompt1_re, prompt2_re)
 
-classic_prompt.look_in_string = True
-
 @CoroutineInputTransformer.wrap
 def ipy_prompt():
     """Strip IPython's In [1]:/...: prompts."""
@@ -410,8 +386,6 b' def ipy_prompt():'
     prompt2_re = re.compile(r'^(In \[\d+\]: |^\ \ \ \.\.\.+: )')
     return _strip_prompts(prompt1_re, prompt2_re)
 
-ipy_prompt.look_in_string = True
-
 
 @CoroutineInputTransformer.wrap
 def leading_indent():
@@ -440,48 +414,27 b' def leading_indent():'
             while line is not None:
                 line = (yield line)
 
-leading_indent.look_in_string = True
-
 
-def _special_assignment(assignment_re, template):
-    """Transform assignment from system & magic commands.
-    
-    This is stateful so that it can handle magic commands continued on several
-    lines.
-    """
-    line = ''
-    while True:
-        line = (yield line)
-        if not line or line.isspace():
-            continue
-        
-        m = assignment_re.match(line)
-        if not m:
-            continue
-        
-        parts = []
-        while line is not None:
-            parts.append(line.rstrip('\\'))
-            if not line.endswith('\\'):
-                break
-            line = (yield None)
-        
-        # Output
-        whole = assignment_re.match(' '.join(parts))
-        line = template % (whole.group('lhs'), whole.group('cmd'))
-
[e-mail]
-def assign_from_system():
+assign_system_re = re.compile(r'(?P<lhs>(\s*)([\w\.]+)((\s*,\s*[\w\.]+)*))'
+                              r'\s*=\s*!\s*(?P<cmd>.*)')
+assign_system_template = '%s = get_ipython().getoutput(%r)'
[e-mail]
+def assign_from_system(line):
     """Transform assignment from system commands (e.g. files = !ls)"""
-    assignment_re = re.compile(r'(?P<lhs>(\s*)([\w\.]+)((\s*,\s*[\w\.]+)*))'
-                               r'\s*=\s*!\s*(?P<cmd>.*)')
-    template = '%s = get_ipython().getoutput(%r)'
-    return _special_assignment(assignment_re, template)
+    m = assign_system_re.match(line)
+    if m is None:
+        return line
+    
+    return assign_system_template % m.group('lhs', 'cmd')
 
[e-mail]
-def assign_from_magic():
+assign_magic_re = re.compile(r'(?P<lhs>(\s*)([\w\.]+)((\s*,\s*[\w\.]+)*))'
+                             r'\s*=\s*%\s*(?P<cmd>.*)')
+assign_magic_template = '%s = get_ipython().magic(%r)'
[e-mail]
+def assign_from_magic(line):
     """Transform assignment from magic commands (e.g. a = %who_ls)"""
-    assignment_re = re.compile(r'(?P<lhs>(\s*)([\w\.]+)((\s*,\s*[\w\.]+)*))'
-                               r'\s*=\s*%\s*(?P<cmd>.*)')
-    template = '%s = get_ipython().magic(%r)'
-    return _special_assignment(assignment_re, template)
+    m = assign_magic_re.match(line)
+    if m is None:
+        return line
+    
+    return assign_magic_template % m.group('lhs', 'cmd')

IPython/core/tests/test_inputtransformer.py

@@ -254,14 +254,9 b' syntax_ml = \\'
 
 def test_assign_system():
     tt.check_pairs(transform_and_reset(ipt.assign_from_system), syntax['assign_system'])
-    for example in syntax_ml['assign_system']:
-        transform_checker(example, ipt.assign_from_system)
 
 def test_assign_magic():
     tt.check_pairs(transform_and_reset(ipt.assign_from_magic), syntax['assign_magic'])
-    for example in syntax_ml['assign_magic']:
-        transform_checker(example, ipt.assign_from_magic)
-
 
 def test_classic_prompt():
     tt.check_pairs(transform_and_reset(ipt.classic_prompt), syntax['classic_prompt'])
@@ -276,39 +271,70 b' def test_ipy_prompt():'
     for example in syntax_ml['ipy_prompt']:
         transform_checker(example, ipt.ipy_prompt)
 
+def test_assemble_logical_lines():
+    tests = \
+    [ [(u"a = \\", None),
+       (u"123", u"a =  123"),
+      ],
+      [(u"a = \\", None),  # Test resetting when within a multi-line string
+       (u"12 *\\", None),
+       (None, u"a =  12 *"),
+      ],
+    ]
+    for example in tests:
+        transform_checker(example, ipt.assemble_logical_lines)
+
+def test_assemble_python_lines():
+    tests = \
+    [ [(u"a = '''", None),
+       (u"abc'''", u"a = '''\nabc'''"),
+      ],
+      [(u"a = '''", None),  # Test resetting when within a multi-line string
+       (u"def", None),
+       (None, u"a = '''\ndef"),
+      ],
+      [(u"a = [1,", None),
+       (u"2]", u"a = [1,\n2]"),
+      ],
+      [(u"a = [1,", None),  # Test resetting when within a multi-line string
+       (u"2,", None),
+       (None, u"a = [1,\n2,"),
+      ],
+    ]
+    for example in tests:
+        transform_checker(example, ipt.assemble_python_lines)
+
+
 def test_help_end():
     tt.check_pairs(transform_and_reset(ipt.help_end), syntax['end_help'])
 
 def test_escaped_noesc():
-    tt.check_pairs(transform_and_reset(ipt.escaped_transformer), syntax['escaped_noesc'])
+    tt.check_pairs(transform_and_reset(ipt.escaped_commands), syntax['escaped_noesc'])
 
 
 def test_escaped_shell():
-    tt.check_pairs(transform_and_reset(ipt.escaped_transformer), syntax['escaped_shell'])
+    tt.check_pairs(transform_and_reset(ipt.escaped_commands), syntax['escaped_shell'])
 
 
 def test_escaped_help():
-    tt.check_pairs(transform_and_reset(ipt.escaped_transformer), syntax['escaped_help'])
+    tt.check_pairs(transform_and_reset(ipt.escaped_commands), syntax['escaped_help'])
 
 
 def test_escaped_magic():
-    tt.check_pairs(transform_and_reset(ipt.escaped_transformer), syntax['escaped_magic'])
+    tt.check_pairs(transform_and_reset(ipt.escaped_commands), syntax['escaped_magic'])
 
 
 def test_escaped_quote():
-    tt.check_pairs(transform_and_reset(ipt.escaped_transformer), syntax['escaped_quote'])
+    tt.check_pairs(transform_and_reset(ipt.escaped_commands), syntax['escaped_quote'])
 
 
 def test_escaped_quote2():
-    tt.check_pairs(transform_and_reset(ipt.escaped_transformer), syntax['escaped_quote2'])
+    tt.check_pairs(transform_and_reset(ipt.escaped_commands), syntax['escaped_quote2'])
 
 
 def test_escaped_paren():
-    tt.check_pairs(transform_and_reset(ipt.escaped_transformer), syntax['escaped_paren'])
+    tt.check_pairs(transform_and_reset(ipt.escaped_commands), syntax['escaped_paren'])
 
-def test_escaped_multiline():
-    for example in syntax_ml['escaped']:
-        transform_checker(example, ipt.escaped_transformer)
 
 def test_cellmagic():
     for example in syntax_ml['cellmagic']: