More input transformers

IPython/core/inputtransformer.py

@@ -17,6 +17,8 b' class InputTransformer(object):'
     @abc.abstractmethod
     def reset(self):
         pass
+    
+    look_in_string = False
 
 class StatelessInputTransformer(InputTransformer):
     """Decorator for a stateless input transformer implemented as a function."""
@@ -96,7 +98,7 b' def cellmagic():'
     line = ''
     while True:
         line = (yield line)
-        if not line.startswith(ESC_MAGIC2):
+        if (not line) or (not line.startswith(ESC_MAGIC2)):
             continue
         
         first = line
@@ -110,3 +112,64 b' def cellmagic():'
         magic_name, _, first = first.partition(' ')
         magic_name = magic_name.lstrip(ESC_MAGIC2)
         line = tpl % (magic_name, first, '\n'.join(body))
+
+def _strip_prompts(prompt1_re, prompt2_re):
+    """Remove matching input prompts from a block of input."""
+    line = ''
+    while True:
+        line = (yield line)
+        
+        if line is None:
+            continue
+        
+        m = prompt1_re.match(line)
+        if m:
+            while m:
+                line = (yield line[len(m.group(0)):])
+                if line is None:
+                    break
+                m = prompt2_re.match(line)
+        else:
+            # Prompts not in input - wait for reset
+            while line is not None:
+                line = (yield line)
+
+@CoroutineInputTransformer
+def classic_prompt():
+    prompt1_re = re.compile(r'^(>>> )')
+    prompt2_re = re.compile(r'^(>>> |^\.\.\. )')
+    return _strip_prompts(prompt1_re, prompt2_re)
+
+classic_prompt.look_in_string = True
+
+@CoroutineInputTransformer
+def ipy_prompt():
+    prompt1_re = re.compile(r'^In \[\d+\]: ')
+    prompt2_re = re.compile(r'^(In \[\d+\]: |^\ \ \ \.\.\.+: )')
+    return _strip_prompts(prompt1_re, prompt2_re)
+
+ipy_prompt.look_in_string = True
+
+@CoroutineInputTransformer
+def leading_indent():
+    space_re = re.compile(r'^[ \t]+')
+    line = ''
+    while True:
+        line = (yield line)
+        
+        if line is None:
+            continue
+        
+        m = space_re.match(line)
+        if m:
+            space = m.group(0)
+            while line is not None:
+                if line.startswith(space):
+                    line = line[len(space):]
+                line = (yield line)
+        else:
+            # No leading spaces - wait for reset
+            while line is not None:
+                line = (yield line)
+
+leading_indent.look_in_string = True

IPython/core/tests/test_inputtransformer.py

@@ -9,25 +9,28 b' from IPython.core.tests.test_inputsplitter import syntax'
 
 def wrap_transform(transformer):
     def transform(inp):
+        results = []
         for line in inp:
             res = transformer.push(line)
             if res is not None:
-                return res
-        return transformer.push(None)
+                results.append(res)
+        transformer.reset()
+        return results
     
     return transform
 
 cellmagic_tests = [
-(['%%foo a'], "get_ipython().run_cell_magic('foo', 'a', '')"),
-(['%%bar 123', 'hello', ''], "get_ipython().run_cell_magic('bar', '123', 'hello')"),
+(['%%foo a', None], ["get_ipython().run_cell_magic('foo', 'a', '')"]),
+(['%%bar 123', 'hello', ''], ["get_ipython().run_cell_magic('bar', '123', 'hello')"]),
 ]
 
 def test_transform_cellmagic():
     tt.check_pairs(wrap_transform(inputtransformer.cellmagic), cellmagic_tests)
 
-esctransform_tests = [(i, py3compat.u_format(o)) for i,o in [
-(['%pdef zip'], "get_ipython().magic({u}'pdef zip')"),
-(['%abc def \\', 'ghi'], "get_ipython().magic({u}'abc def  ghi')"),
+esctransform_tests = [(i, [py3compat.u_format(ol) for ol in o]) for i,o in [
+(['%pdef zip'], ["get_ipython().magic({u}'pdef zip')"]),
+(['%abc def \\', 'ghi'], ["get_ipython().magic({u}'abc def  ghi')"]),
+(['%abc def \\', 'ghi\\', None], ["get_ipython().magic({u}'abc def  ghi')"]),
 ]]
 
 def test_transform_escaped():
@@ -35,3 +38,31 b' def test_transform_escaped():'
 
 def endhelp_test():
     tt.check_pairs(inputtransformer.transform_help_end.push, syntax['end_help'])
+
+classic_prompt_tests = [
+(['>>> a=1'], ['a=1']),
+(['>>> a="""','... 123"""'], ['a="""', '123"""']),
+(['a="""','... 123"""'], ['a="""', '... 123"""']),
+]
+
+def test_classic_prompt():
+    tt.check_pairs(wrap_transform(inputtransformer.classic_prompt), classic_prompt_tests)
+
+ipy_prompt_tests = [
+(['In [1]: a=1'], ['a=1']),
+(['In [2]: a="""','   ...: 123"""'], ['a="""', '123"""']),
+(['a="""','   ...: 123"""'], ['a="""', '   ...: 123"""']),
+]
+
+def test_ipy_prompt():
+    tt.check_pairs(wrap_transform(inputtransformer.ipy_prompt), ipy_prompt_tests)
+
+leading_indent_tests = [
+(['    print "hi"'], ['print "hi"']),
+(['  for a in range(5):', '    a*2'], ['for a in range(5):', '  a*2']),
+(['    a="""','    123"""'], ['a="""', '123"""']),
+(['a="""','    123"""'], ['a="""', '    123"""']),
+]
+
+def test_leading_indent():
+    tt.check_pairs(wrap_transform(inputtransformer.leading_indent), leading_indent_tests)