Deactivate prompt transformers if the first line matches certain patterns

Deactivate the >>> prompt stripper if the first line starts with % or !,
because those are obviously IPython syntax.

Disable the IPython 'In [1]:' prompt stripper if the first line starts
with %%, because we don't want to interfere with the copy of a cell
magic.

This is all 'what did the user mean' guesswork, but I think this makes
our guesses slightly smarter.

We actually had one test that checked that this wasn't happening; I've
adjusted it.

Closes gh-8791

IPython/core/inputtransformer.py

@@ -400,7 +400,7 def cellmagic(end_on_blank_line=False):
         line = tpl % (magic_name, first, u'\n'.join(body))
 
 
-def _strip_prompts(prompt_re, initial_re=None):
+def _strip_prompts(prompt_re, initial_re=None, turnoff_re=None):
     """Remove matching input prompts from a block of input.
     
     Parameters
@@ -418,6 +418,12 def _strip_prompts(prompt_re, initial_re=None):
     If any prompt is found on the first two lines,
     prompts will be stripped from the rest of the block.
     """
+    def pass_thru(line1):
+        "Pass lines through unaltered until the end of the cell"
+        line = line1
+        while line is not None:
+            line = (yield line)
+
     if initial_re is None:
         initial_re = prompt_re
     line = ''
@@ -428,6 +434,14 def _strip_prompts(prompt_re, initial_re=None):
         if line is None:
             continue
         out, n1 = initial_re.subn('', line, count=1)
+        if turnoff_re and not n1:
+            if turnoff_re.match(line):
+                # We're in e.g. a cell magic; disable this transformer for
+                # the rest of the cell.
+                yield from pass_thru(line)
+                line = None
+                continue
+
         line = (yield out)
         
         if line is None:
@@ -446,8 +460,8 def _strip_prompts(prompt_re, initial_re=None):
         
         else:
             # Prompts not in input - wait for reset
-            while line is not None:
-                line = (yield line)
+            yield from pass_thru(line)
+            line = None
 
 @CoroutineInputTransformer.wrap
 def classic_prompt():
@@ -455,14 +469,18 def classic_prompt():
     # FIXME: non-capturing version (?:...) usable?
     prompt_re = re.compile(r'^(>>>|\.\.\.)( |$)')
     initial_re = re.compile(r'^>>>( |$)')
-    return _strip_prompts(prompt_re, initial_re)
+    # Any %magic/!system is IPython syntax, so we needn't look for >>> prompts
+    turnoff_re = re.compile(r'^[%!]')
+    return _strip_prompts(prompt_re, initial_re, turnoff_re)
 
 @CoroutineInputTransformer.wrap
 def ipy_prompt():
     """Strip IPython's In [1]:/...: prompts."""
     # FIXME: non-capturing version (?:...) usable?
     prompt_re = re.compile(r'^(In \[\d+\]: |\s*\.{3,}: ?)')
-    return _strip_prompts(prompt_re)
+    # Disable prompt stripping inside cell magics
+    turnoff_re = re.compile(r'^%%')
+    return _strip_prompts(prompt_re, turnoff_re=turnoff_re)
 
 
 @CoroutineInputTransformer.wrap

IPython/core/tests/test_inputsplitter.py

@@ -452,7 +452,7 class IPythonInputTestCase(InputSplitterTestCase):
             ("%%cellm a\nIn[1]:", u'cellm', u'a', u'In[1]:'),
             ("%%cellm \nline\n>>> hi", u'cellm', u'', u'line\n>>> hi'),
             (">>> %%cellm \nline\n>>> hi", u'cellm', u'', u'line\nhi'),
-            ("%%cellm \n>>> hi", u'cellm', u'', u'hi'),
+            ("%%cellm \n>>> hi", u'cellm', u'', u'>>> hi'),
             ("%%cellm \nline1\nline2", u'cellm', u'', u'line1\nline2'),
             ("%%cellm \nline1\\\\\nline2", u'cellm', u'', u'line1\\\\\nline2'),
         ]:

IPython/core/tests/test_inputtransformer.py

@@ -360,12 +360,25 def test_classic_prompt():
     for example in syntax_ml['multiline_datastructure_prompt']:
         transform_checker(example, ipt.classic_prompt)
 
+    # Check that we don't transform the second line if the first is obviously
+    # IPython syntax
+    transform_checker([
+        (u'%foo', '%foo'),
+        (u'>>> bar', '>>> bar'),
+    ], ipt.classic_prompt)
+
 
 def test_ipy_prompt():
     tt.check_pairs(transform_and_reset(ipt.ipy_prompt), syntax['ipy_prompt'])
     for example in syntax_ml['ipy_prompt']:
         transform_checker(example, ipt.ipy_prompt)
 
+    # Check that we don't transform the second line if we're inside a cell magic
+    transform_checker([
+        (u'%%foo', '%%foo'),
+        (u'In [1]: bar', 'In [1]: bar'),
+    ], ipt.ipy_prompt)
+
 def test_coding_cookie():
     tt.check_pairs(transform_and_reset(ipt.strip_encoding_cookie), syntax['strip_encoding_cookie'])
     for example in syntax_ml['strip_encoding_cookie']: