Fix paste/cpaste bug and refactor/cleanup that code a lot.

In fixing a pasting bug (mishandling of whitespace when input had
prompts) it became clear the pasting code hadn't been updated
when the new prefiltering machinery was added.  Furthermore, the
pasting magics are only for the terminal, but the code was in the
base classes.

This refactors and simplifies the pasting code, moving it to the
terminal shell only, and removing unnecessary methods from the
main class (using small utility functions instead).

The tests were simplified because the previous regexp supported some
odd edge cases that are not valid in the normal prefiltering code.  We
want to have a single location and set of rules for input
prefiltering, so I changed some of the test cases to be consistent
with what prefilter allows.

IPython/core/magic.py

@@ -25,7 +25,6 b' import sys'
 import shutil
 import re
 import time
-import textwrap
 from StringIO import StringIO
 from getopt import getopt,GetoptError
 from pprint import pformat
@@ -3203,65 +3202,6 b' Defaulting color scheme to \'NoColor\'"""'
 
         page.page(self.shell.pycolorize(cont))
 
-    def _rerun_pasted(self):
-        """ Rerun a previously pasted command.
-        """
-        b = self.user_ns.get('pasted_block', None)
-        if b is None:
-            raise UsageError('No previous pasted block available')
-        print "Re-executing '%s...' (%d chars)"% (b.split('\n',1)[0], len(b))
-        exec b in self.user_ns
-
-    def _get_pasted_lines(self, sentinel):
-        """ Yield pasted lines until the user enters the given sentinel value.
-        """
-        from IPython.core import interactiveshell
-        print "Pasting code; enter '%s' alone on the line to stop." % sentinel
-        while True:
-            try:
-                l = self.shell.raw_input_original(':')
-                if l == sentinel:
-                    return
-                else:
-                    yield l
-            except EOFError:
-                print '<EOF>'
-                return
-
-    def _strip_pasted_lines_for_code(self, raw_lines):
-        """ Strip non-code parts of a sequence of lines to return a block of
-        code.
-        """
-        # Regular expressions that declare text we strip from the input:
-        strip_re =  [r'^\s*In \[\d+\]:', # IPython input prompt
-                     r'^\s*(\s?>)+', # Python input prompt
-                     r'^\s*\.{3,}', # Continuation prompts
-                     r'^\++',
-                     ]
-
-        strip_from_start = map(re.compile,strip_re)
-
-        lines = []
-        for l in raw_lines:
-            for pat in strip_from_start:
-                l = pat.sub('',l)
-            lines.append(l)
-
-        block = "\n".join(lines) + '\n'
-        #print "block:\n",block
-        return block
-
-    def _execute_block(self, block, par):
-        """ Execute a block, or store it in a variable, per the user's request.
-        """
-        if not par:
-            b = textwrap.dedent(block)
-            self.user_ns['pasted_block'] = b
-            self.run_cell(b)
-        else:
-            self.user_ns[par] = SList(block.splitlines())
-            print "Block assigned to '%s'" % par
-
     def magic_quickref(self,arg):
         """ Show a quick reference sheet """
         import IPython.core.usage

IPython/core/tests/test_magic.py

@@ -355,14 +355,15 b' def test_cpaste():'
         _ip.user_ns['code_ran'] = True
         return 'run' # return string so '+ run()' doesn't result in success
 
-    tests = {'pass': ["> > > run()",
-                      ">>> > run()",
-                      "+++ run()",
-                      "++ run()",
-                      "  >>> run()"],
-
-             'fail': ["+ + run()",
-                      " ++ run()"]}
+    tests = {'pass': ["run()",
+                      "In [1]: run()",
+                      "In [1]: if 1:\n   ...:     run()",
+                      ">>> run()",
+                      "   >>> run()",
+                      ],
+
+             'fail': ["1 + run()",
+                      "++ run()"]}
 
     _ip.user_ns['run'] = run
 

IPython/frontend/terminal/interactiveshell.py

@@ -19,6 +19,7 b' import bdb'
 import os
 import re
 import sys
+import textwrap
 
 try:
     from contextlib import nested
@@ -35,7 +36,7 b' from IPython.utils import py3compat'
 from IPython.utils.terminal import toggle_set_term_title, set_term_title
 from IPython.utils.process import abbrev_cwd
 from IPython.utils.warn import warn, error
-from IPython.utils.text import num_ini_spaces
+from IPython.utils.text import num_ini_spaces, SList
 from IPython.utils.traitlets import Integer, CBool, Unicode
 
 #-----------------------------------------------------------------------------
@@ -52,6 +53,48 b' def get_default_editor():'
             ed = 'notepad' # same in Windows!
     return ed
 
+
+def get_pasted_lines(sentinel, input=raw_input):
+    """ Yield pasted lines until the user enters the given sentinel value.
+    """
+    print "Pasting code; enter '%s' alone on the line to stop or use Ctrl-D." \
+          % sentinel
+    while True:
+        try:
+            l = input(':')
+            if l == sentinel:
+                return
+            else:
+                yield l
+        except EOFError:
+            print '<EOF>'
+            return
+
+
+def store_or_execute(shell, block, name):
+    """ Execute a block, or store it in a variable, per the user's request.
+    """
+    if name:
+        # If storing it for further editing, run the prefilter on it
+        shell.user_ns[name] = SList(shell.prefilter(block).splitlines())
+        print "Block assigned to '%s'" % name
+    else:
+        # For execution we just dedent it, as all other filtering is
+        # automatically applied by run_cell
+        b = textwrap.dedent(block)
+        shell.user_ns['pasted_block'] = b
+        shell.run_cell(b)
+
+
+def rerun_pasted(shell):
+    """ Rerun a previously pasted command.
+    """
+    b = shell.user_ns.get('pasted_block')
+    if b is None:
+        raise UsageError('No previous pasted block available')
+    print "Re-executing '%s...' (%d chars)"% (b.split('\n',1)[0], len(b))
+    shell.run_cell(b)
+
 #-----------------------------------------------------------------------------
 # Main class
 #-----------------------------------------------------------------------------
@@ -523,9 +566,9 b' class TerminalInteractiveShell(InteractiveShell):'
     def magic_cpaste(self, parameter_s=''):
         """Paste & execute a pre-formatted code block from clipboard.
 
-        You must terminate the block with '--' (two minus-signs) or Ctrl-D alone on the
-        line. You can also provide your own sentinel with '%paste -s %%' ('%%'
-        is the new sentinel for this operation)
+        You must terminate the block with '--' (two minus-signs) or Ctrl-D
+        alone on the line. You can also provide your own sentinel with '%paste
+        -s %%' ('%%' is the new sentinel for this operation)
 
         The block is dedented prior to execution to enable execution of method
         definitions. '>' and '+' characters at the beginning of a line are
@@ -562,18 +605,15 b' class TerminalInteractiveShell(InteractiveShell):'
           Hello world!
         """
 
-        opts,args = self.parse_options(parameter_s,'rs:',mode='string')
-        par = args.strip()
-        if opts.has_key('r'):
-            self._rerun_pasted()
+        opts, args = self.parse_options(parameter_s, 'rs:', mode='string')
+        name = args.strip()
+        if 'r' in opts:
+            rerun_pasted(self.shell)
             return
 
-        sentinel = opts.get('s','--')
-
-        block = self._strip_pasted_lines_for_code(
-            self._get_pasted_lines(sentinel))
-
-        self._execute_block(block, par)
+        sentinel = opts.get('s', '--')
+        block = '\n'.join(get_pasted_lines(sentinel))
+        store_or_execute(self.shell, block, name)
 
     def magic_paste(self, parameter_s=''):
         """Paste & execute a pre-formatted code block from clipboard.
@@ -606,10 +646,10 b' class TerminalInteractiveShell(InteractiveShell):'
         --------
         cpaste: manually paste code into terminal until you mark its end.
         """
-        opts,args = self.parse_options(parameter_s,'rq',mode='string')
-        par = args.strip()
-        if opts.has_key('r'):
-            self._rerun_pasted()
+        opts, args = self.parse_options(parameter_s, 'rq', mode='string')
+        name = args.strip()
+        if 'r' in opts:
+            rerun_pasted(self.shell)
             return
         try:
             text = self.shell.hooks.clipboard_get()
@@ -623,14 +663,14 b' class TerminalInteractiveShell(InteractiveShell):'
             return
 
         # By default, echo back to terminal unless quiet mode is requested
-        if not opts.has_key('q'):
+        if 'q' not in opts:
             write = self.shell.write
             write(self.shell.pycolorize(block))
             if not block.endswith('\n'):
                 write('\n')
             write("## -- End pasted text --\n")
 
-        self._execute_block(block, par)
+        store_or_execute(self.shell, block, name)
 
     if sys.platform == 'win32':
         def magic_cls(self, s):