Merge pull request #2299 from takluyver/remove-duplicate-input-transforms

Remove code from prefilter that duplicates functionality in inputsplitter

This is the first step towards implementing IPEP 2 (#2293). Removed all the static transformations from prefilter, because we're relying on the equivalent functionality in inputsplitter.

Note that this is a backwards-incompatible change for anyone who might have relied on the low-level details of the prefiltering machinery.  Regular users of the IPython applications themselves will not see any changes in behavior.

IPython/core/prefilter.py

@@ -391,85 +391,6 b' class PrefilterTransformer(Configurable):'
             self.__class__.__name__, self.priority, self.enabled)
 
 
-_assign_system_re = re.compile(r'(?P<lhs>(\s*)([\w\.]+)((\s*,\s*[\w\.]+)*))'
-                               r'\s*=\s*!(?P<cmd>.*)')
-
-
-class AssignSystemTransformer(PrefilterTransformer):
-    """Handle the `files = !ls` syntax."""
-
-    priority = Integer(100, config=True)
-
-    def transform(self, line, continue_prompt):
-        m = _assign_system_re.match(line)
-        if m is not None:
-            cmd = m.group('cmd')
-            lhs = m.group('lhs')
-            expr = "sc =%s" % cmd
-            new_line = '%s = get_ipython().magic(%r)' % (lhs, expr)
-            return new_line
-        return line
-
-
-_assign_magic_re = re.compile(r'(?P<lhs>(\s*)([\w\.]+)((\s*,\s*[\w\.]+)*))'
-                               r'\s*=\s*%(?P<cmd>.*)')
-
-class AssignMagicTransformer(PrefilterTransformer):
-    """Handle the `a = %who` syntax."""
-
-    priority = Integer(200, config=True)
-
-    def transform(self, line, continue_prompt):
-        m = _assign_magic_re.match(line)
-        if m is not None:
-            cmd = m.group('cmd')
-            lhs = m.group('lhs')
-            new_line = '%s = get_ipython().magic(%r)' % (lhs, cmd)
-            return new_line
-        return line
-
-
-_classic_prompt_re = re.compile(r'(^[ \t]*>>> |^[ \t]*\.\.\. )')
-
-class PyPromptTransformer(PrefilterTransformer):
-    """Handle inputs that start with '>>> ' syntax."""
-
-    priority = Integer(50, config=True)
-
-    def transform(self, line, continue_prompt):
-
-        if not line or line.isspace() or line.strip() == '...':
-            # This allows us to recognize multiple input prompts separated by
-            # blank lines and pasted in a single chunk, very common when
-            # pasting doctests or long tutorial passages.
-            return ''
-        m = _classic_prompt_re.match(line)
-        if m:
-            return line[len(m.group(0)):]
-        else:
-            return line
-
-
-_ipy_prompt_re = re.compile(r'(^[ \t]*In \[\d+\]: |^[ \t]*\ \ \ \.\.\.+: )')
-
-class IPyPromptTransformer(PrefilterTransformer):
-    """Handle inputs that start classic IPython prompt syntax."""
-
-    priority = Integer(50, config=True)
-
-    def transform(self, line, continue_prompt):
-
-        if not line or line.isspace() or line.strip() == '...':
-            # This allows us to recognize multiple input prompts separated by
-            # blank lines and pasted in a single chunk, very common when
-            # pasting doctests or long tutorial passages.
-            return ''
-        m = _ipy_prompt_re.match(line)
-        if m:
-            return line[len(m.group(0)):]
-        else:
-            return line
-
 #-----------------------------------------------------------------------------
 # Prefilter checkers
 #-----------------------------------------------------------------------------
@@ -511,15 +432,6 b' class EmacsChecker(PrefilterChecker):'
             return None
 
 
-class ShellEscapeChecker(PrefilterChecker):
-
-    priority = Integer(200, config=True)
-
-    def check(self, line_info):
-        if line_info.line.lstrip().startswith(ESC_SHELL):
-            return self.prefilter_manager.get_handler_by_name('shell')
-
-
 class MacroChecker(PrefilterChecker):
 
     priority = Integer(250, config=True)
@@ -546,43 +458,6 b' class IPyAutocallChecker(PrefilterChecker):'
             return None
 
 
-class MultiLineMagicChecker(PrefilterChecker):
-
-    priority = Integer(400, config=True)
-
-    def check(self, line_info):
-        "Allow ! and !! in multi-line statements if multi_line_specials is on"
-        # Note that this one of the only places we check the first character of
-        # ifun and *not* the pre_char.  Also note that the below test matches
-        # both ! and !!.
-        if line_info.continue_prompt \
-            and self.prefilter_manager.multi_line_specials:
-                if line_info.esc == ESC_MAGIC:
-                    return self.prefilter_manager.get_handler_by_name('magic')
-        else:
-            return None
-
-
-class EscCharsChecker(PrefilterChecker):
-
-    priority = Integer(500, config=True)
-
-    def check(self, line_info):
-        """Check for escape character and return either a handler to handle it,
-        or None if there is no escape char."""
-        if line_info.line[-1] == ESC_HELP \
-               and line_info.esc != ESC_SHELL \
-               and line_info.esc != ESC_SH_CAP:
-            # the ? can be at the end, but *not* for either kind of shell escape,
-            # because a ? can be a vaild final char in a shell cmd
-            return self.prefilter_manager.get_handler_by_name('help')
-        else:
-            if line_info.pre:
-                return None
-            # This returns None like it should if no handler exists
-            return self.prefilter_manager.get_handler_by_esc(line_info.esc)
-
-
 class AssignmentChecker(PrefilterChecker):
 
     priority = Integer(600, config=True)
@@ -742,33 +617,6 b' class AliasHandler(PrefilterHandler):'
         return line_out
 
 
-class ShellEscapeHandler(PrefilterHandler):
-
-    handler_name = Unicode('shell')
-    esc_strings = List([ESC_SHELL, ESC_SH_CAP])
-
-    def handle(self, line_info):
-        """Execute the line in a shell, empty return value"""
-        magic_handler = self.prefilter_manager.get_handler_by_name('magic')
-
-        line = line_info.line
-        if line.lstrip().startswith(ESC_SH_CAP):
-            # rewrite LineInfo's line, ifun and the_rest to properly hold the
-            # call to %sx and the actual command to be executed, so
-            # handle_magic can work correctly.  Note that this works even if
-            # the line is indented, so it handles multi_line_specials
-            # properly.
-            new_rest = line.lstrip()[2:]
-            line_info.line = '%ssx %s' % (ESC_MAGIC, new_rest)
-            line_info.ifun = 'sx'
-            line_info.the_rest = new_rest
-            return magic_handler.handle(line_info)
-        else:
-            cmd = line.lstrip().lstrip(ESC_SHELL)
-            line_out = '%sget_ipython().system(%r)' % (line_info.pre_whitespace, cmd)
-        return line_out
-
-
 class MacroHandler(PrefilterHandler):
     handler_name = Unicode("macro")
 
@@ -865,44 +713,6 b' class AutoHandler(PrefilterHandler):'
         return newcmd
 
 
-class HelpHandler(PrefilterHandler):
-
-    handler_name = Unicode('help')
-    esc_strings = List([ESC_HELP])
-
-    def handle(self, line_info):
-        """Try to get some help for the object.
-
-        obj? or ?obj   -> basic information.
-        obj?? or ??obj -> more details.
-        """
-        normal_handler = self.prefilter_manager.get_handler_by_name('normal')
-        line = line_info.line
-        # We need to make sure that we don't process lines which would be
-        # otherwise valid python, such as "x=1 # what?"
-        try:
-            codeop.compile_command(line)
-        except SyntaxError:
-            # We should only handle as help stuff which is NOT valid syntax
-            if line[0]==ESC_HELP:
-                line = line[1:]
-            elif line[-1]==ESC_HELP:
-                line = line[:-1]
-            if line:
-                #print 'line:<%r>' % line  # dbg
-                self.shell.magic('pinfo %s' % line_info.ifun)
-            else:
-                self.shell.show_usage()
-            return '' # Empty string is needed here!
-        except:
-            raise
-            # Pass any other exceptions through to the normal handler
-            return normal_handler.handle(line_info)
-        else:
-            # If the code compiles ok, we should handle it normally
-            return normal_handler.handle(line_info)
-
-
 class EmacsHandler(PrefilterHandler):
 
     handler_name = Unicode('emacs')
@@ -924,19 +734,12 b' class EmacsHandler(PrefilterHandler):'
 
 
 _default_transformers = [
-    AssignSystemTransformer,
-    AssignMagicTransformer,
-    PyPromptTransformer,
-    IPyPromptTransformer,
 ]
 
 _default_checkers = [
     EmacsChecker,
-    ShellEscapeChecker,
     MacroChecker,
     IPyAutocallChecker,
-    MultiLineMagicChecker,
-    EscCharsChecker,
     AssignmentChecker,
     AutoMagicChecker,
     AliasChecker,
@@ -947,10 +750,8 b' _default_checkers = ['
 _default_handlers = [
     PrefilterHandler,
     AliasHandler,
-    ShellEscapeHandler,
     MacroHandler,
     MagicHandler,
     AutoHandler,
-    HelpHandler,
     EmacsHandler
 ]

IPython/core/tests/test_handlers.py

@@ -75,49 +75,8 b' def test_handlers():'
     # line.
     run([(i,py3compat.u_format(o)) for i,o in \
         [('"no change"', '"no change"'),             # normal
-         (u"!true",       "get_ipython().system({u}'true')"),      # shell_escapes
-         (u"!! true",     "get_ipython().magic({u}'sx  true')"),   # shell_escapes + magic
-         (u"!!true",      "get_ipython().magic({u}'sx true')"),    # shell_escapes + magic
-         (u"%lsmagic",    "get_ipython().magic({u}'lsmagic ')"),   # magic
          (u"lsmagic",     "get_ipython().magic({u}'lsmagic ')"),   # magic
          #("a = b # PYTHON-MODE", '_i'),          # emacs -- avoids _in cache
-
-         # post-esc-char whitespace goes inside
-         (u"! true",   "get_ipython().system({u}' true')"),
-
-         # handle_help
-
-         # These are weak tests -- just looking at what the help handlers
-         # logs, which is not how it really does its work.  But it still
-         # lets us check the key paths through the handler.
-
-         ("x=1 # what?", "x=1 # what?"), # no help if valid python
-         ]])
-
-    # multi_line_specials
-    ip.prefilter_manager.multi_line_specials = False
-    # W/ multi_line_specials off, leading ws kills esc chars/autoexpansion
-    run([
-        (u'if 1:\n    !true',    u'if 1:\n    !true'),
-        (u'if 1:\n    lsmagic',  u'if 1:\n    lsmagic'),
-        (u'if 1:\n    an_alias', u'if 1:\n    an_alias'),
-        ])
-
-    ip.prefilter_manager.multi_line_specials = True
-    # initial indents must be preserved.
-    run([(i,py3compat.u_format(o)) for i,o in \
-        [(u'if 1:\n    !true',    "if 1:\n    get_ipython().system({u}'true')"),
-         (u'if 2:\n    lsmagic',  "if 2:\n    get_ipython().magic({u}'lsmagic ')"),
-         (u'if 1:\n    an_alias', "if 1:\n    get_ipython().system({u}'true ')"),
-         # Weird one
-         (u'if 1:\n    !!true',   "if 1:\n    get_ipython().magic({u}'sx true')"),
-
-         # Even with m_l_s on, autocall is off even with special chars
-         ('if 1:\n    /fun 1 2', 'if 1:\n    /fun 1 2'),
-         ('if 1:\n    ;fun 1 2', 'if 1:\n    ;fun 1 2'),
-         ('if 1:\n    ,fun 1 2', 'if 1:\n    ,fun 1 2'),
-         ('if 1:\n    ?fun 1 2', 'if 1:\n    ?fun 1 2'),
-         # What about !!
          ]])
 
     # Objects which are instances of IPyAutocall are *always* autocalled
@@ -133,15 +92,9 b' def test_handlers():'
         ('autocallable',    'autocallable()'),
         # Don't add extra brackets (gh-1117)
         ('autocallable()',    'autocallable()'),
-        (",list 1 2 3",     'list("1", "2", "3")'),
-        (";list 1 2 3",     'list("1 2 3")'),
-        ("/len range(1,4)", 'len(range(1,4))'),
         ])
     ip.magic('autocall 1')
     run([
-        (",list 1 2 3", 'list("1", "2", "3")'),
-        (";list 1 2 3", 'list("1 2 3")'),
-        ("/len range(1,4)", 'len(range(1,4))'),
         ('len "abc"', 'len("abc")'),
         ('len "abc";', 'len("abc");'),  # ; is special -- moves out of parens
         # Autocall is turned off if first arg is [] and the object
@@ -153,9 +106,6 b' def test_handlers():'
         ])
     ip.magic('autocall 2')
     run([
-        (",list 1 2 3", 'list("1", "2", "3")'),
-        (";list 1 2 3", 'list("1 2 3")'),
-        ("/len range(1,4)", 'len(range(1,4))'),
         ('len "abc"', 'len("abc")'),
         ('len "abc";', 'len("abc");'),
         ('len [1,2]', 'len([1,2])'),

IPython/core/tests/test_magic.py

@@ -283,22 +283,18 b' def test_tb_syntaxerror():'
     nt.assert_equal(last_line, "SyntaxError: invalid syntax")
 
 
[e-mail]_refactor_print
-def doctest_time():
-    """
-    In [10]: %time None
-    CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
-    Wall time: 0.00 s
+def test_time():
+    ip = get_ipython()
     
-    In [11]: def f(kmjy):
-       ....:    %time print 2*kmjy
+    with tt.AssertPrints("CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s"):
+        ip.run_cell("%time None")
        
-    In [12]: f(3)
-    6
-    CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
-    Wall time: 0.00 s
-    """
+    ip.run_cell("def f(kmjy):\n"
+                "    %time print (2*kmjy)")
 
+    with tt.AssertPrints("CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s"):
+        with tt.AssertPrints("hihi", suppress=False):
+            ip.run_cell("f('hi')")
 
 def test_doctest_mode():
     "Toggle doctest_mode twice, it should be a no-op and run without error"

IPython/core/tests/test_prefilter.py

@@ -20,17 +20,6 b' def test_prefilter():'
 
     # pairs of (raw, expected correct) input
     pairs = [ ('2+2','2+2'),
-              ('>>> 2+2','2+2'),
-              ('>>> # This is a comment\n'
-               '... 2+2',
-               '# This is a comment\n'
-               '2+2'),
-              # Some IPython input
-              ('In [1]: 1', '1'),
-              ('In [2]: for i in range(5):\n'
-               '   ...:     print i,',
-               'for i in range(5):\n'
-               '    print i,'),
              ]
 
     for raw, correct in pairs:

IPython/testing/plugin/ipdoctest.py

@@ -47,6 +47,12 b' import nose.core'
 from nose.plugins import doctests, Plugin
 from nose.util import anyp, getpackage, test_address, resolve_name, tolist
 
+# Our own imports
+
+# We're temporarily using TerminalMagics.cleanup_input() until the functionality
+# is moved into core.
+from IPython.frontend.terminal.interactiveshell import TerminalMagics
+
 #-----------------------------------------------------------------------------
 # Module globals and other constants
 #-----------------------------------------------------------------------------
@@ -380,17 +386,7 b' class IPDocTestParser(doctest.DocTestParser):'
 
     def ip2py(self,source):
         """Convert input IPython source into valid Python."""
-        out = []
-        newline = out.append
-        #print 'IPSRC:\n',source,'\n###'  # dbg
-        # The input source must be first stripped of all bracketing whitespace
-        # and turned into lines, so it looks to the parser like regular user
-        # input
-        for lnum,line in enumerate(source.strip().splitlines()):
-            newline(_ip.prefilter(line,lnum>0))
-        newline('')  # ensure a closing newline, needed by doctest
-        #print "PYSRC:", '\n'.join(out)  # dbg
-        return '\n'.join(out)
+        return TerminalMagics(_ip).cleanup_input(source)
 
     def parse(self, string, name='<string>'):
         """

docs/source/whatsnew/development.txt

@@ -11,3 +11,13 b' especially intenting/deindenting blocks that is now bound to Ctrl+] and ctr+['
 * Exception types can now be displayed with a custom traceback, by defining a
   ``_render_traceback_()`` method which returns a list of strings, each
   containing one line of the traceback.
+
+Backwards incompatible changes
+------------------------------
+
+* Calling :meth:`InteractiveShell.prefilter` will no longer perform static
+  transformations - the processing of escaped commands such as ``%magic`` and
+  ``!system``, and stripping input prompts from code blocks. This functionality
+  was duplicated in :mod:`IPython.core.inputsplitter`, and the latter version
+  was already what IPython relied on. A new API to transform input will be ready
+  before release.