From 06a7a5748c9004c49ffedbedc18f52415c74c65b 2012-08-13 22:42:06 From: Fernando Perez Date: 2012-08-13 22:42:06 Subject: [PATCH] Merge pull request #2299 from takluyver/remove-duplicate-input-transforms Remove code from prefilter that duplicates functionality in inputsplitter This is the first step towards implementing IPEP 2 (#2293). Removed all the static transformations from prefilter, because we're relying on the equivalent functionality in inputsplitter. Note that this is a backwards-incompatible change for anyone who might have relied on the low-level details of the prefiltering machinery. Regular users of the IPython applications themselves will not see any changes in behavior. --- diff --git a/IPython/core/prefilter.py b/IPython/core/prefilter.py index 3ccbd54..30f8e03 100644 --- a/IPython/core/prefilter.py +++ b/IPython/core/prefilter.py @@ -391,85 +391,6 @@ class PrefilterTransformer(Configurable): self.__class__.__name__, self.priority, self.enabled) -_assign_system_re = re.compile(r'(?P(\s*)([\w\.]+)((\s*,\s*[\w\.]+)*))' - r'\s*=\s*!(?P.*)') - - -class AssignSystemTransformer(PrefilterTransformer): - """Handle the `files = !ls` syntax.""" - - priority = Integer(100, config=True) - - def transform(self, line, continue_prompt): - m = _assign_system_re.match(line) - if m is not None: - cmd = m.group('cmd') - lhs = m.group('lhs') - expr = "sc =%s" % cmd - new_line = '%s = get_ipython().magic(%r)' % (lhs, expr) - return new_line - return line - - -_assign_magic_re = re.compile(r'(?P(\s*)([\w\.]+)((\s*,\s*[\w\.]+)*))' - r'\s*=\s*%(?P.*)') - -class AssignMagicTransformer(PrefilterTransformer): - """Handle the `a = %who` syntax.""" - - priority = Integer(200, config=True) - - def transform(self, line, continue_prompt): - m = _assign_magic_re.match(line) - if m is not None: - cmd = m.group('cmd') - lhs = m.group('lhs') - new_line = '%s = get_ipython().magic(%r)' % (lhs, cmd) - return new_line - return line - - -_classic_prompt_re = re.compile(r'(^[ \t]*>>> |^[ \t]*\.\.\. )') - -class PyPromptTransformer(PrefilterTransformer): - """Handle inputs that start with '>>> ' syntax.""" - - priority = Integer(50, config=True) - - def transform(self, line, continue_prompt): - - if not line or line.isspace() or line.strip() == '...': - # This allows us to recognize multiple input prompts separated by - # blank lines and pasted in a single chunk, very common when - # pasting doctests or long tutorial passages. - return '' - m = _classic_prompt_re.match(line) - if m: - return line[len(m.group(0)):] - else: - return line - - -_ipy_prompt_re = re.compile(r'(^[ \t]*In \[\d+\]: |^[ \t]*\ \ \ \.\.\.+: )') - -class IPyPromptTransformer(PrefilterTransformer): - """Handle inputs that start classic IPython prompt syntax.""" - - priority = Integer(50, config=True) - - def transform(self, line, continue_prompt): - - if not line or line.isspace() or line.strip() == '...': - # This allows us to recognize multiple input prompts separated by - # blank lines and pasted in a single chunk, very common when - # pasting doctests or long tutorial passages. - return '' - m = _ipy_prompt_re.match(line) - if m: - return line[len(m.group(0)):] - else: - return line - #----------------------------------------------------------------------------- # Prefilter checkers #----------------------------------------------------------------------------- @@ -511,15 +432,6 @@ class EmacsChecker(PrefilterChecker): return None -class ShellEscapeChecker(PrefilterChecker): - - priority = Integer(200, config=True) - - def check(self, line_info): - if line_info.line.lstrip().startswith(ESC_SHELL): - return self.prefilter_manager.get_handler_by_name('shell') - - class MacroChecker(PrefilterChecker): priority = Integer(250, config=True) @@ -546,43 +458,6 @@ class IPyAutocallChecker(PrefilterChecker): return None -class MultiLineMagicChecker(PrefilterChecker): - - priority = Integer(400, config=True) - - def check(self, line_info): - "Allow ! and !! in multi-line statements if multi_line_specials is on" - # Note that this one of the only places we check the first character of - # ifun and *not* the pre_char. Also note that the below test matches - # both ! and !!. - if line_info.continue_prompt \ - and self.prefilter_manager.multi_line_specials: - if line_info.esc == ESC_MAGIC: - return self.prefilter_manager.get_handler_by_name('magic') - else: - return None - - -class EscCharsChecker(PrefilterChecker): - - priority = Integer(500, config=True) - - def check(self, line_info): - """Check for escape character and return either a handler to handle it, - or None if there is no escape char.""" - if line_info.line[-1] == ESC_HELP \ - and line_info.esc != ESC_SHELL \ - and line_info.esc != ESC_SH_CAP: - # the ? can be at the end, but *not* for either kind of shell escape, - # because a ? can be a vaild final char in a shell cmd - return self.prefilter_manager.get_handler_by_name('help') - else: - if line_info.pre: - return None - # This returns None like it should if no handler exists - return self.prefilter_manager.get_handler_by_esc(line_info.esc) - - class AssignmentChecker(PrefilterChecker): priority = Integer(600, config=True) @@ -742,33 +617,6 @@ class AliasHandler(PrefilterHandler): return line_out -class ShellEscapeHandler(PrefilterHandler): - - handler_name = Unicode('shell') - esc_strings = List([ESC_SHELL, ESC_SH_CAP]) - - def handle(self, line_info): - """Execute the line in a shell, empty return value""" - magic_handler = self.prefilter_manager.get_handler_by_name('magic') - - line = line_info.line - if line.lstrip().startswith(ESC_SH_CAP): - # rewrite LineInfo's line, ifun and the_rest to properly hold the - # call to %sx and the actual command to be executed, so - # handle_magic can work correctly. Note that this works even if - # the line is indented, so it handles multi_line_specials - # properly. - new_rest = line.lstrip()[2:] - line_info.line = '%ssx %s' % (ESC_MAGIC, new_rest) - line_info.ifun = 'sx' - line_info.the_rest = new_rest - return magic_handler.handle(line_info) - else: - cmd = line.lstrip().lstrip(ESC_SHELL) - line_out = '%sget_ipython().system(%r)' % (line_info.pre_whitespace, cmd) - return line_out - - class MacroHandler(PrefilterHandler): handler_name = Unicode("macro") @@ -865,44 +713,6 @@ class AutoHandler(PrefilterHandler): return newcmd -class HelpHandler(PrefilterHandler): - - handler_name = Unicode('help') - esc_strings = List([ESC_HELP]) - - def handle(self, line_info): - """Try to get some help for the object. - - obj? or ?obj -> basic information. - obj?? or ??obj -> more details. - """ - normal_handler = self.prefilter_manager.get_handler_by_name('normal') - line = line_info.line - # We need to make sure that we don't process lines which would be - # otherwise valid python, such as "x=1 # what?" - try: - codeop.compile_command(line) - except SyntaxError: - # We should only handle as help stuff which is NOT valid syntax - if line[0]==ESC_HELP: - line = line[1:] - elif line[-1]==ESC_HELP: - line = line[:-1] - if line: - #print 'line:<%r>' % line # dbg - self.shell.magic('pinfo %s' % line_info.ifun) - else: - self.shell.show_usage() - return '' # Empty string is needed here! - except: - raise - # Pass any other exceptions through to the normal handler - return normal_handler.handle(line_info) - else: - # If the code compiles ok, we should handle it normally - return normal_handler.handle(line_info) - - class EmacsHandler(PrefilterHandler): handler_name = Unicode('emacs') @@ -924,19 +734,12 @@ class EmacsHandler(PrefilterHandler): _default_transformers = [ - AssignSystemTransformer, - AssignMagicTransformer, - PyPromptTransformer, - IPyPromptTransformer, ] _default_checkers = [ EmacsChecker, - ShellEscapeChecker, MacroChecker, IPyAutocallChecker, - MultiLineMagicChecker, - EscCharsChecker, AssignmentChecker, AutoMagicChecker, AliasChecker, @@ -947,10 +750,8 @@ _default_checkers = [ _default_handlers = [ PrefilterHandler, AliasHandler, - ShellEscapeHandler, MacroHandler, MagicHandler, AutoHandler, - HelpHandler, EmacsHandler ] diff --git a/IPython/core/tests/test_handlers.py b/IPython/core/tests/test_handlers.py index 683f224..5601717 100644 --- a/IPython/core/tests/test_handlers.py +++ b/IPython/core/tests/test_handlers.py @@ -75,49 +75,8 @@ def test_handlers(): # line. run([(i,py3compat.u_format(o)) for i,o in \ [('"no change"', '"no change"'), # normal - (u"!true", "get_ipython().system({u}'true')"), # shell_escapes - (u"!! true", "get_ipython().magic({u}'sx true')"), # shell_escapes + magic - (u"!!true", "get_ipython().magic({u}'sx true')"), # shell_escapes + magic - (u"%lsmagic", "get_ipython().magic({u}'lsmagic ')"), # magic (u"lsmagic", "get_ipython().magic({u}'lsmagic ')"), # magic #("a = b # PYTHON-MODE", '_i'), # emacs -- avoids _in cache - - # post-esc-char whitespace goes inside - (u"! true", "get_ipython().system({u}' true')"), - - # handle_help - - # These are weak tests -- just looking at what the help handlers - # logs, which is not how it really does its work. But it still - # lets us check the key paths through the handler. - - ("x=1 # what?", "x=1 # what?"), # no help if valid python - ]]) - - # multi_line_specials - ip.prefilter_manager.multi_line_specials = False - # W/ multi_line_specials off, leading ws kills esc chars/autoexpansion - run([ - (u'if 1:\n !true', u'if 1:\n !true'), - (u'if 1:\n lsmagic', u'if 1:\n lsmagic'), - (u'if 1:\n an_alias', u'if 1:\n an_alias'), - ]) - - ip.prefilter_manager.multi_line_specials = True - # initial indents must be preserved. - run([(i,py3compat.u_format(o)) for i,o in \ - [(u'if 1:\n !true', "if 1:\n get_ipython().system({u}'true')"), - (u'if 2:\n lsmagic', "if 2:\n get_ipython().magic({u}'lsmagic ')"), - (u'if 1:\n an_alias', "if 1:\n get_ipython().system({u}'true ')"), - # Weird one - (u'if 1:\n !!true', "if 1:\n get_ipython().magic({u}'sx true')"), - - # Even with m_l_s on, autocall is off even with special chars - ('if 1:\n /fun 1 2', 'if 1:\n /fun 1 2'), - ('if 1:\n ;fun 1 2', 'if 1:\n ;fun 1 2'), - ('if 1:\n ,fun 1 2', 'if 1:\n ,fun 1 2'), - ('if 1:\n ?fun 1 2', 'if 1:\n ?fun 1 2'), - # What about !! ]]) # Objects which are instances of IPyAutocall are *always* autocalled @@ -133,15 +92,9 @@ def test_handlers(): ('autocallable', 'autocallable()'), # Don't add extra brackets (gh-1117) ('autocallable()', 'autocallable()'), - (",list 1 2 3", 'list("1", "2", "3")'), - (";list 1 2 3", 'list("1 2 3")'), - ("/len range(1,4)", 'len(range(1,4))'), ]) ip.magic('autocall 1') run([ - (",list 1 2 3", 'list("1", "2", "3")'), - (";list 1 2 3", 'list("1 2 3")'), - ("/len range(1,4)", 'len(range(1,4))'), ('len "abc"', 'len("abc")'), ('len "abc";', 'len("abc");'), # ; is special -- moves out of parens # Autocall is turned off if first arg is [] and the object @@ -153,9 +106,6 @@ def test_handlers(): ]) ip.magic('autocall 2') run([ - (",list 1 2 3", 'list("1", "2", "3")'), - (";list 1 2 3", 'list("1 2 3")'), - ("/len range(1,4)", 'len(range(1,4))'), ('len "abc"', 'len("abc")'), ('len "abc";', 'len("abc");'), ('len [1,2]', 'len([1,2])'), diff --git a/IPython/core/tests/test_magic.py b/IPython/core/tests/test_magic.py index 8a69bff..ae6e211 100644 --- a/IPython/core/tests/test_magic.py +++ b/IPython/core/tests/test_magic.py @@ -283,22 +283,18 @@ def test_tb_syntaxerror(): nt.assert_equal(last_line, "SyntaxError: invalid syntax") -@py3compat.doctest_refactor_print -def doctest_time(): - """ - In [10]: %time None - CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s - Wall time: 0.00 s +def test_time(): + ip = get_ipython() - In [11]: def f(kmjy): - ....: %time print 2*kmjy - - In [12]: f(3) - 6 - CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s - Wall time: 0.00 s - """ - + with tt.AssertPrints("CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s"): + ip.run_cell("%time None") + + ip.run_cell("def f(kmjy):\n" + " %time print (2*kmjy)") + + with tt.AssertPrints("CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s"): + with tt.AssertPrints("hihi", suppress=False): + ip.run_cell("f('hi')") def test_doctest_mode(): "Toggle doctest_mode twice, it should be a no-op and run without error" diff --git a/IPython/core/tests/test_prefilter.py b/IPython/core/tests/test_prefilter.py index c5823e1..3a7eee6 100644 --- a/IPython/core/tests/test_prefilter.py +++ b/IPython/core/tests/test_prefilter.py @@ -20,17 +20,6 @@ def test_prefilter(): # pairs of (raw, expected correct) input pairs = [ ('2+2','2+2'), - ('>>> 2+2','2+2'), - ('>>> # This is a comment\n' - '... 2+2', - '# This is a comment\n' - '2+2'), - # Some IPython input - ('In [1]: 1', '1'), - ('In [2]: for i in range(5):\n' - ' ...: print i,', - 'for i in range(5):\n' - ' print i,'), ] for raw, correct in pairs: diff --git a/IPython/testing/plugin/ipdoctest.py b/IPython/testing/plugin/ipdoctest.py index 66fa697..a939b6d 100644 --- a/IPython/testing/plugin/ipdoctest.py +++ b/IPython/testing/plugin/ipdoctest.py @@ -47,6 +47,12 @@ import nose.core from nose.plugins import doctests, Plugin from nose.util import anyp, getpackage, test_address, resolve_name, tolist +# Our own imports + +# We're temporarily using TerminalMagics.cleanup_input() until the functionality +# is moved into core. +from IPython.frontend.terminal.interactiveshell import TerminalMagics + #----------------------------------------------------------------------------- # Module globals and other constants #----------------------------------------------------------------------------- @@ -380,17 +386,7 @@ class IPDocTestParser(doctest.DocTestParser): def ip2py(self,source): """Convert input IPython source into valid Python.""" - out = [] - newline = out.append - #print 'IPSRC:\n',source,'\n###' # dbg - # The input source must be first stripped of all bracketing whitespace - # and turned into lines, so it looks to the parser like regular user - # input - for lnum,line in enumerate(source.strip().splitlines()): - newline(_ip.prefilter(line,lnum>0)) - newline('') # ensure a closing newline, needed by doctest - #print "PYSRC:", '\n'.join(out) # dbg - return '\n'.join(out) + return TerminalMagics(_ip).cleanup_input(source) def parse(self, string, name=''): """ diff --git a/docs/source/whatsnew/development.txt b/docs/source/whatsnew/development.txt index 0bd90e9..b0a8b48 100644 --- a/docs/source/whatsnew/development.txt +++ b/docs/source/whatsnew/development.txt @@ -11,3 +11,13 @@ especially intenting/deindenting blocks that is now bound to Ctrl+] and ctr+[ * Exception types can now be displayed with a custom traceback, by defining a ``_render_traceback_()`` method which returns a list of strings, each containing one line of the traceback. + +Backwards incompatible changes +------------------------------ + +* Calling :meth:`InteractiveShell.prefilter` will no longer perform static + transformations - the processing of escaped commands such as ``%magic`` and + ``!system``, and stripping input prompts from code blocks. This functionality + was duplicated in :mod:`IPython.core.inputsplitter`, and the latter version + was already what IPython relied on. A new API to transform input will be ready + before release.