##// END OF EJS Templates
Moving main display notebook.
Moving main display notebook.

File last commit:

r17037:45ff0820
r17504:b8e80c94
Show More
test_inputtransformer.py
492 lines | 15.9 KiB | text/x-python | PythonLexer
/ IPython / core / tests / test_inputtransformer.py
Thomas Kluyver
Add TokenInputTransformer
r10103 import tokenize
Thomas Kluyver
First go at new input transformation system
r10090 import nose.tools as nt
from IPython.testing import tools as tt
from IPython.utils import py3compat
Thomas Kluyver
Move and combine transformer tests
r10095 u_fmt = py3compat.u_format
Thomas Kluyver
First go at new input transformation system
r10090
Thomas Kluyver
Move and combine transformer tests
r10095 from IPython.core import inputtransformer as ipt
Thomas Kluyver
First go at new input transformation system
r10090
Thomas Kluyver
Move and combine transformer tests
r10095 def transform_and_reset(transformer):
Thomas Kluyver
Fix tests in IPython.core
r10097 transformer = transformer()
Thomas Kluyver
First go at new input transformation system
r10090 def transform(inp):
Thomas Kluyver
Move and combine transformer tests
r10095 try:
return transformer.push(inp)
finally:
transformer.reset()
Thomas Kluyver
First go at new input transformation system
r10090
return transform
Thomas Kluyver
Move and combine transformer tests
r10095 # Transformer tests
Thomas Kluyver
Allow IPythonInputSplitter to accept cell magics containing blank lines
r10252 def transform_checker(tests, transformer, **kwargs):
Thomas Kluyver
Move and combine transformer tests
r10095 """Utility to loop over test inputs"""
Thomas Kluyver
Allow IPythonInputSplitter to accept cell magics containing blank lines
r10252 transformer = transformer(**kwargs)
Thomas Kluyver
Move and combine transformer tests
r10095 try:
for inp, tr in tests:
Thomas Kluyver
Correction to test framework.
r10098 if inp is None:
out = transformer.reset()
else:
out = transformer.push(inp)
nt.assert_equal(out, tr)
Thomas Kluyver
Move and combine transformer tests
r10095 finally:
transformer.reset()
# Data for all the syntax tests in the form of lists of pairs of
# raw/transformed input. We store it here as a global dict so that we can use
# it both within single-function tests and also to validate the behavior of the
# larger objects
syntax = \
dict(assign_system =
[(i,py3compat.u_format(o)) for i,o in \
[(u'a =! ls', "a = get_ipython().getoutput({u}'ls')"),
(u'b = !ls', "b = get_ipython().getoutput({u}'ls')"),
Thomas Kluyver
Add some tests for assignment from system/magic commands
r15789 (u'c= !ls', "c = get_ipython().getoutput({u}'ls')"),
(u'd == !ls', u'd == !ls'), # Invalid syntax, but we leave == alone.
Thomas Kluyver
Move and combine transformer tests
r10095 ('x=1', 'x=1'), # normal input is unmodified
(' ',' '), # blank lines are kept intact
Thomas Kluyver
Add failing tests for tuple unpacking assignments from system cmds
r15785 # Tuple unpacking
(u"a, b = !echo 'a\\nb'", u"a, b = get_ipython().getoutput({u}\"echo 'a\\\\nb'\")"),
(u"a,= !echo 'a'", u"a, = get_ipython().getoutput({u}\"echo 'a'\")"),
(u"a, *bc = !echo 'a\\nb\\nc'", u"a, *bc = get_ipython().getoutput({u}\"echo 'a\\\\nb\\\\nc'\")"),
Thomas Kluyver
Add some tests for assignment from system/magic commands
r15789 # Tuple unpacking with regular Python expressions, not our syntax.
(u"a, b = range(2)", u"a, b = range(2)"),
(u"a, = range(1)", u"a, = range(1)"),
(u"a, *bc = range(3)", u"a, *bc = range(3)"),
Thomas Kluyver
Move and combine transformer tests
r10095 ]],
assign_magic =
[(i,py3compat.u_format(o)) for i,o in \
[(u'a =% who', "a = get_ipython().magic({u}'who')"),
(u'b = %who', "b = get_ipython().magic({u}'who')"),
Thomas Kluyver
Add some tests for assignment from system/magic commands
r15789 (u'c= %ls', "c = get_ipython().magic({u}'ls')"),
(u'd == %ls', u'd == %ls'), # Invalid syntax, but we leave == alone.
Thomas Kluyver
Move and combine transformer tests
r10095 ('x=1', 'x=1'), # normal input is unmodified
(' ',' '), # blank lines are kept intact
Thomas Kluyver
Add some tests for assignment from system/magic commands
r15789 (u"a, b = %foo", u"a, b = get_ipython().magic({u}'foo')"),
Thomas Kluyver
Move and combine transformer tests
r10095 ]],
classic_prompt =
[('>>> x=1', 'x=1'),
('x=1', 'x=1'), # normal input is unmodified
(' ', ' '), # blank lines are kept intact
],
ipy_prompt =
[('In [1]: x=1', 'x=1'),
('x=1', 'x=1'), # normal input is unmodified
(' ',' '), # blank lines are kept intact
],
MinRK
test strip_encoding_cookie transformer
r11443 strip_encoding_cookie =
[
('# -*- encoding: utf-8 -*-', ''),
('# coding: latin-1', ''),
],
Thomas Kluyver
Move and combine transformer tests
r10095 # Tests for the escape transformer to leave normal code alone
escaped_noesc =
[ (' ', ' '),
('x=1', 'x=1'),
],
# System calls
escaped_shell =
[(i,py3compat.u_format(o)) for i,o in \
[ (u'!ls', "get_ipython().system({u}'ls')"),
# Double-escape shell, this means to capture the output of the
# subprocess and return it
(u'!!ls', "get_ipython().getoutput({u}'ls')"),
]],
# Help/object info
escaped_help =
[(i,py3compat.u_format(o)) for i,o in \
[ (u'?', 'get_ipython().show_usage()'),
(u'?x1', "get_ipython().magic({u}'pinfo x1')"),
(u'??x2', "get_ipython().magic({u}'pinfo2 x2')"),
(u'?a.*s', "get_ipython().magic({u}'psearch a.*s')"),
(u'?%hist1', "get_ipython().magic({u}'pinfo %hist1')"),
(u'?%%hist2', "get_ipython().magic({u}'pinfo %%hist2')"),
(u'?abc = qwe', "get_ipython().magic({u}'pinfo abc')"),
]],
end_help =
[(i,py3compat.u_format(o)) for i,o in \
[ (u'x3?', "get_ipython().magic({u}'pinfo x3')"),
(u'x4??', "get_ipython().magic({u}'pinfo2 x4')"),
(u'%hist1?', "get_ipython().magic({u}'pinfo %hist1')"),
(u'%hist2??', "get_ipython().magic({u}'pinfo2 %hist2')"),
(u'%%hist3?', "get_ipython().magic({u}'pinfo %%hist3')"),
(u'%%hist4??', "get_ipython().magic({u}'pinfo2 %%hist4')"),
(u'f*?', "get_ipython().magic({u}'psearch f*')"),
(u'ax.*aspe*?', "get_ipython().magic({u}'psearch ax.*aspe*')"),
(u'a = abc?', "get_ipython().set_next_input({u}'a = abc');"
"get_ipython().magic({u}'pinfo abc')"),
(u'a = abc.qe??', "get_ipython().set_next_input({u}'a = abc.qe');"
"get_ipython().magic({u}'pinfo2 abc.qe')"),
(u'a = *.items?', "get_ipython().set_next_input({u}'a = *.items');"
"get_ipython().magic({u}'psearch *.items')"),
(u'plot(a?', "get_ipython().set_next_input({u}'plot(a');"
"get_ipython().magic({u}'pinfo a')"),
(u'a*2 #comment?', 'a*2 #comment?'),
]],
# Explicit magic calls
escaped_magic =
[(i,py3compat.u_format(o)) for i,o in \
[ (u'%cd', "get_ipython().magic({u}'cd')"),
(u'%cd /home', "get_ipython().magic({u}'cd /home')"),
# Backslashes need to be escaped.
(u'%cd C:\\User', "get_ipython().magic({u}'cd C:\\\\User')"),
(u' %magic', " get_ipython().magic({u}'magic')"),
]],
# Quoting with separate arguments
escaped_quote =
[ (',f', 'f("")'),
(',f x', 'f("x")'),
(' ,f y', ' f("y")'),
(',f a b', 'f("a", "b")'),
],
# Quoting with single argument
escaped_quote2 =
[ (';f', 'f("")'),
(';f x', 'f("x")'),
(' ;f y', ' f("y")'),
(';f a b', 'f("a b")'),
],
# Simply apply parens
escaped_paren =
[ ('/f', 'f()'),
('/f x', 'f(x)'),
(' /f y', ' f(y)'),
('/f a b', 'f(a, b)'),
],
# Check that we transform prompts before other transforms
mixed =
[(i,py3compat.u_format(o)) for i,o in \
[ (u'In [1]: %lsmagic', "get_ipython().magic({u}'lsmagic')"),
(u'>>> %lsmagic', "get_ipython().magic({u}'lsmagic')"),
(u'In [2]: !ls', "get_ipython().system({u}'ls')"),
(u'In [3]: abs?', "get_ipython().magic({u}'pinfo abs')"),
(u'In [4]: b = %who', "b = get_ipython().magic({u}'who')"),
]],
)
Thomas Kluyver
First go at new input transformation system
r10090
Thomas Kluyver
Move and combine transformer tests
r10095 # multiline syntax examples. Each of these should be a list of lists, with
# each entry itself having pairs of raw/transformed input. The union (with
# '\n'.join() of the transformed inputs is what the splitter should produce
# when fed the raw lines one at a time via push.
syntax_ml = \
dict(classic_prompt =
[ [('>>> for i in range(10):','for i in range(10):'),
('... print i',' print i'),
('... ', ''),
],
[('>>> a="""','a="""'),
('... 123"""','123"""'),
],
[('a="""','a="""'),
MinRK
adjust strip_prompts logic a bit further...
r12309 ('... 123','123'),
('... 456"""','456"""'),
],
[('a="""','a="""'),
('>>> 123','123'),
('... 456"""','456"""'),
Thomas Kluyver
Strip prompts even if the prompt isn't present on the first line....
r10652 ],
[('a="""','a="""'),
('123','123'),
('... 456"""','... 456"""'),
Thomas Kluyver
Move and combine transformer tests
r10095 ],
MinRK
only strip continuation prompts if regular prompts seen first...
r12231 [('....__class__','....__class__'),
],
[('a=5', 'a=5'),
MinRK
adjust strip_prompts logic a bit further...
r12309 ('...', ''),
MinRK
only strip continuation prompts if regular prompts seen first...
r12231 ],
Thomas Kluyver
Allow classic prompts without space....
r11056 [('>>> def f(x):', 'def f(x):'),
('...', ''),
('... return x', ' return x'),
],
Thomas Kluyver
Add failing test for gh-6025
r17037 [('board = """....', 'board = """....'),
('....', '....'),
('...."""', '...."""'),
],
Thomas Kluyver
Move and combine transformer tests
r10095 ],
Thomas Kluyver
First go at new input transformation system
r10090
Thomas Kluyver
Move and combine transformer tests
r10095 ipy_prompt =
[ [('In [24]: for i in range(10):','for i in range(10):'),
(' ....: print i',' print i'),
(' ....: ', ''),
],
Thomas Kluyver
Add failing test for issue gh-4728
r13934 [('In [24]: for i in range(10):','for i in range(10):'),
# Qt console prompts expand with spaces, not dots
(' ...: print i',' print i'),
(' ...: ', ''),
],
Thomas Kluyver
Move and combine transformer tests
r10095 [('In [2]: a="""','a="""'),
(' ...: 123"""','123"""'),
],
[('a="""','a="""'),
MinRK
adjust strip_prompts logic a bit further...
r12309 (' ...: 123','123'),
(' ...: 456"""','456"""'),
],
[('a="""','a="""'),
('In [1]: 123','123'),
(' ...: 456"""','456"""'),
Thomas Kluyver
Strip prompts even if the prompt isn't present on the first line....
r10652 ],
[('a="""','a="""'),
('123','123'),
(' ...: 456"""',' ...: 456"""'),
Thomas Kluyver
Move and combine transformer tests
r10095 ],
],
Thomas Kluyver
First go at new input transformation system
r10090
MinRK
test strip_encoding_cookie transformer
r11443 strip_encoding_cookie =
[
[
('# -*- coding: utf-8 -*-', ''),
('foo', 'foo'),
],
[
('#!/usr/bin/env python', '#!/usr/bin/env python'),
('# -*- coding: latin-1 -*-', ''),
# only the first-two lines
('# -*- coding: latin-1 -*-', '# -*- coding: latin-1 -*-'),
],
],
Thomas Kluyver
Fix for \ at end of comment, and add tests
r10112 multiline_datastructure_prompt =
Thomas Kluyver
Move and combine transformer tests
r10095 [ [('>>> a = [1,','a = [1,'),
('... 2]','2]'),
],
],
Thomas Kluyver
Fix for \ at end of comment, and add tests
r10112
multiline_datastructure =
[ [('b = ("%s"', None),
('# comment', None),
('%foo )', 'b = ("%s"\n# comment\n%foo )'),
],
],
Thomas Kluyver
Move and combine transformer tests
r10095
Thomas Kluyver
help_end transformer shouldn't pick up ? in multiline string...
r12352 multiline_string =
[ [("'''foo?", None),
("bar'''", "'''foo?\nbar'''"),
],
],
Thomas Kluyver
Move and combine transformer tests
r10095 leading_indent =
[ [(' print "hi"','print "hi"'),
Thomas Kluyver
Fix tests in IPython.core
r10097 ],
[(' for a in range(5):','for a in range(5):'),
Thomas Kluyver
Move and combine transformer tests
r10095 (' a*2',' a*2'),
],
[(' a="""','a="""'),
(' 123"""','123"""'),
],
[('a="""','a="""'),
(' 123"""',' 123"""'),
],
],
cellmagic =
Thomas Kluyver
Fix tests in IPython.core
r10097 [ [(u'%%foo a', None),
(None, u_fmt("get_ipython().run_cell_magic({u}'foo', {u}'a', {u}'')")),
Thomas Kluyver
Move and combine transformer tests
r10095 ],
Thomas Kluyver
Fix tests in IPython.core
r10097 [(u'%%bar 123', None),
(u'hello', None),
Thomas Kluyver
Allow IPythonInputSplitter to accept cell magics containing blank lines
r10252 (None , u_fmt("get_ipython().run_cell_magic({u}'bar', {u}'123', {u}'hello')")),
Thomas Kluyver
Move and combine transformer tests
r10095 ],
MinRK
test that cell-magics are not transformed mid-cell
r11473 [(u'a=5', 'a=5'),
(u'%%cellmagic', '%%cellmagic'),
],
Thomas Kluyver
Move and combine transformer tests
r10095 ],
escaped =
[ [('%abc def \\', None),
Thomas Kluyver
Fix for \ at end of comment, and add tests
r10112 ('ghi', u_fmt("get_ipython().magic({u}'abc def ghi')")),
Thomas Kluyver
Move and combine transformer tests
r10095 ],
[('%abc def \\', None),
('ghi\\', None),
Thomas Kluyver
Fix for \ at end of comment, and add tests
r10112 (None, u_fmt("get_ipython().magic({u}'abc def ghi')")),
Thomas Kluyver
Move and combine transformer tests
r10095 ],
],
assign_magic =
Thomas Kluyver
Fix tests in IPython.core
r10097 [ [(u'a = %bc de \\', None),
Thomas Kluyver
Fix for \ at end of comment, and add tests
r10112 (u'fg', u_fmt("a = get_ipython().magic({u}'bc de fg')")),
Thomas Kluyver
Move and combine transformer tests
r10095 ],
Thomas Kluyver
Fix tests in IPython.core
r10097 [(u'a = %bc de \\', None),
(u'fg\\', None),
Thomas Kluyver
Fix for \ at end of comment, and add tests
r10112 (None, u_fmt("a = get_ipython().magic({u}'bc de fg')")),
Thomas Kluyver
Move and combine transformer tests
r10095 ],
],
assign_system =
Thomas Kluyver
Fix tests in IPython.core
r10097 [ [(u'a = !bc de \\', None),
Thomas Kluyver
Fix for \ at end of comment, and add tests
r10112 (u'fg', u_fmt("a = get_ipython().getoutput({u}'bc de fg')")),
Thomas Kluyver
Move and combine transformer tests
r10095 ],
Thomas Kluyver
Fix tests in IPython.core
r10097 [(u'a = !bc de \\', None),
(u'fg\\', None),
Thomas Kluyver
Fix for \ at end of comment, and add tests
r10112 (None, u_fmt("a = get_ipython().getoutput({u}'bc de fg')")),
Thomas Kluyver
Move and combine transformer tests
r10095 ],
],
)
Thomas Kluyver
First go at new input transformation system
r10090
Thomas Kluyver
More input transformers
r10091
Thomas Kluyver
Move and combine transformer tests
r10095 def test_assign_system():
tt.check_pairs(transform_and_reset(ipt.assign_from_system), syntax['assign_system'])
def test_assign_magic():
tt.check_pairs(transform_and_reset(ipt.assign_from_magic), syntax['assign_magic'])
Thomas Kluyver
More input transformers
r10091
def test_classic_prompt():
Thomas Kluyver
Move and combine transformer tests
r10095 tt.check_pairs(transform_and_reset(ipt.classic_prompt), syntax['classic_prompt'])
for example in syntax_ml['classic_prompt']:
transform_checker(example, ipt.classic_prompt)
Thomas Kluyver
Fix for \ at end of comment, and add tests
r10112 for example in syntax_ml['multiline_datastructure_prompt']:
Thomas Kluyver
Fix tests in IPython.core
r10097 transform_checker(example, ipt.classic_prompt)
Thomas Kluyver
More input transformers
r10091
def test_ipy_prompt():
Thomas Kluyver
Move and combine transformer tests
r10095 tt.check_pairs(transform_and_reset(ipt.ipy_prompt), syntax['ipy_prompt'])
for example in syntax_ml['ipy_prompt']:
transform_checker(example, ipt.ipy_prompt)
Thomas Kluyver
More input transformers
r10091
MinRK
test strip_encoding_cookie transformer
r11443 def test_coding_cookie():
tt.check_pairs(transform_and_reset(ipt.strip_encoding_cookie), syntax['strip_encoding_cookie'])
for example in syntax_ml['strip_encoding_cookie']:
transform_checker(example, ipt.strip_encoding_cookie)
Thomas Kluyver
Simplify input transformers...
r10107 def test_assemble_logical_lines():
tests = \
[ [(u"a = \\", None),
Thomas Kluyver
Fix for \ at end of comment, and add tests
r10112 (u"123", u"a = 123"),
Thomas Kluyver
Simplify input transformers...
r10107 ],
[(u"a = \\", None), # Test resetting when within a multi-line string
(u"12 *\\", None),
Thomas Kluyver
Fix for \ at end of comment, and add tests
r10112 (None, u"a = 12 *"),
],
[(u"# foo\\", u"# foo\\"), # Comments can't be continued like this
Thomas Kluyver
Simplify input transformers...
r10107 ],
]
for example in tests:
transform_checker(example, ipt.assemble_logical_lines)
def test_assemble_python_lines():
tests = \
[ [(u"a = '''", None),
(u"abc'''", u"a = '''\nabc'''"),
],
[(u"a = '''", None), # Test resetting when within a multi-line string
(u"def", None),
(None, u"a = '''\ndef"),
],
[(u"a = [1,", None),
(u"2]", u"a = [1,\n2]"),
],
[(u"a = [1,", None), # Test resetting when within a multi-line string
(u"2,", None),
(None, u"a = [1,\n2,"),
],
Thomas Kluyver
Fix for \ at end of comment, and add tests
r10112 ] + syntax_ml['multiline_datastructure']
Thomas Kluyver
Simplify input transformers...
r10107 for example in tests:
transform_checker(example, ipt.assemble_python_lines)
Thomas Kluyver
Move and combine transformer tests
r10095 def test_help_end():
tt.check_pairs(transform_and_reset(ipt.help_end), syntax['end_help'])
Thomas Kluyver
More input transformers
r10091
Thomas Kluyver
Move and combine transformer tests
r10095 def test_escaped_noesc():
Thomas Kluyver
Simplify input transformers...
r10107 tt.check_pairs(transform_and_reset(ipt.escaped_commands), syntax['escaped_noesc'])
Thomas Kluyver
Transformers for assignment from %magic and \!system calls
r10092
Thomas Kluyver
Move and combine transformer tests
r10095 def test_escaped_shell():
Thomas Kluyver
Simplify input transformers...
r10107 tt.check_pairs(transform_and_reset(ipt.escaped_commands), syntax['escaped_shell'])
Thomas Kluyver
Transformers for assignment from %magic and \!system calls
r10092
Thomas Kluyver
Move and combine transformer tests
r10095 def test_escaped_help():
Thomas Kluyver
Simplify input transformers...
r10107 tt.check_pairs(transform_and_reset(ipt.escaped_commands), syntax['escaped_help'])
Thomas Kluyver
Move and combine transformer tests
r10095
def test_escaped_magic():
Thomas Kluyver
Simplify input transformers...
r10107 tt.check_pairs(transform_and_reset(ipt.escaped_commands), syntax['escaped_magic'])
Thomas Kluyver
Move and combine transformer tests
r10095
def test_escaped_quote():
Thomas Kluyver
Simplify input transformers...
r10107 tt.check_pairs(transform_and_reset(ipt.escaped_commands), syntax['escaped_quote'])
Thomas Kluyver
Move and combine transformer tests
r10095
def test_escaped_quote2():
Thomas Kluyver
Simplify input transformers...
r10107 tt.check_pairs(transform_and_reset(ipt.escaped_commands), syntax['escaped_quote2'])
Thomas Kluyver
Move and combine transformer tests
r10095
def test_escaped_paren():
Thomas Kluyver
Simplify input transformers...
r10107 tt.check_pairs(transform_and_reset(ipt.escaped_commands), syntax['escaped_paren'])
Thomas Kluyver
Move and combine transformer tests
r10095
def test_cellmagic():
for example in syntax_ml['cellmagic']:
transform_checker(example, ipt.cellmagic)
Thomas Kluyver
Allow IPythonInputSplitter to accept cell magics containing blank lines
r10252
line_example = [(u'%%bar 123', None),
(u'hello', None),
(u'' , u_fmt("get_ipython().run_cell_magic({u}'bar', {u}'123', {u}'hello')")),
]
transform_checker(line_example, ipt.cellmagic, end_on_blank_line=True)
Thomas Kluyver
Fix tests in IPython.core
r10097
def test_has_comment():
tests = [('text', False),
('text #comment', True),
('text #comment\n', True),
('#comment', True),
('#comment\n', True),
('a = "#string"', False),
('a = "#string" # comment', True),
('a #comment not "string"', True),
]
tt.check_pairs(ipt.has_comment, tests)
Thomas Kluyver
Add TokenInputTransformer
r10103
@ipt.TokenInputTransformer.wrap
def decistmt(tokens):
"""Substitute Decimals for floats in a string of statements.
Based on an example from the tokenize module docs.
"""
result = []
for toknum, tokval, _, _, _ in tokens:
if toknum == tokenize.NUMBER and '.' in tokval: # replace NUMBER tokens
for newtok in [
(tokenize.NAME, 'Decimal'),
(tokenize.OP, '('),
(tokenize.STRING, repr(tokval)),
(tokenize.OP, ')')
]:
yield newtok
else:
yield (toknum, tokval)
def test_token_input_transformer():
tests = [(u'1.2', u_fmt(u"Decimal ({u}'1.2')")),
(u'"1.2"', u'"1.2"'),
]
tt.check_pairs(transform_and_reset(decistmt), tests)
ml_tests = \
[ [(u"a = 1.2; b = '''x", None),
(u"y'''", u_fmt(u"a =Decimal ({u}'1.2');b ='''x\ny'''")),
],
Thomas Kluyver
Revised input transformation framework.
r10106 [(u"a = [1.2,", None),
(u"3]", u_fmt(u"a =[Decimal ({u}'1.2'),\n3 ]")),
Thomas Kluyver
Add TokenInputTransformer
r10103 ],
[(u"a = '''foo", None), # Test resetting when within a multi-line string
(u"bar", None),
(None, u"a = '''foo\nbar"),
],
]
for example in ml_tests:
transform_checker(example, decistmt)