test_inputtransformer.py
470 lines
| 14.7 KiB
| text/x-python
|
PythonLexer
|
r10103 | import tokenize | ||
|
r10090 | import nose.tools as nt | ||
from IPython.testing import tools as tt | ||||
from IPython.utils import py3compat | ||||
|
r10095 | u_fmt = py3compat.u_format | ||
|
r10090 | |||
|
r10095 | from IPython.core import inputtransformer as ipt | ||
|
r10090 | |||
|
r10095 | def transform_and_reset(transformer): | ||
|
r10097 | transformer = transformer() | ||
|
r10090 | def transform(inp): | ||
|
r10095 | try: | ||
return transformer.push(inp) | ||||
finally: | ||||
transformer.reset() | ||||
|
r10090 | |||
return transform | ||||
|
r10095 | # Transformer tests | ||
|
r10252 | def transform_checker(tests, transformer, **kwargs): | ||
|
r10095 | """Utility to loop over test inputs""" | ||
|
r10252 | transformer = transformer(**kwargs) | ||
|
r10095 | try: | ||
for inp, tr in tests: | ||||
|
r10098 | if inp is None: | ||
out = transformer.reset() | ||||
else: | ||||
out = transformer.push(inp) | ||||
nt.assert_equal(out, tr) | ||||
|
r10095 | finally: | ||
transformer.reset() | ||||
# Data for all the syntax tests in the form of lists of pairs of | ||||
# raw/transformed input. We store it here as a global dict so that we can use | ||||
# it both within single-function tests and also to validate the behavior of the | ||||
# larger objects | ||||
syntax = \ | ||||
dict(assign_system = | ||||
[(i,py3compat.u_format(o)) for i,o in \ | ||||
[(u'a =! ls', "a = get_ipython().getoutput({u}'ls')"), | ||||
(u'b = !ls', "b = get_ipython().getoutput({u}'ls')"), | ||||
('x=1', 'x=1'), # normal input is unmodified | ||||
(' ',' '), # blank lines are kept intact | ||||
]], | ||||
assign_magic = | ||||
[(i,py3compat.u_format(o)) for i,o in \ | ||||
[(u'a =% who', "a = get_ipython().magic({u}'who')"), | ||||
(u'b = %who', "b = get_ipython().magic({u}'who')"), | ||||
('x=1', 'x=1'), # normal input is unmodified | ||||
(' ',' '), # blank lines are kept intact | ||||
]], | ||||
classic_prompt = | ||||
[('>>> x=1', 'x=1'), | ||||
('x=1', 'x=1'), # normal input is unmodified | ||||
(' ', ' '), # blank lines are kept intact | ||||
], | ||||
ipy_prompt = | ||||
[('In [1]: x=1', 'x=1'), | ||||
('x=1', 'x=1'), # normal input is unmodified | ||||
(' ',' '), # blank lines are kept intact | ||||
], | ||||
|
r11443 | strip_encoding_cookie = | ||
[ | ||||
('# -*- encoding: utf-8 -*-', ''), | ||||
('# coding: latin-1', ''), | ||||
], | ||||
|
r10095 | # Tests for the escape transformer to leave normal code alone | ||
escaped_noesc = | ||||
[ (' ', ' '), | ||||
('x=1', 'x=1'), | ||||
], | ||||
# System calls | ||||
escaped_shell = | ||||
[(i,py3compat.u_format(o)) for i,o in \ | ||||
[ (u'!ls', "get_ipython().system({u}'ls')"), | ||||
# Double-escape shell, this means to capture the output of the | ||||
# subprocess and return it | ||||
(u'!!ls', "get_ipython().getoutput({u}'ls')"), | ||||
]], | ||||
# Help/object info | ||||
escaped_help = | ||||
[(i,py3compat.u_format(o)) for i,o in \ | ||||
[ (u'?', 'get_ipython().show_usage()'), | ||||
(u'?x1', "get_ipython().magic({u}'pinfo x1')"), | ||||
(u'??x2', "get_ipython().magic({u}'pinfo2 x2')"), | ||||
(u'?a.*s', "get_ipython().magic({u}'psearch a.*s')"), | ||||
(u'?%hist1', "get_ipython().magic({u}'pinfo %hist1')"), | ||||
(u'?%%hist2', "get_ipython().magic({u}'pinfo %%hist2')"), | ||||
(u'?abc = qwe', "get_ipython().magic({u}'pinfo abc')"), | ||||
]], | ||||
end_help = | ||||
[(i,py3compat.u_format(o)) for i,o in \ | ||||
[ (u'x3?', "get_ipython().magic({u}'pinfo x3')"), | ||||
(u'x4??', "get_ipython().magic({u}'pinfo2 x4')"), | ||||
(u'%hist1?', "get_ipython().magic({u}'pinfo %hist1')"), | ||||
(u'%hist2??', "get_ipython().magic({u}'pinfo2 %hist2')"), | ||||
(u'%%hist3?', "get_ipython().magic({u}'pinfo %%hist3')"), | ||||
(u'%%hist4??', "get_ipython().magic({u}'pinfo2 %%hist4')"), | ||||
(u'f*?', "get_ipython().magic({u}'psearch f*')"), | ||||
(u'ax.*aspe*?', "get_ipython().magic({u}'psearch ax.*aspe*')"), | ||||
(u'a = abc?', "get_ipython().set_next_input({u}'a = abc');" | ||||
"get_ipython().magic({u}'pinfo abc')"), | ||||
(u'a = abc.qe??', "get_ipython().set_next_input({u}'a = abc.qe');" | ||||
"get_ipython().magic({u}'pinfo2 abc.qe')"), | ||||
(u'a = *.items?', "get_ipython().set_next_input({u}'a = *.items');" | ||||
"get_ipython().magic({u}'psearch *.items')"), | ||||
(u'plot(a?', "get_ipython().set_next_input({u}'plot(a');" | ||||
"get_ipython().magic({u}'pinfo a')"), | ||||
(u'a*2 #comment?', 'a*2 #comment?'), | ||||
]], | ||||
# Explicit magic calls | ||||
escaped_magic = | ||||
[(i,py3compat.u_format(o)) for i,o in \ | ||||
[ (u'%cd', "get_ipython().magic({u}'cd')"), | ||||
(u'%cd /home', "get_ipython().magic({u}'cd /home')"), | ||||
# Backslashes need to be escaped. | ||||
(u'%cd C:\\User', "get_ipython().magic({u}'cd C:\\\\User')"), | ||||
(u' %magic', " get_ipython().magic({u}'magic')"), | ||||
]], | ||||
# Quoting with separate arguments | ||||
escaped_quote = | ||||
[ (',f', 'f("")'), | ||||
(',f x', 'f("x")'), | ||||
(' ,f y', ' f("y")'), | ||||
(',f a b', 'f("a", "b")'), | ||||
], | ||||
# Quoting with single argument | ||||
escaped_quote2 = | ||||
[ (';f', 'f("")'), | ||||
(';f x', 'f("x")'), | ||||
(' ;f y', ' f("y")'), | ||||
(';f a b', 'f("a b")'), | ||||
], | ||||
# Simply apply parens | ||||
escaped_paren = | ||||
[ ('/f', 'f()'), | ||||
('/f x', 'f(x)'), | ||||
(' /f y', ' f(y)'), | ||||
('/f a b', 'f(a, b)'), | ||||
], | ||||
# Check that we transform prompts before other transforms | ||||
mixed = | ||||
[(i,py3compat.u_format(o)) for i,o in \ | ||||
[ (u'In [1]: %lsmagic', "get_ipython().magic({u}'lsmagic')"), | ||||
(u'>>> %lsmagic', "get_ipython().magic({u}'lsmagic')"), | ||||
(u'In [2]: !ls', "get_ipython().system({u}'ls')"), | ||||
(u'In [3]: abs?', "get_ipython().magic({u}'pinfo abs')"), | ||||
(u'In [4]: b = %who', "b = get_ipython().magic({u}'who')"), | ||||
]], | ||||
) | ||||
|
r10090 | |||
|
r10095 | # multiline syntax examples. Each of these should be a list of lists, with | ||
# each entry itself having pairs of raw/transformed input. The union (with | ||||
# '\n'.join() of the transformed inputs is what the splitter should produce | ||||
# when fed the raw lines one at a time via push. | ||||
syntax_ml = \ | ||||
dict(classic_prompt = | ||||
[ [('>>> for i in range(10):','for i in range(10):'), | ||||
('... print i',' print i'), | ||||
('... ', ''), | ||||
], | ||||
[('>>> a="""','a="""'), | ||||
('... 123"""','123"""'), | ||||
], | ||||
[('a="""','a="""'), | ||||
|
r10652 | ('... 123','123'), | ||
('... 456"""','456"""'), | ||||
], | ||||
[('a="""','a="""'), | ||||
|
r12330 | ('>>> 123','123'), | ||
('... 456"""','456"""'), | ||||
], | ||||
[('a="""','a="""'), | ||||
|
r10652 | ('123','123'), | ||
('... 456"""','... 456"""'), | ||||
|
r10095 | ], | ||
|
r12330 | [('....__class__','....__class__'), | ||
], | ||||
[('a=5', 'a=5'), | ||||
('...', ''), | ||||
], | ||||
|
r11056 | [('>>> def f(x):', 'def f(x):'), | ||
('...', ''), | ||||
('... return x', ' return x'), | ||||
], | ||||
|
r10095 | ], | ||
|
r10090 | |||
|
r10095 | ipy_prompt = | ||
[ [('In [24]: for i in range(10):','for i in range(10):'), | ||||
(' ....: print i',' print i'), | ||||
(' ....: ', ''), | ||||
], | ||||
[('In [2]: a="""','a="""'), | ||||
(' ...: 123"""','123"""'), | ||||
], | ||||
[('a="""','a="""'), | ||||
|
r10652 | (' ...: 123','123'), | ||
(' ...: 456"""','456"""'), | ||||
], | ||||
[('a="""','a="""'), | ||||
|
r12330 | ('In [1]: 123','123'), | ||
(' ...: 456"""','456"""'), | ||||
], | ||||
[('a="""','a="""'), | ||||
|
r10652 | ('123','123'), | ||
(' ...: 456"""',' ...: 456"""'), | ||||
|
r10095 | ], | ||
], | ||||
|
r10090 | |||
|
r11443 | strip_encoding_cookie = | ||
[ | ||||
[ | ||||
('# -*- coding: utf-8 -*-', ''), | ||||
('foo', 'foo'), | ||||
], | ||||
[ | ||||
('#!/usr/bin/env python', '#!/usr/bin/env python'), | ||||
('# -*- coding: latin-1 -*-', ''), | ||||
# only the first-two lines | ||||
('# -*- coding: latin-1 -*-', '# -*- coding: latin-1 -*-'), | ||||
], | ||||
], | ||||
|
r10112 | multiline_datastructure_prompt = | ||
|
r10095 | [ [('>>> a = [1,','a = [1,'), | ||
('... 2]','2]'), | ||||
], | ||||
], | ||||
|
r10112 | |||
multiline_datastructure = | ||||
[ [('b = ("%s"', None), | ||||
('# comment', None), | ||||
('%foo )', 'b = ("%s"\n# comment\n%foo )'), | ||||
], | ||||
], | ||||
|
r10095 | |||
|
r12430 | multiline_string = | ||
[ [("'''foo?", None), | ||||
("bar'''", "'''foo?\nbar'''"), | ||||
], | ||||
], | ||||
|
r10095 | leading_indent = | ||
[ [(' print "hi"','print "hi"'), | ||||
|
r10097 | ], | ||
[(' for a in range(5):','for a in range(5):'), | ||||
|
r10095 | (' a*2',' a*2'), | ||
], | ||||
[(' a="""','a="""'), | ||||
(' 123"""','123"""'), | ||||
], | ||||
[('a="""','a="""'), | ||||
(' 123"""',' 123"""'), | ||||
], | ||||
], | ||||
cellmagic = | ||||
|
r10097 | [ [(u'%%foo a', None), | ||
(None, u_fmt("get_ipython().run_cell_magic({u}'foo', {u}'a', {u}'')")), | ||||
|
r10095 | ], | ||
|
r10097 | [(u'%%bar 123', None), | ||
(u'hello', None), | ||||
|
r10252 | (None , u_fmt("get_ipython().run_cell_magic({u}'bar', {u}'123', {u}'hello')")), | ||
|
r10095 | ], | ||
|
r11473 | [(u'a=5', 'a=5'), | ||
(u'%%cellmagic', '%%cellmagic'), | ||||
], | ||||
|
r10095 | ], | ||
escaped = | ||||
[ [('%abc def \\', None), | ||||
|
r10112 | ('ghi', u_fmt("get_ipython().magic({u}'abc def ghi')")), | ||
|
r10095 | ], | ||
[('%abc def \\', None), | ||||
('ghi\\', None), | ||||
|
r10112 | (None, u_fmt("get_ipython().magic({u}'abc def ghi')")), | ||
|
r10095 | ], | ||
], | ||||
assign_magic = | ||||
|
r10097 | [ [(u'a = %bc de \\', None), | ||
|
r10112 | (u'fg', u_fmt("a = get_ipython().magic({u}'bc de fg')")), | ||
|
r10095 | ], | ||
|
r10097 | [(u'a = %bc de \\', None), | ||
(u'fg\\', None), | ||||
|
r10112 | (None, u_fmt("a = get_ipython().magic({u}'bc de fg')")), | ||
|
r10095 | ], | ||
], | ||||
assign_system = | ||||
|
r10097 | [ [(u'a = !bc de \\', None), | ||
|
r10112 | (u'fg', u_fmt("a = get_ipython().getoutput({u}'bc de fg')")), | ||
|
r10095 | ], | ||
|
r10097 | [(u'a = !bc de \\', None), | ||
(u'fg\\', None), | ||||
|
r10112 | (None, u_fmt("a = get_ipython().getoutput({u}'bc de fg')")), | ||
|
r10095 | ], | ||
], | ||||
) | ||||
|
r10090 | |||
|
r10091 | |||
|
r10095 | def test_assign_system(): | ||
tt.check_pairs(transform_and_reset(ipt.assign_from_system), syntax['assign_system']) | ||||
def test_assign_magic(): | ||||
tt.check_pairs(transform_and_reset(ipt.assign_from_magic), syntax['assign_magic']) | ||||
|
r10091 | |||
def test_classic_prompt(): | ||||
|
r10095 | tt.check_pairs(transform_and_reset(ipt.classic_prompt), syntax['classic_prompt']) | ||
for example in syntax_ml['classic_prompt']: | ||||
transform_checker(example, ipt.classic_prompt) | ||||
|
r10112 | for example in syntax_ml['multiline_datastructure_prompt']: | ||
|
r10097 | transform_checker(example, ipt.classic_prompt) | ||
|
r10091 | |||
def test_ipy_prompt(): | ||||
|
r10095 | tt.check_pairs(transform_and_reset(ipt.ipy_prompt), syntax['ipy_prompt']) | ||
for example in syntax_ml['ipy_prompt']: | ||||
transform_checker(example, ipt.ipy_prompt) | ||||
|
r10091 | |||
|
r11443 | def test_coding_cookie(): | ||
tt.check_pairs(transform_and_reset(ipt.strip_encoding_cookie), syntax['strip_encoding_cookie']) | ||||
for example in syntax_ml['strip_encoding_cookie']: | ||||
transform_checker(example, ipt.strip_encoding_cookie) | ||||
|
r10107 | def test_assemble_logical_lines(): | ||
tests = \ | ||||
[ [(u"a = \\", None), | ||||
|
r10112 | (u"123", u"a = 123"), | ||
|
r10107 | ], | ||
[(u"a = \\", None), # Test resetting when within a multi-line string | ||||
(u"12 *\\", None), | ||||
|
r10112 | (None, u"a = 12 *"), | ||
], | ||||
[(u"# foo\\", u"# foo\\"), # Comments can't be continued like this | ||||
|
r10107 | ], | ||
] | ||||
for example in tests: | ||||
transform_checker(example, ipt.assemble_logical_lines) | ||||
def test_assemble_python_lines(): | ||||
tests = \ | ||||
[ [(u"a = '''", None), | ||||
(u"abc'''", u"a = '''\nabc'''"), | ||||
], | ||||
[(u"a = '''", None), # Test resetting when within a multi-line string | ||||
(u"def", None), | ||||
(None, u"a = '''\ndef"), | ||||
], | ||||
[(u"a = [1,", None), | ||||
(u"2]", u"a = [1,\n2]"), | ||||
], | ||||
[(u"a = [1,", None), # Test resetting when within a multi-line string | ||||
(u"2,", None), | ||||
(None, u"a = [1,\n2,"), | ||||
], | ||||
|
r10112 | ] + syntax_ml['multiline_datastructure'] | ||
|
r10107 | for example in tests: | ||
transform_checker(example, ipt.assemble_python_lines) | ||||
|
r10095 | def test_help_end(): | ||
tt.check_pairs(transform_and_reset(ipt.help_end), syntax['end_help']) | ||||
|
r10091 | |||
|
r10095 | def test_escaped_noesc(): | ||
|
r10107 | tt.check_pairs(transform_and_reset(ipt.escaped_commands), syntax['escaped_noesc']) | ||
|
r10092 | |||
|
r10095 | def test_escaped_shell(): | ||
|
r10107 | tt.check_pairs(transform_and_reset(ipt.escaped_commands), syntax['escaped_shell']) | ||
|
r10092 | |||
|
r10095 | def test_escaped_help(): | ||
|
r10107 | tt.check_pairs(transform_and_reset(ipt.escaped_commands), syntax['escaped_help']) | ||
|
r10095 | |||
def test_escaped_magic(): | ||||
|
r10107 | tt.check_pairs(transform_and_reset(ipt.escaped_commands), syntax['escaped_magic']) | ||
|
r10095 | |||
def test_escaped_quote(): | ||||
|
r10107 | tt.check_pairs(transform_and_reset(ipt.escaped_commands), syntax['escaped_quote']) | ||
|
r10095 | |||
def test_escaped_quote2(): | ||||
|
r10107 | tt.check_pairs(transform_and_reset(ipt.escaped_commands), syntax['escaped_quote2']) | ||
|
r10095 | |||
def test_escaped_paren(): | ||||
|
r10107 | tt.check_pairs(transform_and_reset(ipt.escaped_commands), syntax['escaped_paren']) | ||
|
r10095 | |||
def test_cellmagic(): | ||||
for example in syntax_ml['cellmagic']: | ||||
transform_checker(example, ipt.cellmagic) | ||||
|
r10252 | |||
line_example = [(u'%%bar 123', None), | ||||
(u'hello', None), | ||||
(u'' , u_fmt("get_ipython().run_cell_magic({u}'bar', {u}'123', {u}'hello')")), | ||||
] | ||||
transform_checker(line_example, ipt.cellmagic, end_on_blank_line=True) | ||||
|
r10097 | |||
def test_has_comment(): | ||||
tests = [('text', False), | ||||
('text #comment', True), | ||||
('text #comment\n', True), | ||||
('#comment', True), | ||||
('#comment\n', True), | ||||
('a = "#string"', False), | ||||
('a = "#string" # comment', True), | ||||
('a #comment not "string"', True), | ||||
] | ||||
tt.check_pairs(ipt.has_comment, tests) | ||||
|
r10103 | |||
@ipt.TokenInputTransformer.wrap | ||||
def decistmt(tokens): | ||||
"""Substitute Decimals for floats in a string of statements. | ||||
Based on an example from the tokenize module docs. | ||||
""" | ||||
result = [] | ||||
for toknum, tokval, _, _, _ in tokens: | ||||
if toknum == tokenize.NUMBER and '.' in tokval: # replace NUMBER tokens | ||||
for newtok in [ | ||||
(tokenize.NAME, 'Decimal'), | ||||
(tokenize.OP, '('), | ||||
(tokenize.STRING, repr(tokval)), | ||||
(tokenize.OP, ')') | ||||
]: | ||||
yield newtok | ||||
else: | ||||
yield (toknum, tokval) | ||||
def test_token_input_transformer(): | ||||
tests = [(u'1.2', u_fmt(u"Decimal ({u}'1.2')")), | ||||
(u'"1.2"', u'"1.2"'), | ||||
] | ||||
tt.check_pairs(transform_and_reset(decistmt), tests) | ||||
ml_tests = \ | ||||
[ [(u"a = 1.2; b = '''x", None), | ||||
(u"y'''", u_fmt(u"a =Decimal ({u}'1.2');b ='''x\ny'''")), | ||||
], | ||||
|
r10106 | [(u"a = [1.2,", None), | ||
(u"3]", u_fmt(u"a =[Decimal ({u}'1.2'),\n3 ]")), | ||||
|
r10103 | ], | ||
[(u"a = '''foo", None), # Test resetting when within a multi-line string | ||||
(u"bar", None), | ||||
(None, u"a = '''foo\nbar"), | ||||
], | ||||
] | ||||
for example in ml_tests: | ||||
transform_checker(example, decistmt) | ||||