test_inputtransformer2.py
385 lines
| 10.3 KiB
| text/x-python
|
PythonLexer
Matthias Bussonnier
|
r24406 | """Tests for the token-based transformers in IPython.core.inputtransformer2 | ||
Line-based transformers are the simpler ones; token-based transformers are | ||||
Thomas Kluyver
|
r24407 | more complex. See test_inputtransformer2_line for tests for line-based | ||
transformations. | ||||
Matthias Bussonnier
|
r24406 | """ | ||
Matthias Bussonnier
|
r24566 | import string | ||
Matthias Bussonnier
|
r27020 | import sys | ||
from textwrap import dedent | ||||
Thomas Kluyver
|
r24155 | |||
Matthias Bussonnier
|
r27020 | import pytest | ||
Thomas Kluyver
|
r24155 | |||
Matthias Bussonnier
|
r27020 | from IPython.core import inputtransformer2 as ipt2 | ||
from IPython.core.inputtransformer2 import _find_assign_op, make_tokens_by_line | ||||
Matthias Bussonnier
|
r24637 | |||
Thomas Kluyver
|
r24159 | MULTILINE_MAGIC = ("""\ | ||
a = f() | ||||
%foo \\ | ||||
bar | ||||
g() | ||||
Thomas Kluyver
|
r24161 | """.splitlines(keepends=True), (2, 0), """\ | ||
Thomas Kluyver
|
r24159 | a = f() | ||
get_ipython().run_line_magic('foo', ' bar') | ||||
g() | ||||
""".splitlines(keepends=True)) | ||||
Thomas Kluyver
|
r24160 | INDENTED_MAGIC = ("""\ | ||
for a in range(5): | ||||
%ls | ||||
Thomas Kluyver
|
r24161 | """.splitlines(keepends=True), (2, 4), """\ | ||
Thomas Kluyver
|
r24160 | for a in range(5): | ||
get_ipython().run_line_magic('ls', '') | ||||
""".splitlines(keepends=True)) | ||||
Kyle Cutler
|
r25936 | CRLF_MAGIC = ([ | ||
"a = f()\n", | ||||
"%ls\r\n", | ||||
"g()\n" | ||||
], (2, 0), [ | ||||
"a = f()\n", | ||||
"get_ipython().run_line_magic('ls', '')\n", | ||||
"g()\n" | ||||
]) | ||||
Thomas Kluyver
|
r24155 | MULTILINE_MAGIC_ASSIGN = ("""\ | ||
a = f() | ||||
b = %foo \\ | ||||
bar | ||||
g() | ||||
Thomas Kluyver
|
r24161 | """.splitlines(keepends=True), (2, 4), """\ | ||
Thomas Kluyver
|
r24155 | a = f() | ||
b = get_ipython().run_line_magic('foo', ' bar') | ||||
g() | ||||
""".splitlines(keepends=True)) | ||||
MULTILINE_SYSTEM_ASSIGN = ("""\ | ||||
a = f() | ||||
b = !foo \\ | ||||
bar | ||||
g() | ||||
Thomas Kluyver
|
r24161 | """.splitlines(keepends=True), (2, 4), """\ | ||
Thomas Kluyver
|
r24155 | a = f() | ||
Thomas Kluyver
|
r24156 | b = get_ipython().getoutput('foo bar') | ||
Thomas Kluyver
|
r24155 | g() | ||
""".splitlines(keepends=True)) | ||||
Matthias Bussonnier
|
r24728 | ##### | ||
MULTILINE_SYSTEM_ASSIGN_AFTER_DEDENT = ("""\ | ||||
def test(): | ||||
for i in range(1): | ||||
print(i) | ||||
res =! ls | ||||
""".splitlines(keepends=True), (4, 7), '''\ | ||||
def test(): | ||||
for i in range(1): | ||||
print(i) | ||||
res =get_ipython().getoutput(\' ls\') | ||||
'''.splitlines(keepends=True)) | ||||
###### | ||||
Thomas Kluyver
|
r24160 | AUTOCALL_QUOTE = ( | ||
Thomas Kluyver
|
r24161 | [",f 1 2 3\n"], (1, 0), | ||
Thomas Kluyver
|
r24160 | ['f("1", "2", "3")\n'] | ||
) | ||||
AUTOCALL_QUOTE2 = ( | ||||
Thomas Kluyver
|
r24161 | [";f 1 2 3\n"], (1, 0), | ||
Thomas Kluyver
|
r24160 | ['f("1 2 3")\n'] | ||
) | ||||
AUTOCALL_PAREN = ( | ||||
Thomas Kluyver
|
r24161 | ["/f 1 2 3\n"], (1, 0), | ||
Thomas Kluyver
|
r24160 | ['f(1, 2, 3)\n'] | ||
) | ||||
Thomas Kluyver
|
r24161 | SIMPLE_HELP = ( | ||
["foo?\n"], (1, 0), | ||||
["get_ipython().run_line_magic('pinfo', 'foo')\n"] | ||||
) | ||||
DETAILED_HELP = ( | ||||
["foo??\n"], (1, 0), | ||||
["get_ipython().run_line_magic('pinfo2', 'foo')\n"] | ||||
) | ||||
MAGIC_HELP = ( | ||||
["%foo?\n"], (1, 0), | ||||
["get_ipython().run_line_magic('pinfo', '%foo')\n"] | ||||
) | ||||
HELP_IN_EXPR = ( | ||||
["a = b + c?\n"], (1, 0), | ||||
["get_ipython().set_next_input('a = b + c');" | ||||
"get_ipython().run_line_magic('pinfo', 'c')\n"] | ||||
) | ||||
HELP_CONTINUED_LINE = ("""\ | ||||
a = \\ | ||||
zip? | ||||
""".splitlines(keepends=True), (1, 0), | ||||
[r"get_ipython().set_next_input('a = \\\nzip');get_ipython().run_line_magic('pinfo', 'zip')" + "\n"] | ||||
) | ||||
HELP_MULTILINE = ("""\ | ||||
(a, | ||||
b) = zip? | ||||
""".splitlines(keepends=True), (1, 0), | ||||
[r"get_ipython().set_next_input('(a,\nb) = zip');get_ipython().run_line_magic('pinfo', 'zip')" + "\n"] | ||||
) | ||||
Markus Wageringel
|
r25595 | HELP_UNICODE = ( | ||
["Ï€.foo?\n"], (1, 0), | ||||
["get_ipython().run_line_magic('pinfo', 'Ï€.foo')\n"] | ||||
) | ||||
Matthias Bussonnier
|
r24728 | |||
Tony Fast
|
r24630 | def null_cleanup_transformer(lines): | ||
""" | ||||
A cleanup transform that returns an empty list. | ||||
""" | ||||
return [] | ||||
Nikita Kniazev
|
r27082 | |||
def test_check_make_token_by_line_never_ends_empty(): | ||||
Matthias Bussonnier
|
r24566 | """ | ||
Check that not sequence of single or double characters ends up leading to en empty list of tokens | ||||
""" | ||||
from string import printable | ||||
for c in printable: | ||||
Samuel Gaist
|
r26900 | assert make_tokens_by_line(c)[-1] != [] | ||
Matthias Bussonnier
|
r24566 | for k in printable: | ||
Samuel Gaist
|
r26900 | assert make_tokens_by_line(c + k)[-1] != [] | ||
Matthias Bussonnier
|
r24566 | |||
Thomas Kluyver
|
r24161 | def check_find(transformer, case, match=True): | ||
sample, expected_start, _ = case | ||||
tbl = make_tokens_by_line(sample) | ||||
res = transformer.find(tbl) | ||||
if match: | ||||
# start_line is stored 0-indexed, expected values are 1-indexed | ||||
Samuel Gaist
|
r26900 | assert (res.start_line + 1, res.start_col) == expected_start | ||
Thomas Kluyver
|
r24161 | return res | ||
else: | ||||
Samuel Gaist
|
r26900 | assert res is None | ||
Thomas Kluyver
|
r24161 | |||
def check_transform(transformer_cls, case): | ||||
lines, start, expected = case | ||||
transformer = transformer_cls(start) | ||||
Samuel Gaist
|
r26900 | assert transformer.transform(lines) == expected | ||
Thomas Kluyver
|
r24161 | |||
Thomas Kluyver
|
r24157 | def test_continued_line(): | ||
lines = MULTILINE_MAGIC_ASSIGN[0] | ||||
Samuel Gaist
|
r26900 | assert ipt2.find_end_of_continued_line(lines, 1) == 2 | ||
Thomas Kluyver
|
r24157 | |||
Samuel Gaist
|
r26900 | assert ipt2.assemble_continued_line(lines, (1, 5), 2) == "foo bar" | ||
Thomas Kluyver
|
r24157 | |||
Thomas Kluyver
|
r24155 | def test_find_assign_magic(): | ||
Thomas Kluyver
|
r24161 | check_find(ipt2.MagicAssign, MULTILINE_MAGIC_ASSIGN) | ||
check_find(ipt2.MagicAssign, MULTILINE_SYSTEM_ASSIGN, match=False) | ||||
Matthias Bussonnier
|
r24728 | check_find(ipt2.MagicAssign, MULTILINE_SYSTEM_ASSIGN_AFTER_DEDENT, match=False) | ||
Thomas Kluyver
|
r24155 | |||
def test_transform_assign_magic(): | ||||
Thomas Kluyver
|
r24161 | check_transform(ipt2.MagicAssign, MULTILINE_MAGIC_ASSIGN) | ||
Thomas Kluyver
|
r24156 | |||
def test_find_assign_system(): | ||||
Thomas Kluyver
|
r24161 | check_find(ipt2.SystemAssign, MULTILINE_SYSTEM_ASSIGN) | ||
Matthias Bussonnier
|
r24728 | check_find(ipt2.SystemAssign, MULTILINE_SYSTEM_ASSIGN_AFTER_DEDENT) | ||
Thomas Kluyver
|
r24161 | check_find(ipt2.SystemAssign, (["a = !ls\n"], (1, 5), None)) | ||
check_find(ipt2.SystemAssign, (["a=!ls\n"], (1, 2), None)) | ||||
check_find(ipt2.SystemAssign, MULTILINE_MAGIC_ASSIGN, match=False) | ||||
Thomas Kluyver
|
r24156 | |||
Thomas Kluyver
|
r24161 | def test_transform_assign_system(): | ||
check_transform(ipt2.SystemAssign, MULTILINE_SYSTEM_ASSIGN) | ||||
Matthias Bussonnier
|
r24728 | check_transform(ipt2.SystemAssign, MULTILINE_SYSTEM_ASSIGN_AFTER_DEDENT) | ||
Thomas Kluyver
|
r24156 | |||
Thomas Kluyver
|
r24161 | def test_find_magic_escape(): | ||
check_find(ipt2.EscapedCommand, MULTILINE_MAGIC) | ||||
check_find(ipt2.EscapedCommand, INDENTED_MAGIC) | ||||
check_find(ipt2.EscapedCommand, MULTILINE_MAGIC_ASSIGN, match=False) | ||||
Thomas Kluyver
|
r24156 | |||
Thomas Kluyver
|
r24161 | def test_transform_magic_escape(): | ||
check_transform(ipt2.EscapedCommand, MULTILINE_MAGIC) | ||||
check_transform(ipt2.EscapedCommand, INDENTED_MAGIC) | ||||
Kyle Cutler
|
r25936 | check_transform(ipt2.EscapedCommand, CRLF_MAGIC) | ||
Thomas Kluyver
|
r24156 | |||
Thomas Kluyver
|
r24161 | def test_find_autocalls(): | ||
for case in [AUTOCALL_QUOTE, AUTOCALL_QUOTE2, AUTOCALL_PAREN]: | ||||
print("Testing %r" % case[0]) | ||||
check_find(ipt2.EscapedCommand, case) | ||||
Thomas Kluyver
|
r24159 | |||
Thomas Kluyver
|
r24161 | def test_transform_autocall(): | ||
for case in [AUTOCALL_QUOTE, AUTOCALL_QUOTE2, AUTOCALL_PAREN]: | ||||
print("Testing %r" % case[0]) | ||||
check_transform(ipt2.EscapedCommand, case) | ||||
Thomas Kluyver
|
r24159 | |||
Thomas Kluyver
|
r24161 | def test_find_help(): | ||
for case in [SIMPLE_HELP, DETAILED_HELP, MAGIC_HELP, HELP_IN_EXPR]: | ||||
check_find(ipt2.HelpEnd, case) | ||||
Thomas Kluyver
|
r24160 | |||
Thomas Kluyver
|
r24161 | tf = check_find(ipt2.HelpEnd, HELP_CONTINUED_LINE) | ||
Samuel Gaist
|
r26900 | assert tf.q_line == 1 | ||
assert tf.q_col == 3 | ||||
Thomas Kluyver
|
r24159 | |||
Thomas Kluyver
|
r24161 | tf = check_find(ipt2.HelpEnd, HELP_MULTILINE) | ||
Samuel Gaist
|
r26900 | assert tf.q_line == 1 | ||
assert tf.q_col == 8 | ||||
Thomas Kluyver
|
r24160 | |||
Thomas Kluyver
|
r24161 | # ? in a comment does not trigger help | ||
check_find(ipt2.HelpEnd, (["foo # bar?\n"], None, None), match=False) | ||||
# Nor in a string | ||||
check_find(ipt2.HelpEnd, (["foo = '''bar?\n"], None, None), match=False) | ||||
Thomas Kluyver
|
r24160 | |||
Thomas Kluyver
|
r24161 | def test_transform_help(): | ||
tf = ipt2.HelpEnd((1, 0), (1, 9)) | ||||
Samuel Gaist
|
r26900 | assert tf.transform(HELP_IN_EXPR[0]) == HELP_IN_EXPR[2] | ||
Thomas Kluyver
|
r24160 | |||
Thomas Kluyver
|
r24161 | tf = ipt2.HelpEnd((1, 0), (2, 3)) | ||
Samuel Gaist
|
r26900 | assert tf.transform(HELP_CONTINUED_LINE[0]) == HELP_CONTINUED_LINE[2] | ||
Thomas Kluyver
|
r24161 | |||
tf = ipt2.HelpEnd((1, 0), (2, 8)) | ||||
Samuel Gaist
|
r26900 | assert tf.transform(HELP_MULTILINE[0]) == HELP_MULTILINE[2] | ||
Thomas Kluyver
|
r24165 | |||
Markus Wageringel
|
r25595 | tf = ipt2.HelpEnd((1, 0), (1, 0)) | ||
Samuel Gaist
|
r26900 | assert tf.transform(HELP_UNICODE[0]) == HELP_UNICODE[2] | ||
Markus Wageringel
|
r25595 | |||
Matthias Bussonnier
|
r24728 | def test_find_assign_op_dedent(): | ||
""" | ||||
luz.paz
|
r24756 | be careful that empty token like dedent are not counted as parens | ||
Matthias Bussonnier
|
r24728 | """ | ||
class Tk: | ||||
def __init__(self, s): | ||||
self.string = s | ||||
Samuel Gaist
|
r26900 | assert _find_assign_op([Tk(s) for s in ("", "a", "=", "b")]) == 2 | ||
assert ( | ||||
_find_assign_op([Tk(s) for s in ("", "(", "a", "=", "b", ")", "=", "5")]) == 6 | ||||
) | ||||
Matthias Bussonnier
|
r24728 | |||
Matthias Bussonnier
|
r27020 | examples = [ | ||
pytest.param("a = 1", "complete", None), | ||||
pytest.param("for a in range(5):", "incomplete", 4), | ||||
pytest.param("for a in range(5):\n if a > 0:", "incomplete", 8), | ||||
pytest.param("raise = 2", "invalid", None), | ||||
pytest.param("a = [1,\n2,", "incomplete", 0), | ||||
pytest.param("(\n))", "incomplete", 0), | ||||
pytest.param("\\\r\n", "incomplete", 0), | ||||
pytest.param("a = '''\n hi", "incomplete", 3), | ||||
pytest.param("def a():\n x=1\n global x", "invalid", None), | ||||
pytest.param( | ||||
"a \\ ", | ||||
"invalid", | ||||
None, | ||||
marks=pytest.mark.xfail( | ||||
reason="Bug in python 3.9.8 – bpo 45738", | ||||
condition=sys.version_info[:3] == (3, 9, 8), | ||||
raises=SystemError, | ||||
strict=True, | ||||
), | ||||
), # Nothing allowed after backslash, | ||||
pytest.param("1\\\n+2", "complete", None), | ||||
] | ||||
@pytest.mark.parametrize("code, expected, number", examples) | ||||
def test_check_complete_param(code, expected, number): | ||||
cc = ipt2.TransformerManager().check_complete | ||||
assert cc(code) == (expected, number) | ||||
@pytest.mark.xfail( | ||||
reason="Bug in python 3.9.8 – bpo 45738", | ||||
condition=sys.version_info[:3] == (3, 9, 8), | ||||
) | ||||
Thomas Kluyver
|
r24165 | def test_check_complete(): | ||
Thomas Kluyver
|
r24166 | cc = ipt2.TransformerManager().check_complete | ||
Matthias Bussonnier
|
r24566 | |||
Matthias Bussonnier
|
r24637 | example = dedent(""" | ||
if True: | ||||
a=1""" ) | ||||
Samuel Gaist
|
r26900 | assert cc(example) == ("incomplete", 4) | ||
assert cc(example + "\n") == ("complete", None) | ||||
assert cc(example + "\n ") == ("complete", None) | ||||
Matthias Bussonnier
|
r24637 | |||
Matthias Bussonnier
|
r24566 | # no need to loop on all the letters/numbers. | ||
short = '12abAB'+string.printable[62:] | ||||
for c in short: | ||||
# test does not raise: | ||||
cc(c) | ||||
for k in short: | ||||
cc(c+k) | ||||
Samuel Gaist
|
r26900 | assert cc("def f():\n x=0\n \\\n ") == ("incomplete", 2) | ||
Dominik Miedziński
|
r25393 | |||
Matthias Bussonnier
|
r24701 | def test_check_complete_II(): | ||
""" | ||||
Test that multiple line strings are properly handled. | ||||
Separate test function for convenience | ||||
""" | ||||
cc = ipt2.TransformerManager().check_complete | ||||
Samuel Gaist
|
r26900 | assert cc('''def foo():\n """''') == ("incomplete", 4) | ||
Matthias Bussonnier
|
r24701 | |||
Blazej Michalik
|
r26370 | def test_check_complete_invalidates_sunken_brackets(): | ||
""" | ||||
Test that a single line with more closing brackets than the opening ones is | ||||
Dimitri Papadopoulos
|
r26875 | interpreted as invalid | ||
Blazej Michalik
|
r26370 | """ | ||
cc = ipt2.TransformerManager().check_complete | ||||
Samuel Gaist
|
r26900 | assert cc(")") == ("invalid", None) | ||
assert cc("]") == ("invalid", None) | ||||
assert cc("}") == ("invalid", None) | ||||
assert cc(")(") == ("invalid", None) | ||||
assert cc("][") == ("invalid", None) | ||||
assert cc("}{") == ("invalid", None) | ||||
assert cc("]()(") == ("invalid", None) | ||||
assert cc("())(") == ("invalid", None) | ||||
assert cc(")[](") == ("invalid", None) | ||||
assert cc("()](") == ("invalid", None) | ||||
Blazej Michalik
|
r26370 | |||
Tony Fast
|
r24630 | def test_null_cleanup_transformer(): | ||
manager = ipt2.TransformerManager() | ||||
manager.cleanup_transforms.insert(0, null_cleanup_transformer) | ||||
Samuel Gaist
|
r26900 | assert manager.transform_cell("") == "" | ||
Matthias Bussonnier
|
r25925 | |||
def test_side_effects_I(): | ||||
count = 0 | ||||
def counter(lines): | ||||
nonlocal count | ||||
count += 1 | ||||
return lines | ||||
counter.has_side_effects = True | ||||
manager = ipt2.TransformerManager() | ||||
manager.cleanup_transforms.insert(0, counter) | ||||
assert manager.check_complete("a=1\n") == ('complete', None) | ||||
assert count == 0 | ||||
def test_side_effects_II(): | ||||
count = 0 | ||||
def counter(lines): | ||||
nonlocal count | ||||
count += 1 | ||||
return lines | ||||
counter.has_side_effects = True | ||||
manager = ipt2.TransformerManager() | ||||
manager.line_transforms.insert(0, counter) | ||||
assert manager.check_complete("b=1\n") == ('complete', None) | ||||
assert count == 0 | ||||