|
|
"""Tests for the token-based transformers in IPython.core.inputtransformer2
|
|
|
|
|
|
Line-based transformers are the simpler ones; token-based transformers are
|
|
|
more complex. See test_inputtransformer2_line for tests for line-based
|
|
|
transformations.
|
|
|
"""
|
|
|
import nose.tools as nt
|
|
|
import string
|
|
|
import sys
|
|
|
from textwrap import dedent
|
|
|
|
|
|
import pytest
|
|
|
|
|
|
from IPython.core import inputtransformer2 as ipt2
|
|
|
from IPython.core.inputtransformer2 import _find_assign_op, make_tokens_by_line
|
|
|
from IPython.testing.decorators import skip
|
|
|
|
|
|
MULTILINE_MAGIC = (
|
|
|
"""\
|
|
|
a = f()
|
|
|
%foo \\
|
|
|
bar
|
|
|
g()
|
|
|
""".splitlines(
|
|
|
keepends=True
|
|
|
),
|
|
|
(2, 0),
|
|
|
"""\
|
|
|
a = f()
|
|
|
get_ipython().run_line_magic('foo', ' bar')
|
|
|
g()
|
|
|
""".splitlines(
|
|
|
keepends=True
|
|
|
),
|
|
|
)
|
|
|
|
|
|
INDENTED_MAGIC = (
|
|
|
"""\
|
|
|
for a in range(5):
|
|
|
%ls
|
|
|
""".splitlines(
|
|
|
keepends=True
|
|
|
),
|
|
|
(2, 4),
|
|
|
"""\
|
|
|
for a in range(5):
|
|
|
get_ipython().run_line_magic('ls', '')
|
|
|
""".splitlines(
|
|
|
keepends=True
|
|
|
),
|
|
|
)
|
|
|
|
|
|
CRLF_MAGIC = (
|
|
|
["a = f()\n", "%ls\r\n", "g()\n"],
|
|
|
(2, 0),
|
|
|
["a = f()\n", "get_ipython().run_line_magic('ls', '')\n", "g()\n"],
|
|
|
)
|
|
|
|
|
|
MULTILINE_MAGIC_ASSIGN = (
|
|
|
"""\
|
|
|
a = f()
|
|
|
b = %foo \\
|
|
|
bar
|
|
|
g()
|
|
|
""".splitlines(
|
|
|
keepends=True
|
|
|
),
|
|
|
(2, 4),
|
|
|
"""\
|
|
|
a = f()
|
|
|
b = get_ipython().run_line_magic('foo', ' bar')
|
|
|
g()
|
|
|
""".splitlines(
|
|
|
keepends=True
|
|
|
),
|
|
|
)
|
|
|
|
|
|
MULTILINE_SYSTEM_ASSIGN = ("""\
|
|
|
a = f()
|
|
|
b = !foo \\
|
|
|
bar
|
|
|
g()
|
|
|
""".splitlines(keepends=True), (2, 4), """\
|
|
|
a = f()
|
|
|
b = get_ipython().getoutput('foo bar')
|
|
|
g()
|
|
|
""".splitlines(keepends=True))
|
|
|
|
|
|
#####
|
|
|
|
|
|
MULTILINE_SYSTEM_ASSIGN_AFTER_DEDENT = ("""\
|
|
|
def test():
|
|
|
for i in range(1):
|
|
|
print(i)
|
|
|
res =! ls
|
|
|
""".splitlines(
|
|
|
keepends=True
|
|
|
),
|
|
|
(4, 7),
|
|
|
"""\
|
|
|
def test():
|
|
|
for i in range(1):
|
|
|
print(i)
|
|
|
res =get_ipython().getoutput(\' ls\')
|
|
|
""".splitlines(
|
|
|
keepends=True
|
|
|
),
|
|
|
)
|
|
|
|
|
|
######
|
|
|
|
|
|
AUTOCALL_QUOTE = ([",f 1 2 3\n"], (1, 0), ['f("1", "2", "3")\n'])
|
|
|
|
|
|
AUTOCALL_QUOTE2 = ([";f 1 2 3\n"], (1, 0), ['f("1 2 3")\n'])
|
|
|
|
|
|
AUTOCALL_PAREN = (["/f 1 2 3\n"], (1, 0), ["f(1, 2, 3)\n"])
|
|
|
|
|
|
SIMPLE_HELP = (["foo?\n"], (1, 0), ["get_ipython().run_line_magic('pinfo', 'foo')\n"])
|
|
|
|
|
|
DETAILED_HELP = (
|
|
|
["foo??\n"],
|
|
|
(1, 0),
|
|
|
["get_ipython().run_line_magic('pinfo2', 'foo')\n"],
|
|
|
)
|
|
|
|
|
|
MAGIC_HELP = (["%foo?\n"], (1, 0), ["get_ipython().run_line_magic('pinfo', '%foo')\n"])
|
|
|
|
|
|
HELP_IN_EXPR = (
|
|
|
["a = b + c?\n"],
|
|
|
(1, 0),
|
|
|
["get_ipython().run_line_magic('pinfo', 'c')\n"],
|
|
|
)
|
|
|
|
|
|
HELP_CONTINUED_LINE = (
|
|
|
"""\
|
|
|
a = \\
|
|
|
zip?
|
|
|
""".splitlines(
|
|
|
keepends=True
|
|
|
),
|
|
|
(1, 0),
|
|
|
[r"get_ipython().run_line_magic('pinfo', 'zip')" + "\n"],
|
|
|
)
|
|
|
|
|
|
HELP_MULTILINE = (
|
|
|
"""\
|
|
|
(a,
|
|
|
b) = zip?
|
|
|
""".splitlines(
|
|
|
keepends=True
|
|
|
),
|
|
|
(1, 0),
|
|
|
[r"get_ipython().run_line_magic('pinfo', 'zip')" + "\n"],
|
|
|
)
|
|
|
|
|
|
HELP_UNICODE = (
|
|
|
["π.foo?\n"],
|
|
|
(1, 0),
|
|
|
["get_ipython().run_line_magic('pinfo', 'π.foo')\n"],
|
|
|
)
|
|
|
|
|
|
|
|
|
def null_cleanup_transformer(lines):
|
|
|
"""
|
|
|
A cleanup transform that returns an empty list.
|
|
|
"""
|
|
|
return []
|
|
|
|
|
|
def check_make_token_by_line_never_ends_empty():
|
|
|
"""
|
|
|
Check that not sequence of single or double characters ends up leading to en empty list of tokens
|
|
|
"""
|
|
|
from string import printable
|
|
|
|
|
|
for c in printable:
|
|
|
nt.assert_not_equal(make_tokens_by_line(c)[-1], [])
|
|
|
for k in printable:
|
|
|
nt.assert_not_equal(make_tokens_by_line(c+k)[-1], [])
|
|
|
|
|
|
def check_find(transformer, case, match=True):
|
|
|
sample, expected_start, _ = case
|
|
|
tbl = make_tokens_by_line(sample)
|
|
|
res = transformer.find(tbl)
|
|
|
if match:
|
|
|
# start_line is stored 0-indexed, expected values are 1-indexed
|
|
|
nt.assert_equal((res.start_line+1, res.start_col), expected_start)
|
|
|
return res
|
|
|
else:
|
|
|
nt.assert_is(res, None)
|
|
|
|
|
|
|
|
|
def check_transform(transformer_cls, case):
|
|
|
lines, start, expected = case
|
|
|
transformer = transformer_cls(start)
|
|
|
nt.assert_equal(transformer.transform(lines), expected)
|
|
|
|
|
|
|
|
|
def test_continued_line():
|
|
|
lines = MULTILINE_MAGIC_ASSIGN[0]
|
|
|
nt.assert_equal(ipt2.find_end_of_continued_line(lines, 1), 2)
|
|
|
|
|
|
nt.assert_equal(ipt2.assemble_continued_line(lines, (1, 5), 2), "foo bar")
|
|
|
|
|
|
|
|
|
def test_find_assign_magic():
|
|
|
check_find(ipt2.MagicAssign, MULTILINE_MAGIC_ASSIGN)
|
|
|
check_find(ipt2.MagicAssign, MULTILINE_SYSTEM_ASSIGN, match=False)
|
|
|
check_find(ipt2.MagicAssign, MULTILINE_SYSTEM_ASSIGN_AFTER_DEDENT, match=False)
|
|
|
|
|
|
|
|
|
def test_transform_assign_magic():
|
|
|
check_transform(ipt2.MagicAssign, MULTILINE_MAGIC_ASSIGN)
|
|
|
|
|
|
|
|
|
def test_find_assign_system():
|
|
|
check_find(ipt2.SystemAssign, MULTILINE_SYSTEM_ASSIGN)
|
|
|
check_find(ipt2.SystemAssign, MULTILINE_SYSTEM_ASSIGN_AFTER_DEDENT)
|
|
|
check_find(ipt2.SystemAssign, (["a = !ls\n"], (1, 5), None))
|
|
|
check_find(ipt2.SystemAssign, (["a=!ls\n"], (1, 2), None))
|
|
|
check_find(ipt2.SystemAssign, MULTILINE_MAGIC_ASSIGN, match=False)
|
|
|
|
|
|
|
|
|
def test_transform_assign_system():
|
|
|
check_transform(ipt2.SystemAssign, MULTILINE_SYSTEM_ASSIGN)
|
|
|
check_transform(ipt2.SystemAssign, MULTILINE_SYSTEM_ASSIGN_AFTER_DEDENT)
|
|
|
|
|
|
|
|
|
def test_find_magic_escape():
|
|
|
check_find(ipt2.EscapedCommand, MULTILINE_MAGIC)
|
|
|
check_find(ipt2.EscapedCommand, INDENTED_MAGIC)
|
|
|
check_find(ipt2.EscapedCommand, MULTILINE_MAGIC_ASSIGN, match=False)
|
|
|
|
|
|
|
|
|
def test_transform_magic_escape():
|
|
|
check_transform(ipt2.EscapedCommand, MULTILINE_MAGIC)
|
|
|
check_transform(ipt2.EscapedCommand, INDENTED_MAGIC)
|
|
|
check_transform(ipt2.EscapedCommand, CRLF_MAGIC)
|
|
|
|
|
|
|
|
|
def test_find_autocalls():
|
|
|
for case in [AUTOCALL_QUOTE, AUTOCALL_QUOTE2, AUTOCALL_PAREN]:
|
|
|
print("Testing %r" % case[0])
|
|
|
check_find(ipt2.EscapedCommand, case)
|
|
|
|
|
|
|
|
|
def test_transform_autocall():
|
|
|
for case in [AUTOCALL_QUOTE, AUTOCALL_QUOTE2, AUTOCALL_PAREN]:
|
|
|
print("Testing %r" % case[0])
|
|
|
check_transform(ipt2.EscapedCommand, case)
|
|
|
|
|
|
|
|
|
def test_find_help():
|
|
|
for case in [SIMPLE_HELP, DETAILED_HELP, MAGIC_HELP, HELP_IN_EXPR]:
|
|
|
check_find(ipt2.HelpEnd, case)
|
|
|
|
|
|
tf = check_find(ipt2.HelpEnd, HELP_CONTINUED_LINE)
|
|
|
nt.assert_equal(tf.q_line, 1)
|
|
|
nt.assert_equal(tf.q_col, 3)
|
|
|
|
|
|
tf = check_find(ipt2.HelpEnd, HELP_MULTILINE)
|
|
|
nt.assert_equal(tf.q_line, 1)
|
|
|
nt.assert_equal(tf.q_col, 8)
|
|
|
|
|
|
# ? in a comment does not trigger help
|
|
|
check_find(ipt2.HelpEnd, (["foo # bar?\n"], None, None), match=False)
|
|
|
# Nor in a string
|
|
|
check_find(ipt2.HelpEnd, (["foo = '''bar?\n"], None, None), match=False)
|
|
|
|
|
|
|
|
|
def test_transform_help():
|
|
|
tf = ipt2.HelpEnd((1, 0), (1, 9))
|
|
|
nt.assert_equal(tf.transform(HELP_IN_EXPR[0]), HELP_IN_EXPR[2])
|
|
|
|
|
|
tf = ipt2.HelpEnd((1, 0), (2, 3))
|
|
|
nt.assert_equal(tf.transform(HELP_CONTINUED_LINE[0]), HELP_CONTINUED_LINE[2])
|
|
|
|
|
|
tf = ipt2.HelpEnd((1, 0), (2, 8))
|
|
|
nt.assert_equal(tf.transform(HELP_MULTILINE[0]), HELP_MULTILINE[2])
|
|
|
|
|
|
tf = ipt2.HelpEnd((1, 0), (1, 0))
|
|
|
nt.assert_equal(tf.transform(HELP_UNICODE[0]), HELP_UNICODE[2])
|
|
|
|
|
|
|
|
|
def test_find_assign_op_dedent():
|
|
|
"""
|
|
|
be careful that empty token like dedent are not counted as parens
|
|
|
"""
|
|
|
|
|
|
class Tk:
|
|
|
def __init__(self, s):
|
|
|
self.string = s
|
|
|
|
|
|
nt.assert_equal(_find_assign_op([Tk(s) for s in ('','a','=','b')]), 2)
|
|
|
nt.assert_equal(_find_assign_op([Tk(s) for s in ('','(', 'a','=','b', ')', '=' ,'5')]), 6)
|
|
|
|
|
|
examples = [
|
|
|
pytest.param("a = 1", "complete", None),
|
|
|
pytest.param("for a in range(5):", "incomplete", 4),
|
|
|
pytest.param("for a in range(5):\n if a > 0:", "incomplete", 8),
|
|
|
pytest.param("raise = 2", "invalid", None),
|
|
|
pytest.param("a = [1,\n2,", "incomplete", 0),
|
|
|
pytest.param("(\n))", "incomplete", 0),
|
|
|
pytest.param("\\\r\n", "incomplete", 0),
|
|
|
pytest.param("a = '''\n hi", "incomplete", 3),
|
|
|
pytest.param("def a():\n x=1\n global x", "invalid", None),
|
|
|
pytest.param(
|
|
|
"a \\ ",
|
|
|
"invalid",
|
|
|
None,
|
|
|
marks=pytest.mark.xfail(
|
|
|
reason="Bug in python 3.9.8 – bpo 45738",
|
|
|
condition=sys.version_info[:3] == (3, 9, 8),
|
|
|
raises=SystemError,
|
|
|
strict=True,
|
|
|
),
|
|
|
), # Nothing allowed after backslash,
|
|
|
pytest.param("1\\\n+2", "complete", None),
|
|
|
]
|
|
|
|
|
|
|
|
|
@skip('Tested on master, skip only on iptest not available on 7.x')
|
|
|
@pytest.mark.xfail(
|
|
|
reason="Bug in python 3.9.8 – bpo 45738",
|
|
|
condition=sys.version_info[:3] == (3, 9, 8),
|
|
|
)
|
|
|
def test_check_complete():
|
|
|
cc = ipt2.TransformerManager().check_complete
|
|
|
|
|
|
example = dedent(
|
|
|
"""
|
|
|
if True:
|
|
|
a=1"""
|
|
|
)
|
|
|
|
|
|
nt.assert_equal(cc(example), ('incomplete', 4))
|
|
|
nt.assert_equal(cc(example+'\n'), ('complete', None))
|
|
|
nt.assert_equal(cc(example+'\n '), ('complete', None))
|
|
|
|
|
|
# no need to loop on all the letters/numbers.
|
|
|
short = "12abAB" + string.printable[62:]
|
|
|
for c in short:
|
|
|
# test does not raise:
|
|
|
cc(c)
|
|
|
for k in short:
|
|
|
cc(c + k)
|
|
|
|
|
|
nt.assert_equal(cc("def f():\n x=0\n \\\n "), ('incomplete', 2))
|
|
|
|
|
|
def test_check_complete_II():
|
|
|
"""
|
|
|
Test that multiple line strings are properly handled.
|
|
|
|
|
|
Separate test function for convenience
|
|
|
|
|
|
"""
|
|
|
cc = ipt2.TransformerManager().check_complete
|
|
|
nt.assert_equal(cc('''def foo():\n """'''), ('incomplete', 4))
|
|
|
|
|
|
|
|
|
def test_null_cleanup_transformer():
|
|
|
manager = ipt2.TransformerManager()
|
|
|
manager.cleanup_transforms.insert(0, null_cleanup_transformer)
|
|
|
assert manager.transform_cell("") == ""
|
|
|
|
|
|
|
|
|
def test_side_effects_I():
|
|
|
count = 0
|
|
|
|
|
|
def counter(lines):
|
|
|
nonlocal count
|
|
|
count += 1
|
|
|
return lines
|
|
|
|
|
|
counter.has_side_effects = True
|
|
|
|
|
|
manager = ipt2.TransformerManager()
|
|
|
manager.cleanup_transforms.insert(0, counter)
|
|
|
assert manager.check_complete("a=1\n") == ("complete", None)
|
|
|
assert count == 0
|
|
|
|
|
|
|
|
|
def test_side_effects_II():
|
|
|
count = 0
|
|
|
|
|
|
def counter(lines):
|
|
|
nonlocal count
|
|
|
count += 1
|
|
|
return lines
|
|
|
|
|
|
counter.has_side_effects = True
|
|
|
|
|
|
manager = ipt2.TransformerManager()
|
|
|
manager.line_transforms.insert(0, counter)
|
|
|
assert manager.check_complete("b=1\n") == ("complete", None)
|
|
|
assert count == 0
|
|
|
|
