"""Tests for the token-based transformers in IPython.core.inputtransformer2 Line-based transformers are the simpler ones; token-based transformers are more complex. See test_inputtransformer2_line for tests for line-based transformations. """ import string import sys from textwrap import dedent import pytest from IPython.core import inputtransformer2 as ipt2 from IPython.core.inputtransformer2 import _find_assign_op, make_tokens_by_line from IPython.testing.decorators import skip_iptest_but_not_pytest MULTILINE_MAGIC = ("""\ a = f() %foo \\ bar g() """.splitlines(keepends=True), (2, 0), """\ a = f() get_ipython().run_line_magic('foo', ' bar') g() """.splitlines(keepends=True)) INDENTED_MAGIC = ("""\ for a in range(5): %ls """.splitlines(keepends=True), (2, 4), """\ for a in range(5): get_ipython().run_line_magic('ls', '') """.splitlines(keepends=True)) CRLF_MAGIC = ([ "a = f()\n", "%ls\r\n", "g()\n" ], (2, 0), [ "a = f()\n", "get_ipython().run_line_magic('ls', '')\n", "g()\n" ]) MULTILINE_MAGIC_ASSIGN = ("""\ a = f() b = %foo \\ bar g() """.splitlines(keepends=True), (2, 4), """\ a = f() b = get_ipython().run_line_magic('foo', ' bar') g() """.splitlines(keepends=True)) MULTILINE_SYSTEM_ASSIGN = ("""\ a = f() b = !foo \\ bar g() """.splitlines(keepends=True), (2, 4), """\ a = f() b = get_ipython().getoutput('foo bar') g() """.splitlines(keepends=True)) ##### MULTILINE_SYSTEM_ASSIGN_AFTER_DEDENT = ("""\ def test(): for i in range(1): print(i) res =! ls """.splitlines(keepends=True), (4, 7), '''\ def test(): for i in range(1): print(i) res =get_ipython().getoutput(\' ls\') '''.splitlines(keepends=True)) ###### AUTOCALL_QUOTE = ( [",f 1 2 3\n"], (1, 0), ['f("1", "2", "3")\n'] ) AUTOCALL_QUOTE2 = ( [";f 1 2 3\n"], (1, 0), ['f("1 2 3")\n'] ) AUTOCALL_PAREN = ( ["/f 1 2 3\n"], (1, 0), ['f(1, 2, 3)\n'] ) SIMPLE_HELP = ( ["foo?\n"], (1, 0), ["get_ipython().run_line_magic('pinfo', 'foo')\n"] ) DETAILED_HELP = ( ["foo??\n"], (1, 0), ["get_ipython().run_line_magic('pinfo2', 'foo')\n"] ) MAGIC_HELP = ( ["%foo?\n"], (1, 0), ["get_ipython().run_line_magic('pinfo', '%foo')\n"] ) HELP_IN_EXPR = ( ["a = b + c?\n"], (1, 0), ["get_ipython().set_next_input('a = b + c');" "get_ipython().run_line_magic('pinfo', 'c')\n"] ) HELP_CONTINUED_LINE = ("""\ a = \\ zip? """.splitlines(keepends=True), (1, 0), [r"get_ipython().set_next_input('a = \\\nzip');get_ipython().run_line_magic('pinfo', 'zip')" + "\n"] ) HELP_MULTILINE = ("""\ (a, b) = zip? """.splitlines(keepends=True), (1, 0), [r"get_ipython().set_next_input('(a,\nb) = zip');get_ipython().run_line_magic('pinfo', 'zip')" + "\n"] ) HELP_UNICODE = ( ["π.foo?\n"], (1, 0), ["get_ipython().run_line_magic('pinfo', 'π.foo')\n"] ) def null_cleanup_transformer(lines): """ A cleanup transform that returns an empty list. """ return [] def check_make_token_by_line_never_ends_empty(): """ Check that not sequence of single or double characters ends up leading to en empty list of tokens """ from string import printable for c in printable: assert make_tokens_by_line(c)[-1] != [] for k in printable: assert make_tokens_by_line(c + k)[-1] != [] def check_find(transformer, case, match=True): sample, expected_start, _ = case tbl = make_tokens_by_line(sample) res = transformer.find(tbl) if match: # start_line is stored 0-indexed, expected values are 1-indexed assert (res.start_line + 1, res.start_col) == expected_start return res else: assert res is None def check_transform(transformer_cls, case): lines, start, expected = case transformer = transformer_cls(start) assert transformer.transform(lines) == expected def test_continued_line(): lines = MULTILINE_MAGIC_ASSIGN[0] assert ipt2.find_end_of_continued_line(lines, 1) == 2 assert ipt2.assemble_continued_line(lines, (1, 5), 2) == "foo bar" def test_find_assign_magic(): check_find(ipt2.MagicAssign, MULTILINE_MAGIC_ASSIGN) check_find(ipt2.MagicAssign, MULTILINE_SYSTEM_ASSIGN, match=False) check_find(ipt2.MagicAssign, MULTILINE_SYSTEM_ASSIGN_AFTER_DEDENT, match=False) def test_transform_assign_magic(): check_transform(ipt2.MagicAssign, MULTILINE_MAGIC_ASSIGN) def test_find_assign_system(): check_find(ipt2.SystemAssign, MULTILINE_SYSTEM_ASSIGN) check_find(ipt2.SystemAssign, MULTILINE_SYSTEM_ASSIGN_AFTER_DEDENT) check_find(ipt2.SystemAssign, (["a = !ls\n"], (1, 5), None)) check_find(ipt2.SystemAssign, (["a=!ls\n"], (1, 2), None)) check_find(ipt2.SystemAssign, MULTILINE_MAGIC_ASSIGN, match=False) def test_transform_assign_system(): check_transform(ipt2.SystemAssign, MULTILINE_SYSTEM_ASSIGN) check_transform(ipt2.SystemAssign, MULTILINE_SYSTEM_ASSIGN_AFTER_DEDENT) def test_find_magic_escape(): check_find(ipt2.EscapedCommand, MULTILINE_MAGIC) check_find(ipt2.EscapedCommand, INDENTED_MAGIC) check_find(ipt2.EscapedCommand, MULTILINE_MAGIC_ASSIGN, match=False) def test_transform_magic_escape(): check_transform(ipt2.EscapedCommand, MULTILINE_MAGIC) check_transform(ipt2.EscapedCommand, INDENTED_MAGIC) check_transform(ipt2.EscapedCommand, CRLF_MAGIC) def test_find_autocalls(): for case in [AUTOCALL_QUOTE, AUTOCALL_QUOTE2, AUTOCALL_PAREN]: print("Testing %r" % case[0]) check_find(ipt2.EscapedCommand, case) def test_transform_autocall(): for case in [AUTOCALL_QUOTE, AUTOCALL_QUOTE2, AUTOCALL_PAREN]: print("Testing %r" % case[0]) check_transform(ipt2.EscapedCommand, case) def test_find_help(): for case in [SIMPLE_HELP, DETAILED_HELP, MAGIC_HELP, HELP_IN_EXPR]: check_find(ipt2.HelpEnd, case) tf = check_find(ipt2.HelpEnd, HELP_CONTINUED_LINE) assert tf.q_line == 1 assert tf.q_col == 3 tf = check_find(ipt2.HelpEnd, HELP_MULTILINE) assert tf.q_line == 1 assert tf.q_col == 8 # ? in a comment does not trigger help check_find(ipt2.HelpEnd, (["foo # bar?\n"], None, None), match=False) # Nor in a string check_find(ipt2.HelpEnd, (["foo = '''bar?\n"], None, None), match=False) def test_transform_help(): tf = ipt2.HelpEnd((1, 0), (1, 9)) assert tf.transform(HELP_IN_EXPR[0]) == HELP_IN_EXPR[2] tf = ipt2.HelpEnd((1, 0), (2, 3)) assert tf.transform(HELP_CONTINUED_LINE[0]) == HELP_CONTINUED_LINE[2] tf = ipt2.HelpEnd((1, 0), (2, 8)) assert tf.transform(HELP_MULTILINE[0]) == HELP_MULTILINE[2] tf = ipt2.HelpEnd((1, 0), (1, 0)) assert tf.transform(HELP_UNICODE[0]) == HELP_UNICODE[2] def test_find_assign_op_dedent(): """ be careful that empty token like dedent are not counted as parens """ class Tk: def __init__(self, s): self.string = s assert _find_assign_op([Tk(s) for s in ("", "a", "=", "b")]) == 2 assert ( _find_assign_op([Tk(s) for s in ("", "(", "a", "=", "b", ")", "=", "5")]) == 6 ) examples = [ pytest.param("a = 1", "complete", None), pytest.param("for a in range(5):", "incomplete", 4), pytest.param("for a in range(5):\n if a > 0:", "incomplete", 8), pytest.param("raise = 2", "invalid", None), pytest.param("a = [1,\n2,", "incomplete", 0), pytest.param("(\n))", "incomplete", 0), pytest.param("\\\r\n", "incomplete", 0), pytest.param("a = '''\n hi", "incomplete", 3), pytest.param("def a():\n x=1\n global x", "invalid", None), pytest.param( "a \\ ", "invalid", None, marks=pytest.mark.xfail( reason="Bug in python 3.9.8 – bpo 45738", condition=sys.version_info[:3] == (3, 9, 8), raises=SystemError, strict=True, ), ), # Nothing allowed after backslash, pytest.param("1\\\n+2", "complete", None), ] @skip_iptest_but_not_pytest @pytest.mark.parametrize("code, expected, number", examples) def test_check_complete_param(code, expected, number): cc = ipt2.TransformerManager().check_complete assert cc(code) == (expected, number) @skip_iptest_but_not_pytest @pytest.mark.xfail( reason="Bug in python 3.9.8 – bpo 45738", condition=sys.version_info[:3] == (3, 9, 8), ) def test_check_complete(): cc = ipt2.TransformerManager().check_complete example = dedent(""" if True: a=1""" ) assert cc(example) == ("incomplete", 4) assert cc(example + "\n") == ("complete", None) assert cc(example + "\n ") == ("complete", None) # no need to loop on all the letters/numbers. short = '12abAB'+string.printable[62:] for c in short: # test does not raise: cc(c) for k in short: cc(c+k) assert cc("def f():\n x=0\n \\\n ") == ("incomplete", 2) def test_check_complete_II(): """ Test that multiple line strings are properly handled. Separate test function for convenience """ cc = ipt2.TransformerManager().check_complete assert cc('''def foo():\n """''') == ("incomplete", 4) def test_check_complete_invalidates_sunken_brackets(): """ Test that a single line with more closing brackets than the opening ones is interpreted as invalid """ cc = ipt2.TransformerManager().check_complete assert cc(")") == ("invalid", None) assert cc("]") == ("invalid", None) assert cc("}") == ("invalid", None) assert cc(")(") == ("invalid", None) assert cc("][") == ("invalid", None) assert cc("}{") == ("invalid", None) assert cc("]()(") == ("invalid", None) assert cc("())(") == ("invalid", None) assert cc(")[](") == ("invalid", None) assert cc("()](") == ("invalid", None) def test_null_cleanup_transformer(): manager = ipt2.TransformerManager() manager.cleanup_transforms.insert(0, null_cleanup_transformer) assert manager.transform_cell("") == "" def test_side_effects_I(): count = 0 def counter(lines): nonlocal count count += 1 return lines counter.has_side_effects = True manager = ipt2.TransformerManager() manager.cleanup_transforms.insert(0, counter) assert manager.check_complete("a=1\n") == ('complete', None) assert count == 0 def test_side_effects_II(): count = 0 def counter(lines): nonlocal count count += 1 return lines counter.has_side_effects = True manager = ipt2.TransformerManager() manager.line_transforms.insert(0, counter) assert manager.check_complete("b=1\n") == ('complete', None) assert count == 0