IPython/core/tests/test_inputtransformer2.py

"""Tests for the token-based transformers in IPython.core.inputtransformer2

Line-based transformers are the simpler ones; token-based transformers are
more complex. See test_inputtransformer2_line for tests for line-based
transformations.
"""
import string

from IPython.core import inputtransformer2 as ipt2
from IPython.core.inputtransformer2 import make_tokens_by_line, _find_assign_op

from textwrap import dedent

MULTILINE_MAGIC = ("""\
a = f()
%foo \\
bar
g()
""".splitlines(keepends=True), (2, 0), """\
a = f()
get_ipython().run_line_magic('foo', ' bar')
g()
""".splitlines(keepends=True))

INDENTED_MAGIC = ("""\
for a in range(5):
    %ls
""".splitlines(keepends=True), (2, 4), """\
for a in range(5):
    get_ipython().run_line_magic('ls', '')
""".splitlines(keepends=True))

CRLF_MAGIC = ([
    "a = f()\n",
    "%ls\r\n",
    "g()\n"
], (2, 0), [
    "a = f()\n",
    "get_ipython().run_line_magic('ls', '')\n",
    "g()\n"
])

MULTILINE_MAGIC_ASSIGN = ("""\
a = f()
b = %foo \\
  bar
g()
""".splitlines(keepends=True), (2, 4), """\
a = f()
b = get_ipython().run_line_magic('foo', '   bar')
g()
""".splitlines(keepends=True))

MULTILINE_SYSTEM_ASSIGN = ("""\
a = f()
b = !foo \\
  bar
g()
""".splitlines(keepends=True), (2, 4), """\
a = f()
b = get_ipython().getoutput('foo    bar')
g()
""".splitlines(keepends=True))

#####

MULTILINE_SYSTEM_ASSIGN_AFTER_DEDENT = ("""\
def test():
  for i in range(1):
    print(i)
  res =! ls
""".splitlines(keepends=True), (4, 7), '''\
def test():
  for i in range(1):
    print(i)
  res =get_ipython().getoutput(\' ls\')
'''.splitlines(keepends=True))

######

AUTOCALL_QUOTE = (
    [",f 1 2 3\n"], (1, 0),
    ['f("1", "2", "3")\n']
)

AUTOCALL_QUOTE2 = (
    [";f 1 2 3\n"], (1, 0),
    ['f("1 2 3")\n']
)

AUTOCALL_PAREN = (
    ["/f 1 2 3\n"], (1, 0),
    ['f(1, 2, 3)\n']
)

SIMPLE_HELP = (
    ["foo?\n"], (1, 0),
    ["get_ipython().run_line_magic('pinfo', 'foo')\n"]
)

DETAILED_HELP = (
    ["foo??\n"], (1, 0),
    ["get_ipython().run_line_magic('pinfo2', 'foo')\n"]
)

MAGIC_HELP = (
    ["%foo?\n"], (1, 0),
    ["get_ipython().run_line_magic('pinfo', '%foo')\n"]
)

HELP_IN_EXPR = (
    ["a = b + c?\n"], (1, 0),
    ["get_ipython().set_next_input('a = b + c');"
     "get_ipython().run_line_magic('pinfo', 'c')\n"]
)

HELP_CONTINUED_LINE = ("""\
a = \\
zip?
""".splitlines(keepends=True), (1, 0),
[r"get_ipython().set_next_input('a = \\\nzip');get_ipython().run_line_magic('pinfo', 'zip')" + "\n"]
)

HELP_MULTILINE = ("""\
(a,
b) = zip?
""".splitlines(keepends=True), (1, 0),
[r"get_ipython().set_next_input('(a,\nb) = zip');get_ipython().run_line_magic('pinfo', 'zip')" + "\n"]
)

HELP_UNICODE = (
    ["π.foo?\n"], (1, 0),
    ["get_ipython().run_line_magic('pinfo', 'π.foo')\n"]
)


def null_cleanup_transformer(lines):
    """
    A cleanup transform that returns an empty list.
    """
    return []

def check_make_token_by_line_never_ends_empty():
    """
    Check that not sequence of single or double characters ends up leading to en empty list of tokens
    """
    from string import printable
    for c in printable:
        assert make_tokens_by_line(c)[-1] != []
        for k in printable:
            assert make_tokens_by_line(c + k)[-1] != []


def check_find(transformer, case, match=True):
    sample, expected_start, _  = case
    tbl = make_tokens_by_line(sample)
    res = transformer.find(tbl)
    if match:
        # start_line is stored 0-indexed, expected values are 1-indexed
        assert (res.start_line + 1, res.start_col) == expected_start
        return res
    else:
        assert res is None

def check_transform(transformer_cls, case):
    lines, start, expected = case
    transformer = transformer_cls(start)
    assert transformer.transform(lines) == expected

def test_continued_line():
    lines = MULTILINE_MAGIC_ASSIGN[0]
    assert ipt2.find_end_of_continued_line(lines, 1) == 2

    assert ipt2.assemble_continued_line(lines, (1, 5), 2) == "foo    bar"

def test_find_assign_magic():
    check_find(ipt2.MagicAssign, MULTILINE_MAGIC_ASSIGN)
    check_find(ipt2.MagicAssign, MULTILINE_SYSTEM_ASSIGN, match=False)
    check_find(ipt2.MagicAssign, MULTILINE_SYSTEM_ASSIGN_AFTER_DEDENT, match=False)

def test_transform_assign_magic():
    check_transform(ipt2.MagicAssign, MULTILINE_MAGIC_ASSIGN)

def test_find_assign_system():
    check_find(ipt2.SystemAssign, MULTILINE_SYSTEM_ASSIGN)
    check_find(ipt2.SystemAssign, MULTILINE_SYSTEM_ASSIGN_AFTER_DEDENT)
    check_find(ipt2.SystemAssign, (["a =  !ls\n"], (1, 5), None))
    check_find(ipt2.SystemAssign, (["a=!ls\n"], (1, 2), None))
    check_find(ipt2.SystemAssign, MULTILINE_MAGIC_ASSIGN, match=False)

def test_transform_assign_system():
    check_transform(ipt2.SystemAssign, MULTILINE_SYSTEM_ASSIGN)
    check_transform(ipt2.SystemAssign, MULTILINE_SYSTEM_ASSIGN_AFTER_DEDENT)

def test_find_magic_escape():
    check_find(ipt2.EscapedCommand, MULTILINE_MAGIC)
    check_find(ipt2.EscapedCommand, INDENTED_MAGIC)
    check_find(ipt2.EscapedCommand, MULTILINE_MAGIC_ASSIGN, match=False)

def test_transform_magic_escape():
    check_transform(ipt2.EscapedCommand, MULTILINE_MAGIC)
    check_transform(ipt2.EscapedCommand, INDENTED_MAGIC)
    check_transform(ipt2.EscapedCommand, CRLF_MAGIC)

def test_find_autocalls():
    for case in [AUTOCALL_QUOTE, AUTOCALL_QUOTE2, AUTOCALL_PAREN]:
        print("Testing %r" % case[0])
        check_find(ipt2.EscapedCommand, case)

def test_transform_autocall():
    for case in [AUTOCALL_QUOTE, AUTOCALL_QUOTE2, AUTOCALL_PAREN]:
        print("Testing %r" % case[0])
        check_transform(ipt2.EscapedCommand, case)

def test_find_help():
    for case in [SIMPLE_HELP, DETAILED_HELP, MAGIC_HELP, HELP_IN_EXPR]:
        check_find(ipt2.HelpEnd, case)

    tf = check_find(ipt2.HelpEnd, HELP_CONTINUED_LINE)
    assert tf.q_line == 1
    assert tf.q_col == 3

    tf = check_find(ipt2.HelpEnd, HELP_MULTILINE)
    assert tf.q_line == 1
    assert tf.q_col == 8

    # ? in a comment does not trigger help
    check_find(ipt2.HelpEnd, (["foo # bar?\n"], None, None), match=False)
    # Nor in a string
    check_find(ipt2.HelpEnd, (["foo = '''bar?\n"], None, None), match=False)

def test_transform_help():
    tf = ipt2.HelpEnd((1, 0), (1, 9))
    assert tf.transform(HELP_IN_EXPR[0]) == HELP_IN_EXPR[2]

    tf = ipt2.HelpEnd((1, 0), (2, 3))
    assert tf.transform(HELP_CONTINUED_LINE[0]) == HELP_CONTINUED_LINE[2]

    tf = ipt2.HelpEnd((1, 0), (2, 8))
    assert tf.transform(HELP_MULTILINE[0]) == HELP_MULTILINE[2]

    tf = ipt2.HelpEnd((1, 0), (1, 0))
    assert tf.transform(HELP_UNICODE[0]) == HELP_UNICODE[2]

def test_find_assign_op_dedent():
    """
    be careful that empty token like dedent are not counted as parens
    """
    class Tk:
        def __init__(self, s):
            self.string = s

    assert _find_assign_op([Tk(s) for s in ("", "a", "=", "b")]) == 2
    assert (
        _find_assign_op([Tk(s) for s in ("", "(", "a", "=", "b", ")", "=", "5")]) == 6
    )


def test_check_complete():
    cc = ipt2.TransformerManager().check_complete
    assert cc("a = 1") == ("complete", None)
    assert cc("for a in range(5):") == ("incomplete", 4)
    assert cc("for a in range(5):\n    if a > 0:") == ("incomplete", 8)
    assert cc("raise = 2") == ("invalid", None)
    assert cc("a = [1,\n2,") == ("incomplete", 0)
    assert cc("(\n))") == ("incomplete", 0)
    assert cc("\\\r\n") == ("incomplete", 0)
    assert cc("a = '''\n   hi") == ("incomplete", 3)
    assert cc("def a():\n x=1\n global x") == ("invalid", None)
    assert cc("a \\ ") == ("invalid", None)  # Nothing allowed after backslash
    assert cc("1\\\n+2") == ("complete", None)
    assert cc("exit") == ("complete", None)

    example = dedent("""
        if True:
            a=1""" )

    assert cc(example) == ("incomplete", 4)
    assert cc(example + "\n") == ("complete", None)
    assert cc(example + "\n    ") == ("complete", None)

    # no need to loop on all the letters/numbers.
    short = '12abAB'+string.printable[62:]
    for c in short:
        # test does not raise:
        cc(c)
        for k in short:
            cc(c+k)

    assert cc("def f():\n  x=0\n  \\\n  ") == ("incomplete", 2)


def test_check_complete_II():
    """
    Test that multiple line strings are properly handled.

    Separate test function for convenience

    """
    cc = ipt2.TransformerManager().check_complete
    assert cc('''def foo():\n    """''') == ("incomplete", 4)


def test_check_complete_invalidates_sunken_brackets():
    """
    Test that a single line with more closing brackets than the opening ones is
    interpreted as invalid
    """
    cc = ipt2.TransformerManager().check_complete
    assert cc(")") == ("invalid", None)
    assert cc("]") == ("invalid", None)
    assert cc("}") == ("invalid", None)
    assert cc(")(") == ("invalid", None)
    assert cc("][") == ("invalid", None)
    assert cc("}{") == ("invalid", None)
    assert cc("]()(") == ("invalid", None)
    assert cc("())(") == ("invalid", None)
    assert cc(")[](") == ("invalid", None)
    assert cc("()](") == ("invalid", None)


def test_null_cleanup_transformer():
    manager = ipt2.TransformerManager()
    manager.cleanup_transforms.insert(0, null_cleanup_transformer)
    assert manager.transform_cell("") == ""




def test_side_effects_I():
    count = 0
    def counter(lines):
        nonlocal count
        count += 1
        return lines

    counter.has_side_effects = True

    manager = ipt2.TransformerManager()
    manager.cleanup_transforms.insert(0, counter)
    assert manager.check_complete("a=1\n") == ('complete', None)
    assert count == 0




def test_side_effects_II():
    count = 0
    def counter(lines):
        nonlocal count
        count += 1
        return lines

    counter.has_side_effects = True

    manager = ipt2.TransformerManager()
    manager.line_transforms.insert(0, counter)
    assert manager.check_complete("b=1\n") == ('complete', None)
    assert count == 0