IPython/core/tests/test_inputtransformer.py

import tokenize

from IPython.testing import tools as tt

from IPython.core import inputtransformer as ipt

def transform_and_reset(transformer):
    transformer = transformer()
    def transform(inp):
        try:
            return transformer.push(inp)
        finally:
            transformer.reset()
    
    return transform

# Transformer tests
def transform_checker(tests, transformer, **kwargs):
    """Utility to loop over test inputs"""
    transformer = transformer(**kwargs)
    try:
        for inp, tr in tests:
            if inp is None:
                out = transformer.reset()
            else:
                out = transformer.push(inp)
            assert out == tr
    finally:
        transformer.reset()

# Data for all the syntax tests in the form of lists of pairs of
# raw/transformed input.  We store it here as a global dict so that we can use
# it both within single-function tests and also to validate the behavior of the
# larger objects

syntax = \
  dict(assign_system =
       [('a =! ls', "a = get_ipython().getoutput('ls')"),
        ('b = !ls', "b = get_ipython().getoutput('ls')"),
        ('c= !ls', "c = get_ipython().getoutput('ls')"),
        ('d == !ls', 'd == !ls'), # Invalid syntax, but we leave == alone.
        ('x=1', 'x=1'), # normal input is unmodified
        ('    ','    '),  # blank lines are kept intact
        # Tuple unpacking
        ("a, b = !echo 'a\\nb'", "a, b = get_ipython().getoutput(\"echo 'a\\\\nb'\")"),
        ("a,= !echo 'a'", "a, = get_ipython().getoutput(\"echo 'a'\")"),
        ("a, *bc = !echo 'a\\nb\\nc'", "a, *bc = get_ipython().getoutput(\"echo 'a\\\\nb\\\\nc'\")"),
        # Tuple unpacking with regular Python expressions, not our syntax.
        ("a, b = range(2)", "a, b = range(2)"),
        ("a, = range(1)", "a, = range(1)"),
        ("a, *bc = range(3)", "a, *bc = range(3)"),
        ],

       assign_magic =
       [('a =% who', "a = get_ipython().run_line_magic('who', '')"),
        ('b = %who', "b = get_ipython().run_line_magic('who', '')"),
        ('c= %ls', "c = get_ipython().run_line_magic('ls', '')"),
        ('d == %ls', 'd == %ls'), # Invalid syntax, but we leave == alone.
        ('x=1', 'x=1'), # normal input is unmodified
        ('    ','    '),  # blank lines are kept intact
        ("a, b = %foo", "a, b = get_ipython().run_line_magic('foo', '')"),
    ],
    classic_prompt=[
        (">>> x=1", "x=1"),
        ("x=1", "x=1"),  # normal input is unmodified
        ("    ", "    "),  # blank lines are kept intact
    ],
    ipy_prompt=[
        ("In [1]: x=1", "x=1"),
        ("x=1", "x=1"),  # normal input is unmodified
        ("    ", "    "),  # blank lines are kept intact
    ],
    # Tests for the escape transformer to leave normal code alone
    escaped_noesc=[
        ("    ", "    "),
        ("x=1", "x=1"),
    ],
    # System calls
    escaped_shell=[
        ("!ls", "get_ipython().system('ls')"),
        # Double-escape shell, this means to capture the output of the
        # subprocess and return it
        ("!!ls", "get_ipython().getoutput('ls')"),
    ],
    # Help/object info
    escaped_help=[
        ("?", "get_ipython().show_usage()"),
        ("?x1", "get_ipython().run_line_magic('pinfo', 'x1')"),
        ("??x2", "get_ipython().run_line_magic('pinfo2', 'x2')"),
        ("?a.*s", "get_ipython().run_line_magic('psearch', 'a.*s')"),
        ("?%hist1", "get_ipython().run_line_magic('pinfo', '%hist1')"),
        ("?%%hist2", "get_ipython().run_line_magic('pinfo', '%%hist2')"),
        ("?abc = qwe", "get_ipython().run_line_magic('pinfo', 'abc')"),
    ],
    end_help=[
        ("x3?", "get_ipython().run_line_magic('pinfo', 'x3')"),
        ("x4??", "get_ipython().run_line_magic('pinfo2', 'x4')"),
        ("%hist1?", "get_ipython().run_line_magic('pinfo', '%hist1')"),
        ("%hist2??", "get_ipython().run_line_magic('pinfo2', '%hist2')"),
        ("%%hist3?", "get_ipython().run_line_magic('pinfo', '%%hist3')"),
        ("%%hist4??", "get_ipython().run_line_magic('pinfo2', '%%hist4')"),
        ("π.foo?", "get_ipython().run_line_magic('pinfo', 'π.foo')"),
        ("f*?", "get_ipython().run_line_magic('psearch', 'f*')"),
        ("ax.*aspe*?", "get_ipython().run_line_magic('psearch', 'ax.*aspe*')"),
        ("a = abc?", "get_ipython().run_line_magic('pinfo', 'abc')"),
        ("a = abc.qe??", "get_ipython().run_line_magic('pinfo2', 'abc.qe')"),
        ("a = *.items?", "get_ipython().run_line_magic('psearch', '*.items')"),
        ("plot(a?", "get_ipython().run_line_magic('pinfo', 'a')"),
        ("a*2 #comment?", "a*2 #comment?"),
    ],
    # Explicit magic calls
    escaped_magic=[
        ("%cd", "get_ipython().run_line_magic('cd', '')"),
        ("%cd /home", "get_ipython().run_line_magic('cd', '/home')"),
        # Backslashes need to be escaped.
        ("%cd C:\\User", "get_ipython().run_line_magic('cd', 'C:\\\\User')"),
        ("    %magic", "    get_ipython().run_line_magic('magic', '')"),
    ],
    # Quoting with separate arguments
    escaped_quote=[
        (",f", 'f("")'),
        (",f x", 'f("x")'),
        ("  ,f y", '  f("y")'),
        (",f a b", 'f("a", "b")'),
    ],
    # Quoting with single argument
    escaped_quote2=[
        (";f", 'f("")'),
        (";f x", 'f("x")'),
        ("  ;f y", '  f("y")'),
        (";f a b", 'f("a b")'),
    ],
    # Simply apply parens
    escaped_paren=[
        ("/f", "f()"),
        ("/f x", "f(x)"),
        ("  /f y", "  f(y)"),
        ("/f a b", "f(a, b)"),
    ],
    # Check that we transform prompts before other transforms
    mixed=[
        ("In [1]: %lsmagic", "get_ipython().run_line_magic('lsmagic', '')"),
        (">>> %lsmagic", "get_ipython().run_line_magic('lsmagic', '')"),
        ("In [2]: !ls", "get_ipython().system('ls')"),
        ("In [3]: abs?", "get_ipython().run_line_magic('pinfo', 'abs')"),
        ("In [4]: b = %who", "b = get_ipython().run_line_magic('who', '')"),
    ],
)

# multiline syntax examples.  Each of these should be a list of lists, with
# each entry itself having pairs of raw/transformed input.  The union (with
# '\n'.join() of the transformed inputs is what the splitter should produce
# when fed the raw lines one at a time via push.
syntax_ml = \
  dict(classic_prompt =
       [ [('>>> for i in range(10):','for i in range(10):'),
          ('...     print i','    print i'),
          ('... ', ''),
          ],
         [('>>> a="""','a="""'),
          ('... 123"""','123"""'),
          ],
         [('a="""','a="""'),
          ('... 123','123'),
          ('... 456"""','456"""'),
          ],
         [('a="""','a="""'),
          ('>>> 123','123'),
          ('... 456"""','456"""'),
          ],
         [('a="""','a="""'),
          ('123','123'),
          ('... 456"""','... 456"""'),
          ],
         [('....__class__','....__class__'),
         ],
         [('a=5', 'a=5'),
          ('...', ''),
         ],
         [('>>> def f(x):', 'def f(x):'),
          ('...', ''),
          ('...     return x', '    return x'),
          ],
         [('board = """....', 'board = """....'),
          ('....', '....'),
          ('...."""', '...."""'),
          ],
        ],

       ipy_prompt =
       [ [('In [24]: for i in range(10):','for i in range(10):'),
          ('   ....:     print i','    print i'),
          ('   ....: ', ''),
          ],
         [('In [24]: for i in range(10):','for i in range(10):'),
          # Qt console prompts expand with spaces, not dots
          ('    ...:     print i','    print i'),
          ('    ...: ', ''),
          ],
         [('In [24]: for i in range(10):','for i in range(10):'),
          # Sometimes whitespace preceding '...' has been removed
          ('...:     print i','    print i'),
          ('...: ', ''),
          ],
         [('In [24]: for i in range(10):','for i in range(10):'),
          # Space after last continuation prompt has been removed (issue #6674)
          ('...:     print i','    print i'),
          ('...:', ''),
          ],
         [('In [2]: a="""','a="""'),
          ('   ...: 123"""','123"""'),
          ],
         [('a="""','a="""'),
          ('   ...: 123','123'),
          ('   ...: 456"""','456"""'),
          ],
         [('a="""','a="""'),
          ('In [1]: 123','123'),
          ('   ...: 456"""','456"""'),
          ],
         [('a="""','a="""'),
          ('123','123'),
          ('   ...: 456"""','   ...: 456"""'),
          ],
         ],

       multiline_datastructure_prompt =
       [ [('>>> a = [1,','a = [1,'),
          ('... 2]','2]'),
         ],
       ],
        
       multiline_datastructure =
       [ [('b = ("%s"', None),
          ('# comment', None),
          ('%foo )', 'b = ("%s"\n# comment\n%foo )'),
         ],
       ],
       
       multiline_string =
       [ [("'''foo?", None),
          ("bar'''", "'''foo?\nbar'''"),
         ],
       ],
       
       leading_indent =
       [ [('    print "hi"','print "hi"'),
          ],
         [('  for a in range(5):','for a in range(5):'),
          ('    a*2','  a*2'),
          ],
         [('    a="""','a="""'),
          ('    123"""','123"""'),
           ],
         [('a="""','a="""'),
          ('    123"""','    123"""'),
          ],
       ],
       
       cellmagic =
       [ [('%%foo a', None),
          (None, "get_ipython().run_cell_magic('foo', 'a', '')"),
          ],
         [('%%bar 123', None),
          ('hello', None),
          (None , "get_ipython().run_cell_magic('bar', '123', 'hello')"),
          ],
         [('a=5', 'a=5'),
          ('%%cellmagic', '%%cellmagic'),
          ],
       ],
       
       escaped =
       [ [('%abc def \\', None),
          ('ghi', "get_ipython().run_line_magic('abc', 'def ghi')"),
          ],
         [('%abc def \\', None),
          ('ghi\\', None),
          (None, "get_ipython().run_line_magic('abc', 'def ghi')"),
          ],
       ],
       
       assign_magic =
       [ [('a = %bc de \\', None),
          ('fg', "a = get_ipython().run_line_magic('bc', 'de fg')"),
          ],
         [('a = %bc de \\', None),
          ('fg\\', None),
          (None, "a = get_ipython().run_line_magic('bc', 'de fg')"),
          ],
       ],
       
       assign_system =
       [ [('a = !bc de \\', None),
          ('fg', "a = get_ipython().getoutput('bc de fg')"),
          ],
         [('a = !bc de \\', None),
          ('fg\\', None),
          (None, "a = get_ipython().getoutput('bc de fg')"),
          ],
       ],
       )


def test_assign_system():
    tt.check_pairs(transform_and_reset(ipt.assign_from_system), syntax['assign_system'])

def test_assign_magic():
    tt.check_pairs(transform_and_reset(ipt.assign_from_magic), syntax['assign_magic'])

def test_classic_prompt():
    tt.check_pairs(transform_and_reset(ipt.classic_prompt), syntax['classic_prompt'])
    for example in syntax_ml['classic_prompt']:
        transform_checker(example, ipt.classic_prompt)
    for example in syntax_ml['multiline_datastructure_prompt']:
        transform_checker(example, ipt.classic_prompt)

    # Check that we don't transform the second line if the first is obviously
    # IPython syntax
    transform_checker([
        ('%foo', '%foo'),
        ('>>> bar', '>>> bar'),
    ], ipt.classic_prompt)


def test_ipy_prompt():
    tt.check_pairs(transform_and_reset(ipt.ipy_prompt), syntax['ipy_prompt'])
    for example in syntax_ml['ipy_prompt']:
        transform_checker(example, ipt.ipy_prompt)

    # Check that we don't transform the second line if we're inside a cell magic
    transform_checker([
        ('%%foo', '%%foo'),
        ('In [1]: bar', 'In [1]: bar'),
    ], ipt.ipy_prompt)

def test_assemble_logical_lines():
    tests = \
    [ [("a = \\", None),
       ("123", "a = 123"),
      ],
      [("a = \\", None),  # Test resetting when within a multi-line string
       ("12 *\\", None),
       (None, "a = 12 *"),
      ],
      [("# foo\\", "# foo\\"), # Comments can't be continued like this
      ],
    ]
    for example in tests:
        transform_checker(example, ipt.assemble_logical_lines)

def test_assemble_python_lines():
    tests = \
    [ [("a = '''", None),
       ("abc'''", "a = '''\nabc'''"),
      ],
      [("a = '''", None),  # Test resetting when within a multi-line string
       ("def", None),
       (None, "a = '''\ndef"),
      ],
      [("a = [1,", None),
       ("2]", "a = [1,\n2]"),
      ],
      [("a = [1,", None),  # Test resetting when within a multi-line string
       ("2,", None),
       (None, "a = [1,\n2,"),
      ],
      [("a = '''", None),  # Test line continuation within a multi-line string
       ("abc\\", None),
       ("def", None),
       ("'''", "a = '''\nabc\\\ndef\n'''"),
      ],
    ] + syntax_ml['multiline_datastructure']
    for example in tests:
        transform_checker(example, ipt.assemble_python_lines)


def test_help_end():
    tt.check_pairs(transform_and_reset(ipt.help_end), syntax['end_help'])

def test_escaped_noesc():
    tt.check_pairs(transform_and_reset(ipt.escaped_commands), syntax['escaped_noesc'])


def test_escaped_shell():
    tt.check_pairs(transform_and_reset(ipt.escaped_commands), syntax['escaped_shell'])


def test_escaped_help():
    tt.check_pairs(transform_and_reset(ipt.escaped_commands), syntax['escaped_help'])


def test_escaped_magic():
    tt.check_pairs(transform_and_reset(ipt.escaped_commands), syntax['escaped_magic'])


def test_escaped_quote():
    tt.check_pairs(transform_and_reset(ipt.escaped_commands), syntax['escaped_quote'])


def test_escaped_quote2():
    tt.check_pairs(transform_and_reset(ipt.escaped_commands), syntax['escaped_quote2'])


def test_escaped_paren():
    tt.check_pairs(transform_and_reset(ipt.escaped_commands), syntax['escaped_paren'])


def test_cellmagic():
    for example in syntax_ml['cellmagic']:
        transform_checker(example, ipt.cellmagic)
    
    line_example = [('%%bar 123', None),
                    ('hello', None),
                    ('' , "get_ipython().run_cell_magic('bar', '123', 'hello')"),
                   ]
    transform_checker(line_example, ipt.cellmagic, end_on_blank_line=True)

def test_has_comment():
    tests = [('text', False),
             ('text #comment', True),
             ('text #comment\n', True),
             ('#comment', True),
             ('#comment\n', True),
             ('a = "#string"', False),
             ('a = "#string" # comment', True),
             ('a #comment not "string"', True),
             ]
    tt.check_pairs(ipt.has_comment, tests)

@ipt.TokenInputTransformer.wrap
def decistmt(tokens):
    """Substitute Decimals for floats in a string of statements.

    Based on an example from the tokenize module docs.
    """
    result = []
    for toknum, tokval, _, _, _  in tokens:
        if toknum == tokenize.NUMBER and '.' in tokval:  # replace NUMBER tokens
            yield from [
                (tokenize.NAME, 'Decimal'),
                (tokenize.OP, '('),
                (tokenize.STRING, repr(tokval)),
                (tokenize.OP, ')')
            ]
        else:
            yield (toknum, tokval)



def test_token_input_transformer():
    tests = [('1.2', "Decimal ('1.2')"),
             ('"1.2"', '"1.2"'),
             ]
    tt.check_pairs(transform_and_reset(decistmt), tests)
    ml_tests = \
    [ [("a = 1.2; b = '''x", None),
       ("y'''", "a =Decimal ('1.2');b ='''x\ny'''"),
      ],
      [("a = [1.2,", None),
       ("3]", "a =[Decimal ('1.2'),\n3 ]"),
      ],
      [("a = '''foo", None),  # Test resetting when within a multi-line string
       ("bar", None),
       (None, "a = '''foo\nbar"),
      ],
    ]
    for example in ml_tests:
        transform_checker(example, decistmt)