IPython/core/tests/test_inputsplitter.py

# -*- coding: utf-8 -*-
"""Tests for the inputsplitter module.

Authors
-------
* Fernando Perez
* Robert Kern
"""
#-----------------------------------------------------------------------------
#  Copyright (C) 2010  The IPython Development Team
#
#  Distributed under the terms of the BSD License.  The full license is in
#  the file COPYING, distributed as part of this software.
#-----------------------------------------------------------------------------

#-----------------------------------------------------------------------------
# Imports
#-----------------------------------------------------------------------------
# stdlib
import unittest
import sys

# Third party
import nose.tools as nt

# Our own
from IPython.core import inputsplitter as isp

#-----------------------------------------------------------------------------
# Semi-complete examples (also used as tests)
#-----------------------------------------------------------------------------

# Note: at the bottom, there's a slightly more complete version of this that
# can be useful during development of code here.

def mini_interactive_loop(input_func):
    """Minimal example of the logic of an interactive interpreter loop.

    This serves as an example, and it is used by the test system with a fake
    raw_input that simulates interactive input."""

    from IPython.core.inputsplitter import InputSplitter
    
    isp = InputSplitter()
    # In practice, this input loop would be wrapped in an outside loop to read
    # input indefinitely, until some exit/quit command was issued.  Here we
    # only illustrate the basic inner loop.
    while isp.push_accepts_more():
        indent = ' '*isp.indent_spaces
        prompt = '>>> ' + indent
        line = indent + input_func(prompt)
        isp.push(line)

    # Here we just return input so we can use it in a test suite, but a real
    # interpreter would instead send it for execution somewhere.
    src = isp.source_reset()
    #print 'Input source was:\n', src  # dbg
    return src

#-----------------------------------------------------------------------------
# Test utilities, just for local use
#-----------------------------------------------------------------------------

def assemble(block):
    """Assemble a block into multi-line sub-blocks."""
    return ['\n'.join(sub_block)+'\n' for sub_block in block]


def pseudo_input(lines):
    """Return a function that acts like raw_input but feeds the input list."""
    ilines = iter(lines)
    def raw_in(prompt):
        try:
            return next(ilines)
        except StopIteration:
            return ''
    return raw_in

#-----------------------------------------------------------------------------
# Tests
#-----------------------------------------------------------------------------
def test_spaces():
    tests = [('', 0),
             (' ', 1),
             ('\n', 0),
             (' \n', 1),
             ('x', 0),
             (' x', 1),
             ('  x',2),
             ('    x',4),
             # Note: tabs are counted as a single whitespace!
             ('\tx', 1),
             ('\t x', 2),
             ]
    
    for s, nsp in tests:
        nt.assert_equal(isp.num_ini_spaces(s), nsp)


def test_remove_comments():
    tests = [('text', 'text'),
             ('text # comment', 'text '),
             ('text # comment\n', 'text \n'),
             ('text # comment \n', 'text \n'),
             ('line # c \nline\n','line \nline\n'),
             ('line # c \nline#c2  \nline\nline #c\n\n',
              'line \nline\nline\nline \n\n'),
             ]

    for inp, out in tests:
        nt.assert_equal(isp.remove_comments(inp), out)


def test_get_input_encoding():
    encoding = isp.get_input_encoding()
    nt.assert_true(isinstance(encoding, basestring))
    # simple-minded check that at least encoding a simple string works with the
    # encoding we got.
    nt.assert_equal('test'.encode(encoding), 'test')


class NoInputEncodingTestCase(unittest.TestCase):
    def setUp(self):
        self.old_stdin = sys.stdin
        class X: pass
        fake_stdin = X()
        sys.stdin = fake_stdin
        
    def test(self):
        # Verify that if sys.stdin has no 'encoding' attribute we do the right
        # thing
        enc = isp.get_input_encoding()
        self.assertEqual(enc, 'ascii')
        
    def tearDown(self):
        sys.stdin = self.old_stdin


class InputSplitterTestCase(unittest.TestCase):
    def setUp(self):
        self.isp = isp.InputSplitter()

    def test_reset(self):
        isp = self.isp
        isp.push('x=1')
        isp.reset()
        self.assertEqual(isp._buffer, [])
        self.assertEqual(isp.indent_spaces, 0)
        self.assertEqual(isp.source, '')
        self.assertEqual(isp.code, None)
        self.assertEqual(isp._is_complete, False)

    def test_source(self):
        self.isp._store('1')
        self.isp._store('2')
        self.assertEqual(self.isp.source, '1\n2\n')
        self.assertTrue(len(self.isp._buffer)>0)
        self.assertEqual(self.isp.source_reset(), '1\n2\n')
        self.assertEqual(self.isp._buffer, [])
        self.assertEqual(self.isp.source, '')
        
    def test_indent(self):
        isp = self.isp # shorthand
        isp.push('x=1')
        self.assertEqual(isp.indent_spaces, 0)
        isp.push('if 1:\n    x=1')
        self.assertEqual(isp.indent_spaces, 4)
        isp.push('y=2\n')
        self.assertEqual(isp.indent_spaces, 0)

    def test_indent2(self):
        # In cell mode, inputs must be fed in whole blocks, so skip this test
        if self.isp.input_mode == 'cell': return

        isp = self.isp
        isp.push('if 1:')
        self.assertEqual(isp.indent_spaces, 4)
        isp.push('    x=1')
        self.assertEqual(isp.indent_spaces, 4)
        # Blank lines shouldn't change the indent level
        isp.push(' '*2)
        self.assertEqual(isp.indent_spaces, 4)

    def test_indent3(self):
        # In cell mode, inputs must be fed in whole blocks, so skip this test
        if self.isp.input_mode == 'cell': return

        isp = self.isp
        # When a multiline statement contains parens or multiline strings, we
        # shouldn't get confused.
        isp.push("if 1:")
        isp.push("    x = (1+\n    2)")
        self.assertEqual(isp.indent_spaces, 4)

    def test_dedent(self):
        isp = self.isp # shorthand
        isp.push('if 1:')
        self.assertEqual(isp.indent_spaces, 4)
        isp.push('    pass')
        self.assertEqual(isp.indent_spaces, 0)
        
    def test_push(self):
        isp = self.isp
        self.assertTrue(isp.push('x=1'))

    def test_push2(self):
        isp = self.isp
        self.assertFalse(isp.push('if 1:'))
        for line in ['  x=1', '# a comment', '  y=2']:
            self.assertTrue(isp.push(line))
            
    def test_replace_mode(self):
        isp = self.isp
        isp.input_mode = 'cell'
        isp.push('x=1')
        self.assertEqual(isp.source, 'x=1\n')
        isp.push('x=2')
        self.assertEqual(isp.source, 'x=2\n')

    def test_push_accepts_more(self):
        isp = self.isp
        isp.push('x=1')
        self.assertFalse(isp.push_accepts_more())

    def test_push_accepts_more2(self):
        # In cell mode, inputs must be fed in whole blocks, so skip this test
        if self.isp.input_mode == 'cell': return

        isp = self.isp
        isp.push('if 1:')
        self.assertTrue(isp.push_accepts_more())
        isp.push('  x=1')
        self.assertTrue(isp.push_accepts_more())
        isp.push('')
        self.assertFalse(isp.push_accepts_more())
        
    def test_push_accepts_more3(self):
        isp = self.isp
        isp.push("x = (2+\n3)")
        self.assertFalse(isp.push_accepts_more())

    def test_push_accepts_more4(self):
        # In cell mode, inputs must be fed in whole blocks, so skip this test
        if self.isp.input_mode == 'cell': return

        isp = self.isp
        # When a multiline statement contains parens or multiline strings, we
        # shouldn't get confused.
        # FIXME: we should be able to better handle de-dents in statements like
        # multiline strings and multiline expressions (continued with \ or
        # parens).  Right now we aren't handling the indentation tracking quite
        # correctly with this, though in practice it may not be too much of a
        # problem.  We'll need to see.
        isp.push("if 1:")
        isp.push("    x = (2+")
        isp.push("    3)")
        self.assertTrue(isp.push_accepts_more())
        isp.push("    y = 3")
        self.assertTrue(isp.push_accepts_more())
        isp.push('')
        self.assertFalse(isp.push_accepts_more())
        
    def test_push_accepts_more5(self):
        # In cell mode, inputs must be fed in whole blocks, so skip this test
        if self.isp.input_mode == 'cell': return

        isp = self.isp
        isp.push('try:')
        isp.push('    a = 5')
        isp.push('except:')
        isp.push('    raise')
        self.assertTrue(isp.push_accepts_more())

    def test_continuation(self):
        isp = self.isp
        isp.push("import os, \\")
        self.assertTrue(isp.push_accepts_more())
        isp.push("sys")
        self.assertFalse(isp.push_accepts_more())

    def test_syntax_error(self):
        isp = self.isp
        # Syntax errors immediately produce a 'ready' block, so the invalid
        # Python can be sent to the kernel for evaluation with possible ipython
        # special-syntax conversion.
        isp.push('run foo')
        self.assertFalse(isp.push_accepts_more())

    def check_split(self, block_lines, compile=True):
        blocks = assemble(block_lines)
        lines = ''.join(blocks)
        oblock = self.isp.split_blocks(lines)
        self.assertEqual(oblock, blocks)
        if compile:
            for block in blocks:
                self.isp._compile(block)

    def test_split(self):
        # All blocks of input we want to test in a list.  The format for each
        # block is a list of lists, with each inner lists consisting of all the
        # lines (as single-lines) that should make up a sub-block.

        # Note: do NOT put here sub-blocks that don't compile, as the
        # check_split() routine makes a final verification pass to check that
        # each sub_block, as returned by split_blocks(), does compile
        # correctly.
        all_blocks = [ [['x=1']],

                       [['x=1'],
                        ['y=2']],

                       [['x=1',
                         '# a comment'],
                        ['y=11']],

                       [['if 1:',
                         '  x=1'],
                        ['y=3']],

                       [['def f(x):',
                         '  return x'],
                        ['x=1']],

                       [['def f(x):',
                         '  x+=1',
                         '  ',
                         '  return x'],
                        ['x=1']],

                       [['def f(x):',
                         '  if x>0:',
                         '    y=1',
                         '  # a comment',
                         '  else:',
                         '    y=4',
                         ' ',
                         '  return y'],
                        ['x=1'],
                        ['if 1:',
                         '  y=11'] ],
                       
                       [['for i in range(10):'
                         '  x=i**2']],

                       [['for i in range(10):'
                         '  x=i**2'],
                        ['z = 1']],

                       [['"asdf"']],

                       [['"asdf"'],
                        ['10'],
                       ],

                       [['"""foo',
                         'bar"""']],
                       ]
        for block_lines in all_blocks:
            self.check_split(block_lines)
        
    def test_split_syntax_errors(self):
        # Block splitting with invalid syntax
        all_blocks = [ [['a syntax error']],
            
                       [['x=1',
                         'another syntax error']],

                       [['for i in range(10):'
                         '  yet another error']],
                       
                       ]
        for block_lines in all_blocks:
            self.check_split(block_lines, compile=False)

    def test_unicode(self):
        self.isp.push(u"Pérez")
        self.isp.push(u'\xc3\xa9')
        self.isp.push(u"u'\xc3\xa9'")

class InteractiveLoopTestCase(unittest.TestCase):
    """Tests for an interactive loop like a python shell.
    """
    def check_ns(self, lines, ns):
        """Validate that the given input lines produce the resulting namespace.

        Note: the input lines are given exactly as they would be typed in an
        auto-indenting environment, as mini_interactive_loop above already does
        auto-indenting and prepends spaces to the input.
        """
        src = mini_interactive_loop(pseudo_input(lines))
        test_ns = {}
        exec src in test_ns
        # We can't check that the provided ns is identical to the test_ns,
        # because Python fills test_ns with extra keys (copyright, etc).  But
        # we can check that the given dict is *contained* in test_ns
        for k,v in ns.iteritems():
            self.assertEqual(test_ns[k], v)
        
    def test_simple(self):
        self.check_ns(['x=1'], dict(x=1))

    def test_simple2(self):
        self.check_ns(['if 1:', 'x=2'], dict(x=2))

    def test_xy(self):
        self.check_ns(['x=1; y=2'], dict(x=1, y=2))

    def test_abc(self):
        self.check_ns(['if 1:','a=1','b=2','c=3'], dict(a=1, b=2, c=3))
    
    def test_multi(self):
        self.check_ns(['x =(1+','1+','2)'], dict(x=4))
    

def test_LineInfo():
    """Simple test for LineInfo construction and str()"""
    linfo = isp.LineInfo('  %cd /home')
    nt.assert_equals(str(linfo), 'LineInfo [  |%|cd|/home]')


def test_split_user_input():
    """Unicode test - split_user_input already has good doctests"""
    line = u"Pérez Fernando"
    parts = isp.split_user_input(line)
    parts_expected = (u'', u'', u'', line)
    nt.assert_equal(parts, parts_expected)


# Transformer tests
def transform_checker(tests, func):
    """Utility to loop over test inputs"""
    for inp, tr in tests:
        nt.assert_equals(func(inp), tr)

# Data for all the syntax tests in the form of lists of pairs of
# raw/transformed input.  We store it here as a global dict so that we can use
# it both within single-function tests and also to validate the behavior of the
# larger objects

syntax = \
  dict(assign_system =
       [('a =! ls', 'a = get_ipython().getoutput(u"ls")'),
        ('b = !ls', 'b = get_ipython().getoutput(u"ls")'),
        ('x=1', 'x=1'), # normal input is unmodified
        ('    ','    '),  # blank lines are kept intact
        ],

       assign_magic =
       [('a =% who', 'a = get_ipython().magic(u"who")'),
        ('b = %who', 'b = get_ipython().magic(u"who")'),
        ('x=1', 'x=1'), # normal input is unmodified
        ('    ','    '),  # blank lines are kept intact
        ],

       classic_prompt =
       [('>>> x=1', 'x=1'),
        ('x=1', 'x=1'), # normal input is unmodified
        ('    ', '    '),  # blank lines are kept intact
        ('... ', ''), # continuation prompts
        ],

       ipy_prompt =
       [('In [1]: x=1', 'x=1'),
        ('x=1', 'x=1'), # normal input is unmodified
        ('    ','    '),  # blank lines are kept intact
        ('   ....: ', ''), # continuation prompts
        ],

       # Tests for the escape transformer to leave normal code alone
       escaped_noesc =
       [ ('    ', '    '),
         ('x=1', 'x=1'),
         ],

       # System calls
       escaped_shell =
       [ ('!ls', 'get_ipython().system(u"ls")'),
         # Double-escape shell, this means to capture the output of the
         # subprocess and return it
         ('!!ls', 'get_ipython().getoutput(u"ls")'),
         ],

       # Help/object info
       escaped_help =
       [ ('?', 'get_ipython().show_usage()'),
         ('?x1', 'get_ipython().magic(u"pinfo x1")'),
         ('??x2', 'get_ipython().magic(u"pinfo2 x2")'),
         ('x3?', 'get_ipython().magic(u"pinfo x3")'),
         ('x4??', 'get_ipython().magic(u"pinfo2 x4")'),
         ('%hist?', 'get_ipython().magic(u"pinfo %hist")'),
         ('f*?', 'get_ipython().magic(u"psearch f*")'),
         ('ax.*aspe*?', 'get_ipython().magic(u"psearch ax.*aspe*")'),
         ],

       # Explicit magic calls
       escaped_magic =
       [ ('%cd', 'get_ipython().magic(u"cd")'),
         ('%cd /home', 'get_ipython().magic(u"cd /home")'),
         ('    %magic', '    get_ipython().magic(u"magic")'),
         ],
       
       # Quoting with separate arguments
       escaped_quote =
       [ (',f', 'f("")'),
         (',f x', 'f("x")'),
         ('  ,f y', '  f("y")'),
         (',f a b', 'f("a", "b")'),
         ],
       
       # Quoting with single argument
       escaped_quote2 = 
       [ (';f', 'f("")'),
         (';f x', 'f("x")'),
         ('  ;f y', '  f("y")'),
         (';f a b', 'f("a b")'),
         ],
       
       # Simply apply parens
       escaped_paren = 
       [ ('/f', 'f()'),
         ('/f x', 'f(x)'),
         ('  /f y', '  f(y)'),
         ('/f a b', 'f(a, b)'),
         ],

       )

# multiline syntax examples.  Each of these should be a list of lists, with
# each entry itself having pairs of raw/transformed input.  The union (with
# '\n'.join() of the transformed inputs is what the splitter should produce
# when fed the raw lines one at a time via push.
syntax_ml = \
  dict(classic_prompt =
       [ [('>>> for i in range(10):','for i in range(10):'),
          ('...     print i','    print i'),
          ('... ', ''),
          ],
        ],

       ipy_prompt =
       [ [('In [24]: for i in range(10):','for i in range(10):'),
          ('   ....:     print i','    print i'),
          ('   ....: ', ''),
          ],
         ],
       )


def test_assign_system():
    transform_checker(syntax['assign_system'], isp.transform_assign_system)

    
def test_assign_magic():
    transform_checker(syntax['assign_magic'], isp.transform_assign_magic)


def test_classic_prompt():
    transform_checker(syntax['classic_prompt'], isp.transform_classic_prompt)
    for example in syntax_ml['classic_prompt']:
        transform_checker(example, isp.transform_classic_prompt)
    

def test_ipy_prompt():
    transform_checker(syntax['ipy_prompt'], isp.transform_ipy_prompt)
    for example in syntax_ml['ipy_prompt']:
        transform_checker(example, isp.transform_ipy_prompt)
    

def test_escaped_noesc():
    transform_checker(syntax['escaped_noesc'], isp.transform_escaped)


def test_escaped_shell():
    transform_checker(syntax['escaped_shell'], isp.transform_escaped)


def test_escaped_help():
    transform_checker(syntax['escaped_help'], isp.transform_escaped)


def test_escaped_magic():
    transform_checker(syntax['escaped_magic'], isp.transform_escaped)


def test_escaped_quote():
    transform_checker(syntax['escaped_quote'], isp.transform_escaped)


def test_escaped_quote2():
    transform_checker(syntax['escaped_quote2'], isp.transform_escaped)


def test_escaped_paren():
    transform_checker(syntax['escaped_paren'], isp.transform_escaped)


class IPythonInputTestCase(InputSplitterTestCase):
    """By just creating a new class whose .isp is a different instance, we
    re-run the same test battery on the new input splitter.

    In addition, this runs the tests over the syntax and syntax_ml dicts that
    were tested by individual functions, as part of the OO interface.

    It also makes some checks on the raw buffer storage.
    """

    def setUp(self):
        self.isp = isp.IPythonInputSplitter(input_mode='line')

    def test_syntax(self):
        """Call all single-line syntax tests from the main object"""
        isp = self.isp
        for example in syntax.itervalues():
            for raw, out_t in example:
                if raw.startswith(' '):
                    continue
                
                isp.push(raw)
                out, out_raw = isp.source_raw_reset()
                self.assertEqual(out.rstrip(), out_t)
                self.assertEqual(out_raw.rstrip(), raw.rstrip())

    def test_syntax_multiline(self):
        isp = self.isp
        for example in syntax_ml.itervalues():
            out_t_parts = []
            raw_parts = []
            for line_pairs in example:
                for lraw, out_t_part in line_pairs:
                    isp.push(lraw)
                    out_t_parts.append(out_t_part)
                    raw_parts.append(lraw)

                out, out_raw = isp.source_raw_reset()
                out_t = '\n'.join(out_t_parts).rstrip()
                raw = '\n'.join(raw_parts).rstrip()
                self.assertEqual(out.rstrip(), out_t)
                self.assertEqual(out_raw.rstrip(), raw)
                

class BlockIPythonInputTestCase(IPythonInputTestCase):

    # Deactivate tests that don't make sense for the block mode
    test_push3 = test_split = lambda s: None
    
    def setUp(self):
        self.isp = isp.IPythonInputSplitter(input_mode='cell')

    def test_syntax_multiline(self):
        isp = self.isp
        for example in syntax_ml.itervalues():
            raw_parts = []
            out_t_parts = []
            for line_pairs in example:
                for raw, out_t_part in line_pairs:
                    raw_parts.append(raw)
                    out_t_parts.append(out_t_part)

                raw = '\n'.join(raw_parts)
                out_t = '\n'.join(out_t_parts)

                isp.push(raw)
                out, out_raw = isp.source_raw_reset()
                # Match ignoring trailing whitespace
                self.assertEqual(out.rstrip(), out_t.rstrip())
                self.assertEqual(out_raw.rstrip(), raw.rstrip())
    

#-----------------------------------------------------------------------------
# Main - use as a script, mostly for developer experiments
#-----------------------------------------------------------------------------

if __name__ == '__main__':
    # A simple demo for interactive experimentation.  This code will not get
    # picked up by any test suite.
    from IPython.core.inputsplitter import InputSplitter, IPythonInputSplitter

    # configure here the syntax to use, prompt and whether to autoindent
    #isp, start_prompt = InputSplitter(), '>>> '
    isp, start_prompt = IPythonInputSplitter(), 'In> '

    autoindent = True
    #autoindent = False
    
    try:
        while True:
            prompt = start_prompt
            while isp.push_accepts_more():
                indent = ' '*isp.indent_spaces
                if autoindent:
                    line = indent + raw_input(prompt+indent)
                else:
                    line = raw_input(prompt)
                isp.push(line)
                prompt = '... '

            # Here we just return input so we can use it in a test suite, but a
            # real interpreter would instead send it for execution somewhere.
            #src = isp.source; raise EOFError # dbg
            src, raw = isp.source_raw_reset()
            print 'Input source was:\n', src
            print 'Raw source was:\n', raw
    except EOFError:
        print 'Bye'