IPython/core/tests/test_completer.py

"""Tests for the IPython tab-completion machinery.
"""
#-----------------------------------------------------------------------------
# Module imports
#-----------------------------------------------------------------------------

# stdlib
import os
import sys
import unittest

# third party
import nose.tools as nt

# our own packages
from IPython.config.loader import Config
from IPython.core import completer
from IPython.external.decorators import knownfailureif
from IPython.utils.tempdir import TemporaryDirectory
from IPython.utils.generics import complete_object
from IPython.utils import py3compat
from IPython.utils.py3compat import string_types, unicode_type
from IPython.testing import decorators as dec

#-----------------------------------------------------------------------------
# Test functions
#-----------------------------------------------------------------------------
def test_protect_filename():
    pairs = [ ('abc','abc'),
              (' abc',r'\ abc'),
              ('a bc',r'a\ bc'),
              ('a  bc',r'a\ \ bc'),
              ('  bc',r'\ \ bc'),
              ]
    # On posix, we also protect parens and other special characters
    if sys.platform != 'win32':
        pairs.extend( [('a(bc',r'a\(bc'),
                       ('a)bc',r'a\)bc'),
                       ('a( )bc',r'a\(\ \)bc'),
                       ('a[1]bc', r'a\[1\]bc'),
                       ('a{1}bc', r'a\{1\}bc'),
                       ('a#bc', r'a\#bc'),
                       ('a?bc', r'a\?bc'),
                       ('a=bc', r'a\=bc'),
                       ('a\\bc', r'a\\bc'),
                       ('a|bc', r'a\|bc'),
                       ('a;bc', r'a\;bc'),
                       ('a:bc', r'a\:bc'),
                       ("a'bc", r"a\'bc"),
                       ('a*bc', r'a\*bc'),
                       ('a"bc', r'a\"bc'),
                       ('a^bc', r'a\^bc'),
                       ('a&bc', r'a\&bc'),
                       ] )
    # run the actual tests
    for s1, s2 in pairs:
        s1p = completer.protect_filename(s1)
        nt.assert_equal(s1p, s2)


def check_line_split(splitter, test_specs):
    for part1, part2, split in test_specs:
        cursor_pos = len(part1)
        line = part1+part2
        out = splitter.split_line(line, cursor_pos)
        nt.assert_equal(out, split)


def test_line_split():
    """Basic line splitter test with default specs."""
    sp = completer.CompletionSplitter()
    # The format of the test specs is: part1, part2, expected answer.  Parts 1
    # and 2 are joined into the 'line' sent to the splitter, as if the cursor
    # was at the end of part1.  So an empty part2 represents someone hitting
    # tab at the end of the line, the most common case.
    t = [('run some/scrip', '', 'some/scrip'),
         ('run scripts/er', 'ror.py foo', 'scripts/er'),
         ('echo $HOM', '', 'HOM'),
         ('print sys.pa', '', 'sys.pa'),
         ('print(sys.pa', '', 'sys.pa'),
         ("execfile('scripts/er", '', 'scripts/er'),
         ('a[x.', '', 'x.'),
         ('a[x.', 'y', 'x.'),
         ('cd "some_file/', '', 'some_file/'),
         ]
    check_line_split(sp, t)
    # Ensure splitting works OK with unicode by re-running the tests with
    # all inputs turned into unicode
    check_line_split(sp, [ map(unicode_type, p) for p in t] )


def test_custom_completion_error():
    """Test that errors from custom attribute completers are silenced."""
    ip = get_ipython()
    class A(object): pass
    ip.user_ns['a'] = A()
    
    @complete_object.when_type(A)
    def complete_A(a, existing_completions):
        raise TypeError("this should be silenced")
    
    ip.complete("a.")


def test_unicode_completions():
    ip = get_ipython()
    # Some strings that trigger different types of completion.  Check them both
    # in str and unicode forms
    s = ['ru', '%ru', 'cd /', 'floa', 'float(x)/']
    for t in s + list(map(unicode_type, s)):
        # We don't need to check exact completion values (they may change
        # depending on the state of the namespace, but at least no exceptions
        # should be thrown and the return value should be a pair of text, list
        # values.
        text, matches = ip.complete(t)
        nt.assert_true(isinstance(text, string_types))
        nt.assert_true(isinstance(matches, list))


class CompletionSplitterTestCase(unittest.TestCase):
    def setUp(self):
        self.sp = completer.CompletionSplitter()

    def test_delim_setting(self):
        self.sp.delims = ' '
        nt.assert_equal(self.sp.delims, ' ')
        nt.assert_equal(self.sp._delim_expr, '[\ ]')

    def test_spaces(self):
        """Test with only spaces as split chars."""
        self.sp.delims = ' '
        t = [('foo', '', 'foo'),
             ('run foo', '', 'foo'),
             ('run foo', 'bar', 'foo'),
             ]
        check_line_split(self.sp, t)


def test_has_open_quotes1():
    for s in ["'", "'''", "'hi' '"]:
        nt.assert_equal(completer.has_open_quotes(s), "'")


def test_has_open_quotes2():
    for s in ['"', '"""', '"hi" "']:
        nt.assert_equal(completer.has_open_quotes(s), '"')


def test_has_open_quotes3():
    for s in ["''", "''' '''", "'hi' 'ipython'"]:
        nt.assert_false(completer.has_open_quotes(s))


def test_has_open_quotes4():
    for s in ['""', '""" """', '"hi" "ipython"']:
        nt.assert_false(completer.has_open_quotes(s))


@knownfailureif(sys.platform == 'win32', "abspath completions fail on Windows")
def test_abspath_file_completions():
    ip = get_ipython()
    with TemporaryDirectory() as tmpdir:
        prefix = os.path.join(tmpdir, 'foo')
        suffixes = ['1', '2']
        names = [prefix+s for s in suffixes]
        for n in names:
            open(n, 'w').close()

        # Check simple completion
        c = ip.complete(prefix)[1]
        nt.assert_equal(c, names)

        # Now check with a function call
        cmd = 'a = f("%s' % prefix
        c = ip.complete(prefix, cmd)[1]
        comp = [prefix+s for s in suffixes]
        nt.assert_equal(c, comp)


def test_local_file_completions():
    ip = get_ipython()
    cwd = py3compat.getcwd()
    try:
        with TemporaryDirectory() as tmpdir:
            os.chdir(tmpdir)
            prefix = './foo'
            suffixes = ['1', '2']
            names = [prefix+s for s in suffixes]
            for n in names:
                open(n, 'w').close()

            # Check simple completion
            c = ip.complete(prefix)[1]
            nt.assert_equal(c, names)

            # Now check with a function call
            cmd = 'a = f("%s' % prefix
            c = ip.complete(prefix, cmd)[1]
            comp = [prefix+s for s in suffixes]
            nt.assert_equal(c, comp)
    finally:
        # prevent failures from making chdir stick
        os.chdir(cwd)


def test_greedy_completions():
    ip = get_ipython()
    greedy_original = ip.Completer.greedy
    try:
        ip.Completer.greedy = False
        ip.ex('a=list(range(5))')
        _,c = ip.complete('.',line='a[0].')
        nt.assert_false('a[0].real' in c,
                        "Shouldn't have completed on a[0]: %s"%c)
        ip.Completer.greedy = True
        _,c = ip.complete('.',line='a[0].')
        nt.assert_true('a[0].real' in c, "Should have completed on a[0]: %s"%c)
    finally:
        ip.Completer.greedy = greedy_original


def test_omit__names():
    # also happens to test IPCompleter as a configurable
    ip = get_ipython()
    ip._hidden_attr = 1
    c = ip.Completer
    ip.ex('ip=get_ipython()')
    cfg = Config()
    cfg.IPCompleter.omit__names = 0
    c.update_config(cfg)
    s,matches = c.complete('ip.')
    nt.assert_in('ip.__str__', matches)
    nt.assert_in('ip._hidden_attr', matches)
    cfg.IPCompleter.omit__names = 1
    c.update_config(cfg)
    s,matches = c.complete('ip.')
    nt.assert_not_in('ip.__str__', matches)
    nt.assert_in('ip._hidden_attr', matches)
    cfg.IPCompleter.omit__names = 2
    c.update_config(cfg)
    s,matches = c.complete('ip.')
    nt.assert_not_in('ip.__str__', matches)
    nt.assert_not_in('ip._hidden_attr', matches)
    del ip._hidden_attr


def test_limit_to__all__False_ok():
    ip = get_ipython()
    c = ip.Completer
    ip.ex('class D: x=24')
    ip.ex('d=D()')
    cfg = Config()
    cfg.IPCompleter.limit_to__all__ = False
    c.update_config(cfg)
    s, matches = c.complete('d.')
    nt.assert_in('d.x', matches)


def test_limit_to__all__True_ok():
    ip = get_ipython()
    c = ip.Completer
    ip.ex('class D: x=24')
    ip.ex('d=D()')
    ip.ex("d.__all__=['z']")
    cfg = Config()
    cfg.IPCompleter.limit_to__all__ = True
    c.update_config(cfg)
    s, matches = c.complete('d.')
    nt.assert_in('d.z', matches)
    nt.assert_not_in('d.x', matches)


def test_get__all__entries_ok():
    class A(object):
        __all__ = ['x', 1]
    words = completer.get__all__entries(A())
    nt.assert_equal(words, ['x'])


def test_get__all__entries_no__all__ok():
    class A(object):
        pass
    words = completer.get__all__entries(A())
    nt.assert_equal(words, [])


def test_func_kw_completions():
    ip = get_ipython()
    c = ip.Completer
    ip.ex('def myfunc(a=1,b=2): return a+b')
    s, matches = c.complete(None, 'myfunc(1,b')
    nt.assert_in('b=', matches)
    # Simulate completing with cursor right after b (pos==10):
    s, matches = c.complete(None, 'myfunc(1,b)', 10)
    nt.assert_in('b=', matches)
    s, matches = c.complete(None, 'myfunc(a="escaped\\")string",b')
    nt.assert_in('b=', matches)
    #builtin function
    s, matches = c.complete(None, 'min(k, k')
    nt.assert_in('key=', matches)


def test_default_arguments_from_docstring():
    doc = min.__doc__
    ip = get_ipython()
    c = ip.Completer
    kwd = c._default_arguments_from_docstring(
        'min(iterable[, key=func]) -> value')
    nt.assert_equal(kwd, ['key'])
    #with cython type etc
    kwd = c._default_arguments_from_docstring(
        'Minuit.migrad(self, int ncall=10000, resume=True, int nsplit=1)\n')
    nt.assert_equal(kwd, ['ncall', 'resume', 'nsplit'])
    #white spaces
    kwd = c._default_arguments_from_docstring(
        '\n Minuit.migrad(self, int ncall=10000, resume=True, int nsplit=1)\n')
    nt.assert_equal(kwd, ['ncall', 'resume', 'nsplit'])

def test_line_magics():
    ip = get_ipython()
    c = ip.Completer
    s, matches = c.complete(None, 'lsmag')
    nt.assert_in('%lsmagic', matches)
    s, matches = c.complete(None, '%lsmag')
    nt.assert_in('%lsmagic', matches)


def test_cell_magics():
    from IPython.core.magic import register_cell_magic

    @register_cell_magic
    def _foo_cellm(line, cell):
        pass
    
    ip = get_ipython()
    c = ip.Completer

    s, matches = c.complete(None, '_foo_ce')
    nt.assert_in('%%_foo_cellm', matches)
    s, matches = c.complete(None, '%%_foo_ce')
    nt.assert_in('%%_foo_cellm', matches)


def test_line_cell_magics():
    from IPython.core.magic import register_line_cell_magic

    @register_line_cell_magic
    def _bar_cellm(line, cell):
        pass
    
    ip = get_ipython()
    c = ip.Completer

    # The policy here is trickier, see comments in completion code.  The
    # returned values depend on whether the user passes %% or not explicitly,
    # and this will show a difference if the same name is both a line and cell
    # magic.
    s, matches = c.complete(None, '_bar_ce')
    nt.assert_in('%_bar_cellm', matches)
    nt.assert_in('%%_bar_cellm', matches)
    s, matches = c.complete(None, '%_bar_ce')
    nt.assert_in('%_bar_cellm', matches)
    nt.assert_in('%%_bar_cellm', matches)
    s, matches = c.complete(None, '%%_bar_ce')
    nt.assert_not_in('%_bar_cellm', matches)
    nt.assert_in('%%_bar_cellm', matches)


def test_magic_completion_order():

    ip = get_ipython()
    c = ip.Completer

    # Test ordering of magics and non-magics with the same name
    # We want the non-magic first

    # Before importing matplotlib, there should only be one option:

    text, matches = c.complete('mat')
    nt.assert_equal(matches, ["%matplotlib"])


    ip.run_cell("matplotlib = 1")  # introduce name into namespace

    # After the import, there should be two options, ordered like this:
    text, matches = c.complete('mat')
    nt.assert_equal(matches, ["matplotlib", "%matplotlib"])


    ip.run_cell("timeit = 1")  # define a user variable called 'timeit'

    # Order of user variable and line and cell magics with same name:
    text, matches = c.complete('timeit')
    nt.assert_equal(matches, ["timeit", "%timeit","%%timeit"])



def test_dict_key_completion_string():
    """Test dictionary key completion for string keys"""
    ip = get_ipython()
    complete = ip.Completer.complete

    ip.user_ns['d'] = {'abc': None}

    # check completion at different stages
    _, matches = complete(line_buffer="d[")
    nt.assert_in("'abc']", matches)

    _, matches = complete(line_buffer="d['")
    nt.assert_in("abc']", matches)

    _, matches = complete(line_buffer="d['a")
    nt.assert_in("abc']", matches)

    # check use of different quoting
    _, matches = complete(line_buffer="d[\"")
    nt.assert_in("abc\"]", matches)

    _, matches = complete(line_buffer="d[\"a")
    nt.assert_in("abc\"]", matches)

    # check sensitivity to following context
    _, matches = complete(line_buffer="d[]", cursor_pos=2)
    nt.assert_in("'abc'", matches)

    _, matches = complete(line_buffer="d['']", cursor_pos=3)
    nt.assert_in("abc", matches)

    # check multiple solutions are correctly returned and that noise is not
    ip.user_ns['d'] = {'abc': None, 'abd': None, 'bad': None, object(): None,
                       5: None}

    _, matches = complete(line_buffer="d['a")
    nt.assert_in("abc']", matches)
    nt.assert_in("abd']", matches)
    nt.assert_not_in("bad']", matches)

    # check escaping and whitespace
    ip.user_ns['d'] = {'a\nb': None, 'a\'b': None, 'a"b': None, 'a word': None}
    _, matches = complete(line_buffer="d['a")
    nt.assert_in("a\\nb']", matches)
    nt.assert_in("a\\'b']", matches)
    nt.assert_in("a\"b']", matches)
    nt.assert_in("a word']", matches)

    # - can complete on non-initial word of the string
    _, matches = complete(line_buffer="d['a w")
    nt.assert_in("word']", matches)

    # - understands quote escaping
    _, matches = complete(line_buffer="d['a\\'")
    nt.assert_in("b']", matches)

    # - default quoting should work like repr
    _, matches = complete(line_buffer="d[")
    nt.assert_in("\"a'b\"]", matches)

    # - when opening quote with ", possible to match with unescaped apostrophe
    _, matches = complete(line_buffer="d[\"a'")
    nt.assert_in("b\"]", matches)


def test_dict_key_completion_contexts():
    """Test expression contexts in which dict key completion occurs"""
    ip = get_ipython()
    complete = ip.Completer.complete
    d = {'abc': None}
    ip.user_ns['d'] = d

    class C:
        data = d
    ip.user_ns['C'] = C
    ip.user_ns['get'] = lambda: d

    def assert_no_completion(**kwargs):
        _, matches = complete(**kwargs)
        nt.assert_not_in('abc', matches)
        nt.assert_not_in('abc\'', matches)
        nt.assert_not_in('abc\']', matches)
        nt.assert_not_in('\'abc\'', matches)
        nt.assert_not_in('\'abc\']', matches)

    def assert_completion(**kwargs):
        _, matches = complete(**kwargs)
        nt.assert_in("'abc']", matches)

    # no completion after string closed, even if reopened
    ip.Completer.greedy = False
    assert_no_completion(line_buffer="d['a'")
    assert_no_completion(line_buffer="d[\"a\"")
    assert_no_completion(line_buffer="d['a' + ")
    assert_no_completion(line_buffer="d['a' + '")

    # completion in non-trivial expressions
    assert_completion(line_buffer="+ d[")
    assert_completion(line_buffer="(d[")
    assert_completion(line_buffer="C.data[")

    # greedy flag
    assert_no_completion(line_buffer="get()[")
    ip.Completer.greedy = True
    assert_completion(line_buffer="get()[")



@dec.onlyif(sys.version_info[0] >= 3, 'This test only applies in Py>=3')
def test_dict_key_completion_bytes():
    """Test handling of bytes in dict key completion"""
    ip = get_ipython()
    complete = ip.Completer.complete

    ip.user_ns['d'] = {'abc': None, b'abd': None}

    _, matches = complete(line_buffer="d[")
    nt.assert_in("'abc']", matches)
    nt.assert_in("b'abd']", matches)

    if False:  # not currently implemented
        _, matches = complete(line_buffer="d[b")
        nt.assert_in("b'abd']", matches)
        nt.assert_not_in("b'abc']", matches)

        _, matches = complete(line_buffer="d[b'")
        nt.assert_in("abd']", matches)
        nt.assert_not_in("abc']", matches)

        _, matches = complete(line_buffer="d[B'")
        nt.assert_in("abd']", matches)
        nt.assert_not_in("abc']", matches)

        _, matches = complete(line_buffer="d['")
        nt.assert_in("abc']", matches)
        nt.assert_not_in("abd']", matches)


@dec.onlyif(sys.version_info[0] < 3, 'This test only applies in Py<3')
def test_dict_key_completion_unicode_py2():
    """Test handling of unicode in dict key completion"""
    ip = get_ipython()
    complete = ip.Completer.complete

    ip.user_ns['d'] = {u'abc': None,
                       unicode_type('a\xd7\x90', 'utf8'): None}

    _, matches = complete(line_buffer="d[")
    nt.assert_in("u'abc']", matches)
    nt.assert_in("u'a\\u05d0']", matches)

    _, matches = complete(line_buffer="d['a")
    nt.assert_in("abc']", matches)
    nt.assert_not_in("a\\u05d0']", matches)

    _, matches = complete(line_buffer="d[u'a")
    nt.assert_in("abc']", matches)
    nt.assert_in("a\\u05d0']", matches)

    _, matches = complete(line_buffer="d[U'a")
    nt.assert_in("abc']", matches)
    nt.assert_in("a\\u05d0']", matches)

    # query using escape
    _, matches = complete(line_buffer="d[u'a\\u05d0")
    nt.assert_in("u05d0']", matches)  # tokenized after \\

    # query using character
    _, matches = complete(line_buffer=unicode_type("d[u'a\xd7\x90", 'utf8'))
    nt.assert_in("']", matches)


@dec.onlyif(sys.version_info[0] >= 3, 'This test only applies in Py>=3')
def test_dict_key_completion_unicode_py3():
    """Test handling of unicode in dict key completion"""
    ip = get_ipython()
    complete = ip.Completer.complete

    ip.user_ns['d'] = {unicode_type(b'a\xd7\x90', 'utf8'): None}

    # query using escape
    _, matches = complete(line_buffer="d['a\\u05d0")
    nt.assert_in("u05d0']", matches)  # tokenized after \\

    # query using character
    _, matches = complete(line_buffer=unicode_type(b"d['a\xd7\x90", 'utf8'))
    nt.assert_in(unicode_type(b"a\xd7\x90']", 'utf8'), matches)


@dec.skip_without('numpy')
def test_struct_array_key_completion():
    """Test dict key completion applies to numpy struct arrays"""
    import numpy
    ip = get_ipython()
    complete = ip.Completer.complete
    ip.user_ns['d'] = numpy.array([], dtype=[('hello', 'f'), ('world', 'f')])
    _, matches = complete(line_buffer="d['")
    nt.assert_in("hello']", matches)
    nt.assert_in("world']", matches)


@dec.skip_without('pandas')
def test_dataframe_key_completion():
    """Test dict key completion applies to pandas DataFrames"""
    import pandas
    ip = get_ipython()
    complete = ip.Completer.complete
    ip.user_ns['d'] = pandas.DataFrame({'hello': [1], 'world': [2]})
    _, matches = complete(line_buffer="d['")
    nt.assert_in("hello']", matches)
    nt.assert_in("world']", matches)


def test_dict_key_completion_invalids():
    """Smoke test cases dict key completion can't handle"""
    ip = get_ipython()
    complete = ip.Completer.complete

    ip.user_ns['no_getitem'] = None
    ip.user_ns['no_keys'] = []
    ip.user_ns['cant_call_keys'] = dict
    ip.user_ns['empty'] = {}
    ip.user_ns['d'] = {'abc': 5}

    _, matches = complete(line_buffer="no_getitem['")
    _, matches = complete(line_buffer="no_keys['")
    _, matches = complete(line_buffer="cant_call_keys['")
    _, matches = complete(line_buffer="empty['")
    _, matches = complete(line_buffer="name_error['")
    _, matches = complete(line_buffer="d['\\")  # incomplete escape