Merge pull request #5790 from minrk/dict-completion-bug

fix some issues in dict key completion

IPython/core/completer.py

@@ -79,6 +79,7 from IPython.config.configurable import Configurable
 from IPython.core.error import TryNext
 from IPython.core.inputsplitter import ESC_MAGIC
 from IPython.utils import generics
+from IPython.utils import io
 from IPython.utils.dir2 import dir2
 from IPython.utils.process import arg_split
 from IPython.utils.py3compat import builtin_mod, string_types
@@ -429,16 +430,18 def get__all__entries(obj):
 def match_dict_keys(keys, prefix):
     """Used by dict_key_matches, matching the prefix to a list of keys"""
     if not prefix:
-        return None, [repr(k) for k in keys
+        return None, 0, [repr(k) for k in keys
                       if isinstance(k, (string_types, bytes))]
     quote_match = re.search('["\']', prefix)
     quote = quote_match.group()
     try:
         prefix_str = eval(prefix + quote, {})
     except Exception:
-        return None, []
-
-    token_prefix = re.search('\w*$', prefix).group()
+        return None, 0, []
+    
+    token_match = re.search(r'\w*$', prefix, re.UNICODE)
+    token_start = token_match.start()
+    token_prefix = token_match.group()
 
     # TODO: support bytes in Py3k
     matched = []
@@ -471,7 +474,7 def match_dict_keys(keys, prefix):
 
         # then reinsert prefix from start of token
         matched.append('%s%s' % (token_prefix, rem_repr))
-    return quote, matched
+    return quote, token_start, matched
 
 
 def _safe_isinstance(obj, module, class_name):
@@ -919,17 +922,47 class IPCompleter(Completer):
         keys = get_keys(obj)
         if not keys:
             return keys
-        closing_quote, matches = match_dict_keys(keys, prefix)
+        closing_quote, token_offset, matches = match_dict_keys(keys, prefix)
+        if not matches:
+            return matches
+        
+        # get the cursor position of
+        # - the text being completed
+        # - the start of the key text
+        # - the start of the completion
+        text_start = len(self.text_until_cursor) - len(text)
+        if prefix:
+            key_start = match.start(2)
+            completion_start = key_start + token_offset
+        else:
+            key_start = completion_start = match.end()
+        
+        # grab the leading prefix, to make sure all completions start with `text`
+        if text_start > key_start:
+            leading = ''
+        else:
+            leading = text[text_start:completion_start]
+        
+        # the index of the `[` character
+        bracket_idx = match.end(1)
 
         # append closing quote and bracket as appropriate
+        # this is *not* appropriate if the opening quote or bracket is outside
+        # the text given to this method
+        suf = ''
         continuation = self.line_buffer[len(self.text_until_cursor):]
-        if closing_quote and continuation.startswith(closing_quote):
-            suf = ''
-        elif continuation.startswith(']'):
-            suf = closing_quote or ''
-        else:
-            suf = (closing_quote or '') + ']'
-        return [k + suf for k in matches]
+        if key_start > text_start and closing_quote:
+            # quotes were opened inside text, maybe close them
+            if continuation.startswith(closing_quote):
+                continuation = continuation[len(closing_quote):]
+            else:
+                suf += closing_quote
+        if bracket_idx > text_start:
+            # brackets were opened inside text, maybe close them
+            if not continuation.startswith(']'):
+                suf += ']'
+        
+        return [leading + k + suf for k in matches]
 
     def dispatch_custom_completer(self, text):
         #io.rprint("Custom! '%s' %s" % (text, self.custom_completers)) # dbg

IPython/core/tests/test_completer.py

@@ -1,18 +1,17
-"""Tests for the IPython tab-completion machinery.
-"""
-#-----------------------------------------------------------------------------
-# Module imports
-#-----------------------------------------------------------------------------
+# encoding: utf-8
+"""Tests for the IPython tab-completion machinery."""
+
+# Copyright (c) IPython Development Team.
+# Distributed under the terms of the Modified BSD License.
 
-# stdlib
 import os
 import sys
 import unittest
 
-# third party
+from contextlib import contextmanager
+
 import nose.tools as nt
 
-# our own packages
 from IPython.config.loader import Config
 from IPython.core import completer
 from IPython.external.decorators import knownfailureif
@@ -25,6 +24,17 from IPython.testing import decorators as dec
 #-----------------------------------------------------------------------------
 # Test functions
 #-----------------------------------------------------------------------------
+
+@contextmanager
+def greedy_completion():
+    ip = get_ipython()
+    greedy_original = ip.Completer.greedy
+    try:
+        ip.Completer.greedy = True
+        yield
+    finally:
+        ip.Completer.greedy = greedy_original
+
 def test_protect_filename():
     pairs = [ ('abc','abc'),
               (' abc',r'\ abc'),
@@ -205,18 +215,13 def test_local_file_completions():
 
 def test_greedy_completions():
     ip = get_ipython()
-    greedy_original = ip.Completer.greedy
-    try:
-        ip.Completer.greedy = False
-        ip.ex('a=list(range(5))')
-        _,c = ip.complete('.',line='a[0].')
-        nt.assert_false('a[0].real' in c,
-                        "Shouldn't have completed on a[0]: %s"%c)
-        ip.Completer.greedy = True
+    ip.ex('a=list(range(5))')
+    _,c = ip.complete('.',line='a[0].')
+    nt.assert_false('a[0].real' in c,
+                    "Shouldn't have completed on a[0]: %s"%c)
+    with greedy_completion():
         _,c = ip.complete('.',line='a[0].')
         nt.assert_true('a[0].real' in c, "Should have completed on a[0]: %s"%c)
-    finally:
-        ip.Completer.greedy = greedy_original
 
 
 def test_omit__names():
@@ -394,7 +399,6 def test_magic_completion_order():
     nt.assert_equal(matches, ["timeit", "%timeit","%%timeit"])
 
 
-
 def test_dict_key_completion_string():
     """Test dictionary key completion for string keys"""
     ip = get_ipython()
@@ -404,20 +408,25 def test_dict_key_completion_string():
 
     # check completion at different stages
     _, matches = complete(line_buffer="d[")
-    nt.assert_in("'abc']", matches)
+    nt.assert_in("'abc'", matches)
+    nt.assert_not_in("'abc']", matches)
 
     _, matches = complete(line_buffer="d['")
-    nt.assert_in("abc']", matches)
+    nt.assert_in("abc", matches)
+    nt.assert_not_in("abc']", matches)
 
     _, matches = complete(line_buffer="d['a")
-    nt.assert_in("abc']", matches)
+    nt.assert_in("abc", matches)
+    nt.assert_not_in("abc']", matches)
 
     # check use of different quoting
     _, matches = complete(line_buffer="d[\"")
-    nt.assert_in("abc\"]", matches)
+    nt.assert_in("abc", matches)
+    nt.assert_not_in('abc\"]', matches)
 
     _, matches = complete(line_buffer="d[\"a")
-    nt.assert_in("abc\"]", matches)
+    nt.assert_in("abc", matches)
+    nt.assert_not_in('abc\"]', matches)
 
     # check sensitivity to following context
     _, matches = complete(line_buffer="d[]", cursor_pos=2)
@@ -425,39 +434,43 def test_dict_key_completion_string():
 
     _, matches = complete(line_buffer="d['']", cursor_pos=3)
     nt.assert_in("abc", matches)
+    nt.assert_not_in("abc'", matches)
+    nt.assert_not_in("abc']", matches)
 
     # check multiple solutions are correctly returned and that noise is not
     ip.user_ns['d'] = {'abc': None, 'abd': None, 'bad': None, object(): None,
                        5: None}
 
     _, matches = complete(line_buffer="d['a")
-    nt.assert_in("abc']", matches)
-    nt.assert_in("abd']", matches)
-    nt.assert_not_in("bad']", matches)
+    nt.assert_in("abc", matches)
+    nt.assert_in("abd", matches)
+    nt.assert_not_in("bad", matches)
+    assert not any(m.endswith((']', '"', "'")) for m in matches), matches
 
     # check escaping and whitespace
     ip.user_ns['d'] = {'a\nb': None, 'a\'b': None, 'a"b': None, 'a word': None}
     _, matches = complete(line_buffer="d['a")
-    nt.assert_in("a\\nb']", matches)
-    nt.assert_in("a\\'b']", matches)
-    nt.assert_in("a\"b']", matches)
-    nt.assert_in("a word']", matches)
+    nt.assert_in("a\\nb", matches)
+    nt.assert_in("a\\'b", matches)
+    nt.assert_in("a\"b", matches)
+    nt.assert_in("a word", matches)
+    assert not any(m.endswith((']', '"', "'")) for m in matches), matches
 
     # - can complete on non-initial word of the string
     _, matches = complete(line_buffer="d['a w")
-    nt.assert_in("word']", matches)
+    nt.assert_in("word", matches)
 
     # - understands quote escaping
     _, matches = complete(line_buffer="d['a\\'")
-    nt.assert_in("b']", matches)
+    nt.assert_in("b", matches)
 
     # - default quoting should work like repr
     _, matches = complete(line_buffer="d[")
-    nt.assert_in("\"a'b\"]", matches)
+    nt.assert_in("\"a'b\"", matches)
 
     # - when opening quote with ", possible to match with unescaped apostrophe
     _, matches = complete(line_buffer="d[\"a'")
-    nt.assert_in("b\"]", matches)
+    nt.assert_in("b", matches)
 
 
 def test_dict_key_completion_contexts():
@@ -482,10 +495,10 def test_dict_key_completion_contexts():
 
     def assert_completion(**kwargs):
         _, matches = complete(**kwargs)
-        nt.assert_in("'abc']", matches)
+        nt.assert_in("'abc'", matches)
+        nt.assert_not_in("'abc']", matches)
 
     # no completion after string closed, even if reopened
-    ip.Completer.greedy = False
     assert_no_completion(line_buffer="d['a'")
     assert_no_completion(line_buffer="d[\"a\"")
     assert_no_completion(line_buffer="d['a' + ")
@@ -497,9 +510,17 def test_dict_key_completion_contexts():
     assert_completion(line_buffer="C.data[")
 
     # greedy flag
+    def assert_completion(**kwargs):
+        _, matches = complete(**kwargs)
+        nt.assert_in("get()['abc']", matches)
+    
     assert_no_completion(line_buffer="get()[")
-    ip.Completer.greedy = True
-    assert_completion(line_buffer="get()[")
+    with greedy_completion():
+        assert_completion(line_buffer="get()[")
+        assert_completion(line_buffer="get()['")
+        assert_completion(line_buffer="get()['a")
+        assert_completion(line_buffer="get()['ab")
+        assert_completion(line_buffer="get()['abc")
 
 
 
@@ -512,25 +533,25 def test_dict_key_completion_bytes():
     ip.user_ns['d'] = {'abc': None, b'abd': None}
 
     _, matches = complete(line_buffer="d[")
-    nt.assert_in("'abc']", matches)
-    nt.assert_in("b'abd']", matches)
+    nt.assert_in("'abc'", matches)
+    nt.assert_in("b'abd'", matches)
 
     if False:  # not currently implemented
         _, matches = complete(line_buffer="d[b")
-        nt.assert_in("b'abd']", matches)
-        nt.assert_not_in("b'abc']", matches)
+        nt.assert_in("b'abd'", matches)
+        nt.assert_not_in("b'abc'", matches)
 
         _, matches = complete(line_buffer="d[b'")
-        nt.assert_in("abd']", matches)
-        nt.assert_not_in("abc']", matches)
+        nt.assert_in("abd", matches)
+        nt.assert_not_in("abc", matches)
 
         _, matches = complete(line_buffer="d[B'")
-        nt.assert_in("abd']", matches)
-        nt.assert_not_in("abc']", matches)
+        nt.assert_in("abd", matches)
+        nt.assert_not_in("abc", matches)
 
         _, matches = complete(line_buffer="d['")
-        nt.assert_in("abc']", matches)
-        nt.assert_not_in("abd']", matches)
+        nt.assert_in("abc", matches)
+        nt.assert_not_in("abd", matches)
 
 
 @dec.onlyif(sys.version_info[0] < 3, 'This test only applies in Py<3')
@@ -540,31 +561,56 def test_dict_key_completion_unicode_py2():
     complete = ip.Completer.complete
 
     ip.user_ns['d'] = {u'abc': None,
-                       unicode_type('a\xd7\x90', 'utf8'): None}
+                       u'a\u05d0b': None}
 
     _, matches = complete(line_buffer="d[")
-    nt.assert_in("u'abc']", matches)
-    nt.assert_in("u'a\\u05d0']", matches)
+    nt.assert_in("u'abc'", matches)
+    nt.assert_in("u'a\\u05d0b'", matches)
 
     _, matches = complete(line_buffer="d['a")
-    nt.assert_in("abc']", matches)
-    nt.assert_not_in("a\\u05d0']", matches)
+    nt.assert_in("abc", matches)
+    nt.assert_not_in("a\\u05d0b", matches)
 
     _, matches = complete(line_buffer="d[u'a")
-    nt.assert_in("abc']", matches)
-    nt.assert_in("a\\u05d0']", matches)
+    nt.assert_in("abc", matches)
+    nt.assert_in("a\\u05d0b", matches)
 
     _, matches = complete(line_buffer="d[U'a")
-    nt.assert_in("abc']", matches)
-    nt.assert_in("a\\u05d0']", matches)
+    nt.assert_in("abc", matches)
+    nt.assert_in("a\\u05d0b", matches)
 
     # query using escape
-    _, matches = complete(line_buffer="d[u'a\\u05d0")
-    nt.assert_in("u05d0']", matches)  # tokenized after \\
+    _, matches = complete(line_buffer=u"d[u'a\\u05d0")
+    nt.assert_in("u05d0b", matches)  # tokenized after \\
 
     # query using character
-    _, matches = complete(line_buffer=unicode_type("d[u'a\xd7\x90", 'utf8'))
-    nt.assert_in("']", matches)
+    _, matches = complete(line_buffer=u"d[u'a\u05d0")
+    nt.assert_in(u"a\u05d0b", matches)
+    
+    with greedy_completion():
+        _, matches = complete(line_buffer="d[")
+        nt.assert_in("d[u'abc']", matches)
+        nt.assert_in("d[u'a\\u05d0b']", matches)
+
+        _, matches = complete(line_buffer="d['a")
+        nt.assert_in("d['abc']", matches)
+        nt.assert_not_in("d[u'a\\u05d0b']", matches)
+
+        _, matches = complete(line_buffer="d[u'a")
+        nt.assert_in("d[u'abc']", matches)
+        nt.assert_in("d[u'a\\u05d0b']", matches)
+
+        _, matches = complete(line_buffer="d[U'a")
+        nt.assert_in("d[U'abc']", matches)
+        nt.assert_in("d[U'a\\u05d0b']", matches)
+
+        # query using escape
+        _, matches = complete(line_buffer=u"d[u'a\\u05d0")
+        nt.assert_in("d[u'a\\u05d0b']", matches)  # tokenized after \\
+
+        # query using character
+        _, matches = complete(line_buffer=u"d[u'a\u05d0")
+        nt.assert_in(u"d[u'a\u05d0b']", matches)
 
 
 @dec.onlyif(sys.version_info[0] >= 3, 'This test only applies in Py>=3')
@@ -573,15 +619,25 def test_dict_key_completion_unicode_py3():
     ip = get_ipython()
     complete = ip.Completer.complete
 
-    ip.user_ns['d'] = {unicode_type(b'a\xd7\x90', 'utf8'): None}
+    ip.user_ns['d'] = {u'a\u05d0': None}
 
     # query using escape
     _, matches = complete(line_buffer="d['a\\u05d0")
-    nt.assert_in("u05d0']", matches)  # tokenized after \\
+    nt.assert_in("u05d0", matches)  # tokenized after \\
 
     # query using character
-    _, matches = complete(line_buffer=unicode_type(b"d['a\xd7\x90", 'utf8'))
-    nt.assert_in(unicode_type(b"a\xd7\x90']", 'utf8'), matches)
+    _, matches = complete(line_buffer="d['a\u05d0")
+    nt.assert_in(u"a\u05d0", matches)
+    
+    with greedy_completion():
+        # query using escape
+        _, matches = complete(line_buffer="d['a\\u05d0")
+        nt.assert_in("d['a\\u05d0']", matches)  # tokenized after \\
+
+        # query using character
+        _, matches = complete(line_buffer="d['a\u05d0")
+        nt.assert_in(u"d['a\u05d0']", matches)
+        
 
 
 @dec.skip_without('numpy')
@@ -592,8 +648,8 def test_struct_array_key_completion():
     complete = ip.Completer.complete
     ip.user_ns['d'] = numpy.array([], dtype=[('hello', 'f'), ('world', 'f')])
     _, matches = complete(line_buffer="d['")
-    nt.assert_in("hello']", matches)
-    nt.assert_in("world']", matches)
+    nt.assert_in("hello", matches)
+    nt.assert_in("world", matches)
 
 
 @dec.skip_without('pandas')
@@ -604,8 +660,8 def test_dataframe_key_completion():
     complete = ip.Completer.complete
     ip.user_ns['d'] = pandas.DataFrame({'hello': [1], 'world': [2]})
     _, matches = complete(line_buffer="d['")
-    nt.assert_in("hello']", matches)
-    nt.assert_in("world']", matches)
+    nt.assert_in("hello", matches)
+    nt.assert_in("world", matches)
 
 
 def test_dict_key_completion_invalids():