From 9cdf92d358f91dfd927e7c24ad3ec1e92a7cf60d 2024-12-08 11:23:52 From: M Bussonnier Date: 2024-12-08 11:23:52 Subject: [PATCH] Fix completion tuple (#14594) In progress work toward #14585 guarded eval strip leading characters until it find soemthing, this is problematic as `(1, x`, becomes valid after 1 char strip: `1, x` is a tuple; So now we trim until it is valid an not a tuple. This is still imperfect as things like `(1, a[" "].y` will be trimmed to `y`, while it should stop with `a[" "].y` ? I think maybe we should back-propagate; build back up from `y`, to `a[" "].y`, greedily until we get the last valid expression – skipping any unbalanced parentheses/quotes if we encounter imblanced. --- diff --git a/IPython/core/completer.py b/IPython/core/completer.py index 4873be6..b39a922 100644 --- a/IPython/core/completer.py +++ b/IPython/core/completer.py @@ -184,6 +184,7 @@ import glob import inspect import itertools import keyword +import ast import os import re import string @@ -347,7 +348,7 @@ def provisionalcompleter(action='ignore'): yield -def has_open_quotes(s): +def has_open_quotes(s: str) -> Union[str, bool]: """Return whether a string has open quotes. This simply counts whether the number of quote characters of either type in @@ -368,7 +369,7 @@ def has_open_quotes(s): return False -def protect_filename(s, protectables=PROTECTABLES): +def protect_filename(s: str, protectables: str = PROTECTABLES) -> str: """Escape a string to protect certain characters.""" if set(s) & set(protectables): if sys.platform == "win32": @@ -449,11 +450,11 @@ def completions_sorting_key(word): if word.startswith('%%'): # If there's another % in there, this is something else, so leave it alone - if not "%" in word[2:]: + if "%" not in word[2:]: word = word[2:] prio2 = 2 elif word.startswith('%'): - if not "%" in word[1:]: + if "%" not in word[1:]: word = word[1:] prio2 = 1 @@ -752,7 +753,7 @@ def completion_matcher( priority: Optional[float] = None, identifier: Optional[str] = None, api_version: int = 1, -): +) -> Callable[[Matcher], Matcher]: """Adds attributes describing the matcher. Parameters @@ -961,8 +962,8 @@ class CompletionSplitter(object): def split_line(self, line, cursor_pos=None): """Split a line of text with a cursor at the given position. """ - l = line if cursor_pos is None else line[:cursor_pos] - return self._delim_re.split(l)[-1] + cut_line = line if cursor_pos is None else line[:cursor_pos] + return self._delim_re.split(cut_line)[-1] @@ -1141,8 +1142,13 @@ class Completer(Configurable): """ return self._attr_matches(text)[0] - def _attr_matches(self, text, include_prefix=True) -> Tuple[Sequence[str], str]: - m2 = re.match(r"(.+)\.(\w*)$", self.line_buffer) + # we simple attribute matching with normal identifiers. + _ATTR_MATCH_RE = re.compile(r"(.+)\.(\w*)$") + + def _attr_matches( + self, text: str, include_prefix: bool = True + ) -> Tuple[Sequence[str], str]: + m2 = self._ATTR_MATCH_RE.match(self.line_buffer) if not m2: return [], "" expr, attr = m2.group(1, 2) @@ -1204,6 +1210,30 @@ class Completer(Configurable): "." + attr, ) + def _trim_expr(self, code: str) -> str: + """ + Trim the code until it is a valid expression and not a tuple; + + return the trimmed expression for guarded_eval. + """ + while code: + code = code[1:] + try: + res = ast.parse(code) + except SyntaxError: + continue + + assert res is not None + if len(res.body) != 1: + continue + expr = res.body[0].value + if isinstance(expr, ast.Tuple) and not code[-1] == ")": + # we skip implicit tuple, like when trimming `fun(a,b` + # as `a,b` would be a tuple, and we actually expect to get only `b` + continue + return code + return "" + def _evaluate_expr(self, expr): obj = not_found done = False @@ -1225,14 +1255,14 @@ class Completer(Configurable): # e.g. user starts `(d[`, so we get `expr = '(d'`, # where parenthesis is not closed. # TODO: make this faster by reusing parts of the computation? - expr = expr[1:] + expr = self._trim_expr(expr) return obj def get__all__entries(obj): """returns the strings in the __all__ attribute""" try: words = getattr(obj, '__all__') - except: + except Exception: return [] return [w for w in words if isinstance(w, str)] @@ -1447,7 +1477,7 @@ def match_dict_keys( try: if not str_key.startswith(prefix_str): continue - except (AttributeError, TypeError, UnicodeError) as e: + except (AttributeError, TypeError, UnicodeError): # Python 3+ TypeError on b'a'.startswith('a') or vice-versa continue @@ -1495,7 +1525,7 @@ def cursor_to_position(text:str, line:int, column:int)->int: lines = text.split('\n') assert line <= len(lines), '{} <= {}'.format(str(line), str(len(lines))) - return sum(len(l) + 1 for l in lines[:line]) + column + return sum(len(line) + 1 for line in lines[:line]) + column def position_to_cursor(text:str, offset:int)->Tuple[int, int]: """ @@ -2112,7 +2142,7 @@ class IPCompleter(Completer): result["suppress"] = is_magic_prefix and bool(result["completions"]) return result - def magic_matches(self, text: str): + def magic_matches(self, text: str) -> List[str]: """Match magics. .. deprecated:: 8.6 @@ -2469,7 +2499,8 @@ class IPCompleter(Completer): # parenthesis before the cursor # e.g. for "foo (1+bar(x), pa,a=1)", the candidate is "foo" tokens = regexp.findall(self.text_until_cursor) - iterTokens = reversed(tokens); openPar = 0 + iterTokens = reversed(tokens) + openPar = 0 for token in iterTokens: if token == ')': @@ -2489,7 +2520,8 @@ class IPCompleter(Completer): try: ids.append(next(iterTokens)) if not isId(ids[-1]): - ids.pop(); break + ids.pop() + break if not next(iterTokens) == '.': break except StopIteration: @@ -3215,7 +3247,7 @@ class IPCompleter(Completer): else: api_version = _get_matcher_api_version(matcher) raise ValueError(f"Unsupported API version {api_version}") - except: + except BaseException: # Show the ugly traceback if the matcher causes an # exception, but do NOT crash the kernel! sys.excepthook(*sys.exc_info()) diff --git a/IPython/core/tests/test_completer.py b/IPython/core/tests/test_completer.py index fe078f9..a65d5d9 100644 --- a/IPython/core/tests/test_completer.py +++ b/IPython/core/tests/test_completer.py @@ -9,10 +9,10 @@ import pytest import sys import textwrap import unittest +import random from importlib.metadata import version - from contextlib import contextmanager from traitlets.config.loader import Config @@ -21,6 +21,7 @@ from IPython.core import completer from IPython.utils.tempdir import TemporaryDirectory, TemporaryWorkingDirectory from IPython.utils.generics import complete_object from IPython.testing import decorators as dec +from IPython.core.latex_symbols import latex_symbols from IPython.core.completer import ( Completion, @@ -31,11 +32,24 @@ from IPython.core.completer import ( completion_matcher, SimpleCompletion, CompletionContext, + _unicode_name_compute, + _UNICODE_RANGES, ) from packaging.version import parse +@contextmanager +def jedi_status(status: bool): + completer = get_ipython().Completer + try: + old = completer.use_jedi + completer.use_jedi = status + yield + finally: + completer.use_jedi = old + + # ----------------------------------------------------------------------------- # Test functions # ----------------------------------------------------------------------------- @@ -66,7 +80,7 @@ def recompute_unicode_ranges(): rg = list(ranges(valid)) lens = [] gap_lens = [] - pstart, pstop = 0, 0 + _pstart, pstop = 0, 0 for start, stop in rg: lens.append(stop - start) gap_lens.append( @@ -77,7 +91,7 @@ def recompute_unicode_ranges(): f"{round((start - pstop)/0xe01f0*100)}%", ) ) - pstart, pstop = start, stop + _pstart, pstop = start, stop return sorted(gap_lens)[-1] @@ -87,7 +101,6 @@ def test_unicode_range(): Test that the ranges we test for unicode names give the same number of results than testing the full length. """ - from IPython.core.completer import _unicode_name_compute, _UNICODE_RANGES expected_list = _unicode_name_compute([(0, 0x110000)]) test = _unicode_name_compute(_UNICODE_RANGES) @@ -148,45 +161,45 @@ def custom_matchers(matchers): ip.Completer.custom_matchers.clear() -def test_protect_filename(): - if sys.platform == "win32": - pairs = [ - ("abc", "abc"), - (" abc", '" abc"'), - ("a bc", '"a bc"'), - ("a bc", '"a bc"'), - (" bc", '" bc"'), - ] - else: - pairs = [ - ("abc", "abc"), - (" abc", r"\ abc"), - ("a bc", r"a\ bc"), - ("a bc", r"a\ \ bc"), - (" bc", r"\ \ bc"), - # On posix, we also protect parens and other special characters. - ("a(bc", r"a\(bc"), - ("a)bc", r"a\)bc"), - ("a( )bc", r"a\(\ \)bc"), - ("a[1]bc", r"a\[1\]bc"), - ("a{1}bc", r"a\{1\}bc"), - ("a#bc", r"a\#bc"), - ("a?bc", r"a\?bc"), - ("a=bc", r"a\=bc"), - ("a\\bc", r"a\\bc"), - ("a|bc", r"a\|bc"), - ("a;bc", r"a\;bc"), - ("a:bc", r"a\:bc"), - ("a'bc", r"a\'bc"), - ("a*bc", r"a\*bc"), - ('a"bc', r"a\"bc"), - ("a^bc", r"a\^bc"), - ("a&bc", r"a\&bc"), - ] - # run the actual tests - for s1, s2 in pairs: - s1p = completer.protect_filename(s1) - assert s1p == s2 +if sys.platform == "win32": + pairs = [ + ("abc", "abc"), + (" abc", '" abc"'), + ("a bc", '"a bc"'), + ("a bc", '"a bc"'), + (" bc", '" bc"'), + ] +else: + pairs = [ + ("abc", "abc"), + (" abc", r"\ abc"), + ("a bc", r"a\ bc"), + ("a bc", r"a\ \ bc"), + (" bc", r"\ \ bc"), + # On posix, we also protect parens and other special characters. + ("a(bc", r"a\(bc"), + ("a)bc", r"a\)bc"), + ("a( )bc", r"a\(\ \)bc"), + ("a[1]bc", r"a\[1\]bc"), + ("a{1}bc", r"a\{1\}bc"), + ("a#bc", r"a\#bc"), + ("a?bc", r"a\?bc"), + ("a=bc", r"a\=bc"), + ("a\\bc", r"a\\bc"), + ("a|bc", r"a\|bc"), + ("a;bc", r"a\;bc"), + ("a:bc", r"a\:bc"), + ("a'bc", r"a\'bc"), + ("a*bc", r"a\*bc"), + ('a"bc', r"a\"bc"), + ("a^bc", r"a\^bc"), + ("a&bc", r"a\&bc"), + ] + + +@pytest.mark.parametrize("s1,expected", pairs) +def test_protect_filename(s1, expected): + assert completer.protect_filename(s1) == expected def check_line_split(splitter, test_specs): @@ -297,8 +310,6 @@ class TestCompleter(unittest.TestCase): self.assertIsInstance(matches, list) def test_latex_completions(self): - from IPython.core.latex_symbols import latex_symbols - import random ip = get_ipython() # Test some random unicode symbols @@ -1735,6 +1746,45 @@ class TestCompleter(unittest.TestCase): @pytest.mark.parametrize( + "setup,code,expected,not_expected", + [ + ('a="str"; b=1', "(a, b.", [".bit_count", ".conjugate"], [".count"]), + ('a="str"; b=1', "(a, b).", [".count"], [".bit_count", ".capitalize"]), + ('x="str"; y=1', "x = {1, y.", [".bit_count"], [".count"]), + ('x="str"; y=1', "x = [1, y.", [".bit_count"], [".count"]), + ('x="str"; y=1; fun=lambda x:x', "x = fun(1, y.", [".bit_count"], [".count"]), + ], +) +def test_misc_no_jedi_completions(setup, code, expected, not_expected): + ip = get_ipython() + c = ip.Completer + ip.ex(setup) + with provisionalcompleter(), jedi_status(False): + matches = c.all_completions(code) + assert set(expected) - set(matches) == set(), set(matches) + assert set(matches).intersection(set(not_expected)) == set() + + +@pytest.mark.parametrize( + "code,expected", + [ + (" (a, b", "b"), + ("(a, b", "b"), + ("(a, b)", ""), # trim always start by trimming + (" (a, b)", "(a, b)"), + (" [a, b]", "[a, b]"), + (" a, b", "b"), + ("x = {1, y", "y"), + ("x = [1, y", "y"), + ("x = fun(1, y", "y"), + ], +) +def test_trim_expr(code, expected): + c = get_ipython().Completer + assert c._trim_expr(code) == expected + + +@pytest.mark.parametrize( "input, expected", [ ["1.234", "1.234"],