From 8e3376e9f4888482aebd4cae47367e7e9d1ab1f2 2022-12-21 17:56:14 From: Emilio Graff <1@emil.io> Date: 2022-12-21 17:56:14 Subject: [PATCH] Merge branch 'main' into shaperilio/autoreload-verbosity --- diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml index 8d1927d..e05678f 100644 --- a/.github/workflows/mypy.yml +++ b/.github/workflows/mypy.yml @@ -31,6 +31,8 @@ jobs: run: | mypy -p IPython.terminal mypy -p IPython.core.magics + mypy -p IPython.core.guarded_eval + mypy -p IPython.core.completer - name: Lint with pyflakes run: | flake8 IPython/core/magics/script.py diff --git a/IPython/__init__.py b/IPython/__init__.py index 03b3116..c224f9a 100644 --- a/IPython/__init__.py +++ b/IPython/__init__.py @@ -1,3 +1,4 @@ +# PYTHON_ARGCOMPLETE_OK """ IPython: tools for interactive and parallel computing in Python. diff --git a/IPython/__main__.py b/IPython/__main__.py index d5123f3..8e9f989 100644 --- a/IPython/__main__.py +++ b/IPython/__main__.py @@ -1,3 +1,4 @@ +# PYTHON_ARGCOMPLETE_OK # encoding: utf-8 """Terminal-based IPython entry point. """ diff --git a/IPython/core/completer.py b/IPython/core/completer.py index 2dff9ef..f2853d3 100644 --- a/IPython/core/completer.py +++ b/IPython/core/completer.py @@ -50,7 +50,7 @@ Backward latex completion It is sometime challenging to know how to type a character, if you are using IPython, or any compatible frontend you can prepend backslash to the character -and press ```` to expand it to its latex form. +and press :kbd:`Tab` to expand it to its latex form. .. code:: @@ -59,7 +59,7 @@ and press ```` to expand it to its latex form. Both forward and backward completions can be deactivated by setting the -``Completer.backslash_combining_completions`` option to ``False``. +:any:`Completer.backslash_combining_completions` option to ``False``. Experimental @@ -95,7 +95,7 @@ having to execute any code: ... myvar[1].bi Tab completion will be able to infer that ``myvar[1]`` is a real number without -executing any code unlike the previously available ``IPCompleter.greedy`` +executing almost any code unlike the deprecated :any:`IPCompleter.greedy` option. Be sure to update :any:`jedi` to the latest stable version or to try the @@ -178,6 +178,7 @@ The suppression behaviour can is user-configurable via from __future__ import annotations import builtins as builtin_mod +import enum import glob import inspect import itertools @@ -186,14 +187,16 @@ import os import re import string import sys +import tokenize import time import unicodedata import uuid import warnings +from ast import literal_eval +from collections import defaultdict from contextlib import contextmanager from dataclasses import dataclass from functools import cached_property, partial -from importlib import import_module from types import SimpleNamespace from typing import ( Iterable, @@ -204,14 +207,15 @@ from typing import ( Any, Sequence, Dict, - NamedTuple, - Pattern, Optional, TYPE_CHECKING, Set, + Sized, + TypeVar, Literal, ) +from IPython.core.guarded_eval import guarded_eval, EvaluationContext from IPython.core.error import TryNext from IPython.core.inputtransformer2 import ESC_MAGIC from IPython.core.latex_symbols import latex_symbols, reverse_latex_symbol @@ -231,7 +235,6 @@ from traitlets import ( Unicode, Dict as DictTrait, Union as UnionTrait, - default, observe, ) from traitlets.config.configurable import Configurable @@ -254,10 +257,11 @@ except ImportError: if TYPE_CHECKING or GENERATING_DOCUMENTATION: from typing import cast - from typing_extensions import TypedDict, NotRequired, Protocol, TypeAlias + from typing_extensions import TypedDict, NotRequired, Protocol, TypeAlias, TypeGuard else: + from typing import Generic - def cast(obj, type_): + def cast(type_, obj): """Workaround for `TypeError: MatcherAPIv2() takes no arguments`""" return obj @@ -266,6 +270,7 @@ else: TypedDict = Dict # by extension of `NotRequired` requires 3.11 too Protocol = object # requires Python >=3.8 TypeAlias = Any # requires Python >=3.10 + TypeGuard = Generic # requires Python >=3.10 if GENERATING_DOCUMENTATION: from typing import TypedDict @@ -296,6 +301,9 @@ MATCHES_LIMIT = 500 # Completion type reported when no type can be inferred. _UNKNOWN_TYPE = "" +# sentinel value to signal lack of a match +not_found = object() + class ProvisionalCompleterWarning(FutureWarning): """ Exception raise by an experimental feature in this module. @@ -466,8 +474,9 @@ class _FakeJediCompletion: self.complete = name self.type = 'crashed' self.name_with_symbols = name - self.signature = '' - self._origin = 'fake' + self.signature = "" + self._origin = "fake" + self.text = "crashed" def __repr__(self): return '' @@ -503,11 +512,23 @@ class Completion: __slots__ = ['start', 'end', 'text', 'type', 'signature', '_origin'] - def __init__(self, start: int, end: int, text: str, *, type: str=None, _origin='', signature='') -> None: - warnings.warn("``Completion`` is a provisional API (as of IPython 6.0). " - "It may change without warnings. " - "Use in corresponding context manager.", - category=ProvisionalCompleterWarning, stacklevel=2) + def __init__( + self, + start: int, + end: int, + text: str, + *, + type: Optional[str] = None, + _origin="", + signature="", + ) -> None: + warnings.warn( + "``Completion`` is a provisional API (as of IPython 6.0). " + "It may change without warnings. " + "Use in corresponding context manager.", + category=ProvisionalCompleterWarning, + stacklevel=2, + ) self.start = start self.end = end @@ -520,7 +541,7 @@ class Completion: return '' % \ (self.start, self.end, self.text, self.type or '?', self.signature or '?') - def __eq__(self, other)->Bool: + def __eq__(self, other) -> bool: """ Equality and hash do not hash the type (as some completer may not be able to infer the type), but are use to (partially) de-duplicate @@ -554,7 +575,7 @@ class SimpleCompletion: __slots__ = ["text", "type"] - def __init__(self, text: str, *, type: str = None): + def __init__(self, text: str, *, type: Optional[str] = None): self.text = text self.type = type @@ -588,14 +609,18 @@ class SimpleMatcherResult(_MatcherResultBase, TypedDict): # in order to get __orig_bases__ for documentation #: List of candidate completions - completions: Sequence[SimpleCompletion] + completions: Sequence[SimpleCompletion] | Iterator[SimpleCompletion] class _JediMatcherResult(_MatcherResultBase): """Matching result returned by Jedi (will be processed differently)""" #: list of candidate completions - completions: Iterable[_JediCompletionLike] + completions: Iterator[_JediCompletionLike] + + +AnyMatcherCompletion = Union[_JediCompletionLike, SimpleCompletion] +AnyCompletion = TypeVar("AnyCompletion", AnyMatcherCompletion, Completion) @dataclass @@ -642,16 +667,21 @@ MatcherResult = Union[SimpleMatcherResult, _JediMatcherResult] class _MatcherAPIv1Base(Protocol): - def __call__(self, text: str) -> list[str]: + def __call__(self, text: str) -> List[str]: """Call signature.""" + ... + + #: Used to construct the default matcher identifier + __qualname__: str class _MatcherAPIv1Total(_MatcherAPIv1Base, Protocol): #: API version matcher_api_version: Optional[Literal[1]] - def __call__(self, text: str) -> list[str]: + def __call__(self, text: str) -> List[str]: """Call signature.""" + ... #: Protocol describing Matcher API v1. @@ -666,26 +696,61 @@ class MatcherAPIv2(Protocol): def __call__(self, context: CompletionContext) -> MatcherResult: """Call signature.""" + ... + + #: Used to construct the default matcher identifier + __qualname__: str Matcher: TypeAlias = Union[MatcherAPIv1, MatcherAPIv2] +def _is_matcher_v1(matcher: Matcher) -> TypeGuard[MatcherAPIv1]: + api_version = _get_matcher_api_version(matcher) + return api_version == 1 + + +def _is_matcher_v2(matcher: Matcher) -> TypeGuard[MatcherAPIv2]: + api_version = _get_matcher_api_version(matcher) + return api_version == 2 + + +def _is_sizable(value: Any) -> TypeGuard[Sized]: + """Determines whether objects is sizable""" + return hasattr(value, "__len__") + + +def _is_iterator(value: Any) -> TypeGuard[Iterator]: + """Determines whether objects is sizable""" + return hasattr(value, "__next__") + + def has_any_completions(result: MatcherResult) -> bool: """Check if any result includes any completions.""" - if hasattr(result["completions"], "__len__"): - return len(result["completions"]) != 0 - try: - old_iterator = result["completions"] - first = next(old_iterator) - result["completions"] = itertools.chain([first], old_iterator) - return True - except StopIteration: - return False + completions = result["completions"] + if _is_sizable(completions): + return len(completions) != 0 + if _is_iterator(completions): + try: + old_iterator = completions + first = next(old_iterator) + result["completions"] = cast( + Iterator[SimpleCompletion], + itertools.chain([first], old_iterator), + ) + return True + except StopIteration: + return False + raise ValueError( + "Completions returned by matcher need to be an Iterator or a Sizable" + ) def completion_matcher( - *, priority: float = None, identifier: str = None, api_version: int = 1 + *, + priority: Optional[float] = None, + identifier: Optional[str] = None, + api_version: int = 1, ): """Adds attributes describing the matcher. @@ -708,14 +773,14 @@ def completion_matcher( """ def wrapper(func: Matcher): - func.matcher_priority = priority or 0 - func.matcher_identifier = identifier or func.__qualname__ - func.matcher_api_version = api_version + func.matcher_priority = priority or 0 # type: ignore + func.matcher_identifier = identifier or func.__qualname__ # type: ignore + func.matcher_api_version = api_version # type: ignore if TYPE_CHECKING: if api_version == 1: - func = cast(func, MatcherAPIv1) + func = cast(MatcherAPIv1, func) elif api_version == 2: - func = cast(func, MatcherAPIv2) + func = cast(MatcherAPIv2, func) return func return wrapper @@ -902,12 +967,44 @@ class CompletionSplitter(object): class Completer(Configurable): - greedy = Bool(False, - help="""Activate greedy completion - PENDING DEPRECATION. this is now mostly taken care of with Jedi. + greedy = Bool( + False, + help="""Activate greedy completion. + + .. deprecated:: 8.8 + Use :any:`Completer.evaluation` and :any:`Completer.auto_close_dict_keys` instead. + + When enabled in IPython 8.8 or newer, changes configuration as follows: - This will enable completion on elements of lists, results of function calls, etc., - but can be unsafe because the code is actually evaluated on TAB. + - ``Completer.evaluation = 'unsafe'`` + - ``Completer.auto_close_dict_keys = True`` + """, + ).tag(config=True) + + evaluation = Enum( + ("forbidden", "minimal", "limited", "unsafe", "dangerous"), + default_value="limited", + help="""Policy for code evaluation under completion. + + Successive options allow to enable more eager evaluation for better + completion suggestions, including for nested dictionaries, nested lists, + or even results of function calls. + Setting ``unsafe`` or higher can lead to evaluation of arbitrary user + code on :kbd:`Tab` with potentially unwanted or dangerous side effects. + + Allowed values are: + + - ``forbidden``: no evaluation of code is permitted, + - ``minimal``: evaluation of literals and access to built-in namespace; + no item/attribute evaluationm no access to locals/globals, + no evaluation of any operations or comparisons. + - ``limited``: access to all namespaces, evaluation of hard-coded methods + (for example: :any:`dict.keys`, :any:`object.__getattr__`, + :any:`object.__getitem__`) on allow-listed objects (for example: + :any:`dict`, :any:`list`, :any:`tuple`, ``pandas.Series``), + - ``unsafe``: evaluation of all methods and function calls but not of + syntax with side-effects like `del x`, + - ``dangerous``: completely arbitrary evaluation. """, ).tag(config=True) @@ -931,6 +1028,18 @@ class Completer(Configurable): "Includes completion of latex commands, unicode names, and expanding " "unicode characters back to latex commands.").tag(config=True) + auto_close_dict_keys = Bool( + False, + help=""" + Enable auto-closing dictionary keys. + + When enabled string keys will be suffixed with a final quote + (matching the opening quote), tuple keys will also receive a + separating comma if needed, and keys which are final will + receive a closing bracket (``]``). + """, + ).tag(config=True) + def __init__(self, namespace=None, global_namespace=None, **kwargs): """Create a new completer for the command line. @@ -1029,28 +1138,16 @@ class Completer(Configurable): with a __getattr__ hook is evaluated. """ + m2 = re.match(r"(.+)\.(\w*)$", self.line_buffer) + if not m2: + return [] + expr, attr = m2.group(1, 2) - # Another option, seems to work great. Catches things like ''. - m = re.match(r"(\S+(\.\w+)*)\.(\w*)$", text) + obj = self._evaluate_expr(expr) - if m: - expr, attr = m.group(1, 3) - elif self.greedy: - m2 = re.match(r"(.+)\.(\w*)$", self.line_buffer) - if not m2: - return [] - expr, attr = m2.group(1,2) - else: + if obj is not_found: return [] - try: - obj = eval(expr, self.namespace) - except: - try: - obj = eval(expr, self.global_namespace) - except: - return [] - if self.limit_to__all__ and hasattr(obj, '__all__'): words = get__all__entries(obj) else: @@ -1068,8 +1165,31 @@ class Completer(Configurable): pass # Build match list to return n = len(attr) - return [u"%s.%s" % (expr, w) for w in words if w[:n] == attr ] + return ["%s.%s" % (expr, w) for w in words if w[:n] == attr] + def _evaluate_expr(self, expr): + obj = not_found + done = False + while not done and expr: + try: + obj = guarded_eval( + expr, + EvaluationContext( + globals=self.global_namespace, + locals=self.namespace, + evaluation=self.evaluation, + ), + ) + done = True + except Exception as e: + if self.debug: + print("Evaluation exception", e) + # trim the expression to remove any invalid prefix + # e.g. user starts `(d[`, so we get `expr = '(d'`, + # where parenthesis is not closed. + # TODO: make this faster by reusing parts of the computation? + expr = expr[1:] + return obj def get__all__entries(obj): """returns the strings in the __all__ attribute""" @@ -1081,8 +1201,82 @@ def get__all__entries(obj): return [w for w in words if isinstance(w, str)] -def match_dict_keys(keys: List[Union[str, bytes, Tuple[Union[str, bytes]]]], prefix: str, delims: str, - extra_prefix: Optional[Tuple[str, bytes]]=None) -> Tuple[str, int, List[str]]: +class _DictKeyState(enum.Flag): + """Represent state of the key match in context of other possible matches. + + - given `d1 = {'a': 1}` completion on `d1['` will yield `{'a': END_OF_ITEM}` as there is no tuple. + - given `d2 = {('a', 'b'): 1}`: `d2['a', '` will yield `{'b': END_OF_TUPLE}` as there is no tuple members to add beyond `'b'`. + - given `d3 = {('a', 'b'): 1}`: `d3['` will yield `{'a': IN_TUPLE}` as `'a'` can be added. + - given `d4 = {'a': 1, ('a', 'b'): 2}`: `d4['` will yield `{'a': END_OF_ITEM & END_OF_TUPLE}` + """ + + BASELINE = 0 + END_OF_ITEM = enum.auto() + END_OF_TUPLE = enum.auto() + IN_TUPLE = enum.auto() + + +def _parse_tokens(c): + """Parse tokens even if there is an error.""" + tokens = [] + token_generator = tokenize.generate_tokens(iter(c.splitlines()).__next__) + while True: + try: + tokens.append(next(token_generator)) + except tokenize.TokenError: + return tokens + except StopIteration: + return tokens + + +def _match_number_in_dict_key_prefix(prefix: str) -> Union[str, None]: + """Match any valid Python numeric literal in a prefix of dictionary keys. + + References: + - https://docs.python.org/3/reference/lexical_analysis.html#numeric-literals + - https://docs.python.org/3/library/tokenize.html + """ + if prefix[-1].isspace(): + # if user typed a space we do not have anything to complete + # even if there was a valid number token before + return None + tokens = _parse_tokens(prefix) + rev_tokens = reversed(tokens) + skip_over = {tokenize.ENDMARKER, tokenize.NEWLINE} + number = None + for token in rev_tokens: + if token.type in skip_over: + continue + if number is None: + if token.type == tokenize.NUMBER: + number = token.string + continue + else: + # we did not match a number + return None + if token.type == tokenize.OP: + if token.string == ",": + break + if token.string in {"+", "-"}: + number = token.string + number + else: + return None + return number + + +_INT_FORMATS = { + "0b": bin, + "0o": oct, + "0x": hex, +} + + +def match_dict_keys( + keys: List[Union[str, bytes, Tuple[Union[str, bytes], ...]]], + prefix: str, + delims: str, + extra_prefix: Optional[Tuple[Union[str, bytes], ...]] = None, +) -> Tuple[str, int, Dict[str, _DictKeyState]]: """Used by dict_key_matches, matching the prefix to a list of keys Parameters @@ -1102,47 +1296,89 @@ def match_dict_keys(keys: List[Union[str, bytes, Tuple[Union[str, bytes]]]], pre A tuple of three elements: ``quote``, ``token_start``, ``matched``, with ``quote`` being the quote that need to be used to close current string. ``token_start`` the position where the replacement should start occurring, - ``matches`` a list of replacement/completion - + ``matches`` a dictionary of replacement/completion keys on keys and values + indicating whether the state. """ prefix_tuple = extra_prefix if extra_prefix else () - Nprefix = len(prefix_tuple) + + prefix_tuple_size = sum( + [ + # for pandas, do not count slices as taking space + not isinstance(k, slice) + for k in prefix_tuple + ] + ) + text_serializable_types = (str, bytes, int, float, slice) + def filter_prefix_tuple(key): # Reject too short keys - if len(key) <= Nprefix: + if len(key) <= prefix_tuple_size: return False - # Reject keys with non str/bytes in it + # Reject keys which cannot be serialised to text for k in key: - if not isinstance(k, (str, bytes)): + if not isinstance(k, text_serializable_types): return False # Reject keys that do not match the prefix for k, pt in zip(key, prefix_tuple): - if k != pt: + if k != pt and not isinstance(pt, slice): return False # All checks passed! return True - filtered_keys:List[Union[str,bytes]] = [] - def _add_to_filtered_keys(key): - if isinstance(key, (str, bytes)): - filtered_keys.append(key) + filtered_key_is_final: Dict[ + Union[str, bytes, int, float], _DictKeyState + ] = defaultdict(lambda: _DictKeyState.BASELINE) for k in keys: + # If at least one of the matches is not final, mark as undetermined. + # This can happen with `d = {111: 'b', (111, 222): 'a'}` where + # `111` appears final on first match but is not final on the second. + if isinstance(k, tuple): if filter_prefix_tuple(k): - _add_to_filtered_keys(k[Nprefix]) + key_fragment = k[prefix_tuple_size] + filtered_key_is_final[key_fragment] |= ( + _DictKeyState.END_OF_TUPLE + if len(k) == prefix_tuple_size + 1 + else _DictKeyState.IN_TUPLE + ) + elif prefix_tuple_size > 0: + # we are completing a tuple but this key is not a tuple, + # so we should ignore it + pass else: - _add_to_filtered_keys(k) + if isinstance(k, text_serializable_types): + filtered_key_is_final[k] |= _DictKeyState.END_OF_ITEM + + filtered_keys = filtered_key_is_final.keys() if not prefix: - return '', 0, [repr(k) for k in filtered_keys] - quote_match = re.search('["\']', prefix) - assert quote_match is not None # silence mypy - quote = quote_match.group() - try: - prefix_str = eval(prefix + quote, {}) - except Exception: - return '', 0, [] + return "", 0, {repr(k): v for k, v in filtered_key_is_final.items()} + + quote_match = re.search("(?:\"|')", prefix) + is_user_prefix_numeric = False + + if quote_match: + quote = quote_match.group() + valid_prefix = prefix + quote + try: + prefix_str = literal_eval(valid_prefix) + except Exception: + return "", 0, {} + else: + # If it does not look like a string, let's assume + # we are dealing with a number or variable. + number_match = _match_number_in_dict_key_prefix(prefix) + + # We do not want the key matcher to suggest variable names so we yield: + if number_match is None: + # The alternative would be to assume that user forgort the quote + # and if the substring matches, suggest adding it at the start. + return "", 0, {} + + prefix_str = number_match + is_user_prefix_numeric = True + quote = "" pattern = '[^' + ''.join('\\' + c for c in delims) + ']*$' token_match = re.search(pattern, prefix, re.UNICODE) @@ -1150,17 +1386,36 @@ def match_dict_keys(keys: List[Union[str, bytes, Tuple[Union[str, bytes]]]], pre token_start = token_match.start() token_prefix = token_match.group() - matched:List[str] = [] + matched: Dict[str, _DictKeyState] = {} + + str_key: Union[str, bytes] + for key in filtered_keys: + if isinstance(key, (int, float)): + # User typed a number but this key is not a number. + if not is_user_prefix_numeric: + continue + str_key = str(key) + if isinstance(key, int): + int_base = prefix_str[:2].lower() + # if user typed integer using binary/oct/hex notation: + if int_base in _INT_FORMATS: + int_format = _INT_FORMATS[int_base] + str_key = int_format(key) + else: + # User typed a string but this key is a number. + if is_user_prefix_numeric: + continue + str_key = key try: - if not key.startswith(prefix_str): + if not str_key.startswith(prefix_str): continue - except (AttributeError, TypeError, UnicodeError): + except (AttributeError, TypeError, UnicodeError) as e: # Python 3+ TypeError on b'a'.startswith('a') or vice-versa continue # reformat remainder of key to begin with prefix - rem = key[len(prefix_str):] + rem = str_key[len(prefix_str) :] # force repr wrapped in ' rem_repr = repr(rem + '"') if isinstance(rem, str) else repr(rem + b'"') rem_repr = rem_repr[1 + rem_repr.index("'"):-2] @@ -1171,7 +1426,9 @@ def match_dict_keys(keys: List[Union[str, bytes, Tuple[Union[str, bytes]]]], pre rem_repr = rem_repr.replace('"', '\\"') # then reinsert prefix from start of token - matched.append('%s%s' % (token_prefix, rem_repr)) + match = "%s%s" % (token_prefix, rem_repr) + + matched[match] = filtered_key_is_final[key] return quote, token_start, matched @@ -1237,11 +1494,14 @@ def position_to_cursor(text:str, offset:int)->Tuple[int, int]: return line, col -def _safe_isinstance(obj, module, class_name): +def _safe_isinstance(obj, module, class_name, *attrs): """Checks if obj is an instance of module.class_name if loaded """ - return (module in sys.modules and - isinstance(obj, getattr(import_module(module), class_name))) + if module in sys.modules: + m = sys.modules[module] + for attr in [class_name, *attrs]: + m = getattr(m, attr) + return isinstance(obj, m) @context_matcher() @@ -1394,10 +1654,59 @@ def _make_signature(completion)-> str: _CompleteResult = Dict[str, MatcherResult] +DICT_MATCHER_REGEX = re.compile( + r"""(?x) +( # match dict-referring - or any get item object - expression + .+ +) +\[ # open bracket +\s* # and optional whitespace +# Capture any number of serializable objects (e.g. "a", "b", 'c') +# and slices +((?:(?: + (?: # closed string + [uUbB]? # string prefix (r not handled) + (?: + '(?:[^']|(? SimpleMatcherResult: """Utility to help with transition""" @@ -1407,27 +1716,29 @@ def _convert_matcher_v1_result_to_v2( } if fragment is not None: result["matched_fragment"] = fragment - return result + return cast(SimpleMatcherResult, result) class IPCompleter(Completer): """Extension of the completer class with IPython-specific features""" - __dict_key_regexps: Optional[Dict[bool,Pattern]] = None - @observe('greedy') def _greedy_changed(self, change): """update the splitter and readline delims when greedy is changed""" - if change['new']: + if change["new"]: + self.evaluation = "unsafe" + self.auto_close_dict_keys = True self.splitter.delims = GREEDY_DELIMS else: + self.evaluation = "limited" + self.auto_close_dict_keys = False self.splitter.delims = DELIMS dict_keys_only = Bool( False, help=""" Whether to show dict key matches only. - + (disables all matchers except for `IPCompleter.dict_key_matcher`). """, ) @@ -1607,7 +1918,7 @@ class IPCompleter(Completer): if not self.backslash_combining_completions: for matcher in self._backslash_combining_matchers: - self.disable_matchers.append(matcher.matcher_identifier) + self.disable_matchers.append(_get_matcher_id(matcher)) if not self.merge_completions: self.suppress_competing_matchers = True @@ -1897,7 +2208,7 @@ class IPCompleter(Completer): def _jedi_matches( self, cursor_column: int, cursor_line: int, text: str - ) -> Iterable[_JediCompletionLike]: + ) -> Iterator[_JediCompletionLike]: """ Return a list of :any:`jedi.api.Completion`s object from a ``text`` and cursor position. @@ -1963,15 +2274,23 @@ class IPCompleter(Completer): print("Error detecting if completing a non-finished string :", e, '|') if not try_jedi: - return [] + return iter([]) try: return filter(completion_filter, interpreter.complete(column=cursor_column, line=cursor_line + 1)) except Exception as e: if self.debug: - return [_FakeJediCompletion('Oops Jedi has crashed, please report a bug with the following:\n"""\n%s\ns"""' % (e))] + return iter( + [ + _FakeJediCompletion( + 'Oops Jedi has crashed, please report a bug with the following:\n"""\n%s\ns"""' + % (e) + ) + ] + ) else: - return [] + return iter([]) + @completion_matcher(api_version=1) def python_matches(self, text: str) -> Iterable[str]: """Match attributes or global python names""" if "." in text: @@ -2149,12 +2468,16 @@ class IPCompleter(Completer): return method() # Special case some common in-memory dict-like types - if isinstance(obj, dict) or\ - _safe_isinstance(obj, 'pandas', 'DataFrame'): + if isinstance(obj, dict) or _safe_isinstance(obj, "pandas", "DataFrame"): try: return list(obj.keys()) except Exception: return [] + elif _safe_isinstance(obj, "pandas", "core", "indexing", "_LocIndexer"): + try: + return list(obj.obj.keys()) + except Exception: + return [] elif _safe_isinstance(obj, 'numpy', 'ndarray') or\ _safe_isinstance(obj, 'numpy', 'void'): return obj.dtype.names or [] @@ -2175,74 +2498,49 @@ class IPCompleter(Completer): You can use :meth:`dict_key_matcher` instead. """ - if self.__dict_key_regexps is not None: - regexps = self.__dict_key_regexps - else: - dict_key_re_fmt = r'''(?x) - ( # match dict-referring expression wrt greedy setting - %s - ) - \[ # open bracket - \s* # and optional whitespace - # Capture any number of str-like objects (e.g. "a", "b", 'c') - ((?:[uUbB]? # string prefix (r not handled) - (?: - '(?:[^']|(? text_start and closing_quote: - # quotes were opened inside text, maybe close them - if continuation.startswith(closing_quote): - continuation = continuation[len(closing_quote):] - else: - suf += closing_quote - if bracket_idx > text_start: - # brackets were opened inside text, maybe close them - if not continuation.startswith(']'): - suf += ']' + # the text given to this method, e.g. `d["""a\nt + can_close_quote = False + can_close_bracket = False + + continuation = self.line_buffer[len(self.text_until_cursor) :].strip() - return [leading + k + suf for k in matches] + if continuation.startswith(closing_quote): + # do not close if already closed, e.g. `d['a'` + continuation = continuation[len(closing_quote) :] + else: + can_close_quote = True + + continuation = continuation.strip() + + # e.g. `pandas.DataFrame` has different tuple indexer behaviour, + # handling it is out of scope, so let's avoid appending suffixes. + has_known_tuple_handling = isinstance(obj, dict) + + can_close_bracket = ( + not continuation.startswith("]") and self.auto_close_dict_keys + ) + can_close_tuple_item = ( + not continuation.startswith(",") + and has_known_tuple_handling + and self.auto_close_dict_keys + ) + can_close_quote = can_close_quote and self.auto_close_dict_keys + + # fast path if closing qoute should be appended but not suffix is allowed + if not can_close_quote and not can_close_bracket and closing_quote: + return [leading + k for k in matches] + + results = [] + + end_of_tuple_or_item = _DictKeyState.END_OF_TUPLE | _DictKeyState.END_OF_ITEM + + for k, state_flag in matches.items(): + result = leading + k + if can_close_quote and closing_quote: + result += closing_quote + + if state_flag == end_of_tuple_or_item: + # We do not know which suffix to add, + # e.g. both tuple item and string + # match this item. + pass + + if state_flag in end_of_tuple_or_item and can_close_bracket: + result += "]" + if state_flag == _DictKeyState.IN_TUPLE and can_close_tuple_item: + result += ", " + results.append(result) + return results @context_matcher() def unicode_name_matcher(self, context: CompletionContext): @@ -2516,17 +2849,23 @@ class IPCompleter(Completer): jedi_matcher_id = _get_matcher_id(self._jedi_matcher) + def is_non_jedi_result( + result: MatcherResult, identifier: str + ) -> TypeGuard[SimpleMatcherResult]: + return identifier != jedi_matcher_id + results = self._complete( full_text=full_text, cursor_line=cursor_line, cursor_pos=cursor_column ) + non_jedi_results: Dict[str, SimpleMatcherResult] = { identifier: result for identifier, result in results.items() - if identifier != jedi_matcher_id + if is_non_jedi_result(result, identifier) } jedi_matches = ( - cast(results[jedi_matcher_id], _JediMatcherResult)["completions"] + cast(_JediMatcherResult, results[jedi_matcher_id])["completions"] if jedi_matcher_id in results else () ) @@ -2581,8 +2920,8 @@ class IPCompleter(Completer): signature="", ) - ordered = [] - sortable = [] + ordered: List[Completion] = [] + sortable: List[Completion] = [] for origin, result in non_jedi_results.items(): matched_text = result["matched_fragment"] @@ -2672,8 +3011,8 @@ class IPCompleter(Completer): abort_if_offset_changes: bool, ): - sortable = [] - ordered = [] + sortable: List[AnyMatcherCompletion] = [] + ordered: List[AnyMatcherCompletion] = [] most_recent_fragment = None for identifier, result in results.items(): if identifier in skip_matchers: @@ -2772,11 +3111,11 @@ class IPCompleter(Completer): ) # Start with a clean slate of completions - results = {} + results: Dict[str, MatcherResult] = {} jedi_matcher_id = _get_matcher_id(self._jedi_matcher) - suppressed_matchers = set() + suppressed_matchers: Set[str] = set() matchers = { _get_matcher_id(matcher): matcher @@ -2786,7 +3125,6 @@ class IPCompleter(Completer): } for matcher_id, matcher in matchers.items(): - api_version = _get_matcher_api_version(matcher) matcher_id = _get_matcher_id(matcher) if matcher_id in self.disable_matchers: @@ -2798,14 +3136,16 @@ class IPCompleter(Completer): if matcher_id in suppressed_matchers: continue + result: MatcherResult try: - if api_version == 1: + if _is_matcher_v1(matcher): result = _convert_matcher_v1_result_to_v2( matcher(text), type=_UNKNOWN_TYPE ) - elif api_version == 2: - result = cast(matcher, MatcherAPIv2)(context) + elif _is_matcher_v2(matcher): + result = matcher(context) else: + api_version = _get_matcher_api_version(matcher) raise ValueError(f"Unsupported API version {api_version}") except: # Show the ugly traceback if the matcher causes an @@ -2817,7 +3157,9 @@ class IPCompleter(Completer): result["matched_fragment"] = result.get("matched_fragment", context.token) if not suppressed_matchers: - suppression_recommended = result.get("suppress", False) + suppression_recommended: Union[bool, Set[str]] = result.get( + "suppress", False + ) suppression_config = ( self.suppress_competing_matchers.get(matcher_id, None) @@ -2830,10 +3172,12 @@ class IPCompleter(Completer): ) and has_any_completions(result) if should_suppress: - suppression_exceptions = result.get("do_not_suppress", set()) - try: + suppression_exceptions: Set[str] = result.get( + "do_not_suppress", set() + ) + if isinstance(suppression_recommended, Iterable): to_suppress = set(suppression_recommended) - except TypeError: + else: to_suppress = set(matchers) suppressed_matchers = to_suppress - suppression_exceptions @@ -2860,9 +3204,9 @@ class IPCompleter(Completer): @staticmethod def _deduplicate( - matches: Sequence[SimpleCompletion], - ) -> Iterable[SimpleCompletion]: - filtered_matches = {} + matches: Sequence[AnyCompletion], + ) -> Iterable[AnyCompletion]: + filtered_matches: Dict[str, AnyCompletion] = {} for match in matches: text = match.text if ( @@ -2874,7 +3218,7 @@ class IPCompleter(Completer): return filtered_matches.values() @staticmethod - def _sort(matches: Sequence[SimpleCompletion]): + def _sort(matches: Sequence[AnyCompletion]): return sorted(matches, key=lambda x: completions_sorting_key(x.text)) @context_matcher() diff --git a/IPython/core/guarded_eval.py b/IPython/core/guarded_eval.py new file mode 100644 index 0000000..d60a5c5 --- /dev/null +++ b/IPython/core/guarded_eval.py @@ -0,0 +1,738 @@ +from typing import ( + Any, + Callable, + Dict, + Set, + Sequence, + Tuple, + NamedTuple, + Type, + Literal, + Union, + TYPE_CHECKING, +) +import ast +import builtins +import collections +import operator +import sys +from functools import cached_property +from dataclasses import dataclass, field + +from IPython.utils.docs import GENERATING_DOCUMENTATION +from IPython.utils.decorators import undoc + + +if TYPE_CHECKING or GENERATING_DOCUMENTATION: + from typing_extensions import Protocol +else: + # do not require on runtime + Protocol = object # requires Python >=3.8 + + +@undoc +class HasGetItem(Protocol): + def __getitem__(self, key) -> None: + ... + + +@undoc +class InstancesHaveGetItem(Protocol): + def __call__(self, *args, **kwargs) -> HasGetItem: + ... + + +@undoc +class HasGetAttr(Protocol): + def __getattr__(self, key) -> None: + ... + + +@undoc +class DoesNotHaveGetAttr(Protocol): + pass + + +# By default `__getattr__` is not explicitly implemented on most objects +MayHaveGetattr = Union[HasGetAttr, DoesNotHaveGetAttr] + + +def _unbind_method(func: Callable) -> Union[Callable, None]: + """Get unbound method for given bound method. + + Returns None if cannot get unbound method, or method is already unbound. + """ + owner = getattr(func, "__self__", None) + owner_class = type(owner) + name = getattr(func, "__name__", None) + instance_dict_overrides = getattr(owner, "__dict__", None) + if ( + owner is not None + and name + and ( + not instance_dict_overrides + or (instance_dict_overrides and name not in instance_dict_overrides) + ) + ): + return getattr(owner_class, name) + return None + + +@undoc +@dataclass +class EvaluationPolicy: + """Definition of evaluation policy.""" + + allow_locals_access: bool = False + allow_globals_access: bool = False + allow_item_access: bool = False + allow_attr_access: bool = False + allow_builtins_access: bool = False + allow_all_operations: bool = False + allow_any_calls: bool = False + allowed_calls: Set[Callable] = field(default_factory=set) + + def can_get_item(self, value, item): + return self.allow_item_access + + def can_get_attr(self, value, attr): + return self.allow_attr_access + + def can_operate(self, dunders: Tuple[str, ...], a, b=None): + if self.allow_all_operations: + return True + + def can_call(self, func): + if self.allow_any_calls: + return True + + if func in self.allowed_calls: + return True + + owner_method = _unbind_method(func) + + if owner_method and owner_method in self.allowed_calls: + return True + + +def _get_external(module_name: str, access_path: Sequence[str]): + """Get value from external module given a dotted access path. + + Raises: + * `KeyError` if module is removed not found, and + * `AttributeError` if acess path does not match an exported object + """ + member_type = sys.modules[module_name] + for attr in access_path: + member_type = getattr(member_type, attr) + return member_type + + +def _has_original_dunder_external( + value, + module_name: str, + access_path: Sequence[str], + method_name: str, +): + if module_name not in sys.modules: + # LBYLB as it is faster + return False + try: + member_type = _get_external(module_name, access_path) + value_type = type(value) + if type(value) == member_type: + return True + if method_name == "__getattribute__": + # we have to short-circuit here due to an unresolved issue in + # `isinstance` implementation: https://bugs.python.org/issue32683 + return False + if isinstance(value, member_type): + method = getattr(value_type, method_name, None) + member_method = getattr(member_type, method_name, None) + if member_method == method: + return True + except (AttributeError, KeyError): + return False + + +def _has_original_dunder( + value, allowed_types, allowed_methods, allowed_external, method_name +): + # note: Python ignores `__getattr__`/`__getitem__` on instances, + # we only need to check at class level + value_type = type(value) + + # strict type check passes → no need to check method + if value_type in allowed_types: + return True + + method = getattr(value_type, method_name, None) + + if method is None: + return None + + if method in allowed_methods: + return True + + for module_name, *access_path in allowed_external: + if _has_original_dunder_external(value, module_name, access_path, method_name): + return True + + return False + + +@undoc +@dataclass +class SelectivePolicy(EvaluationPolicy): + allowed_getitem: Set[InstancesHaveGetItem] = field(default_factory=set) + allowed_getitem_external: Set[Tuple[str, ...]] = field(default_factory=set) + + allowed_getattr: Set[MayHaveGetattr] = field(default_factory=set) + allowed_getattr_external: Set[Tuple[str, ...]] = field(default_factory=set) + + allowed_operations: Set = field(default_factory=set) + allowed_operations_external: Set[Tuple[str, ...]] = field(default_factory=set) + + _operation_methods_cache: Dict[str, Set[Callable]] = field( + default_factory=dict, init=False + ) + + def can_get_attr(self, value, attr): + has_original_attribute = _has_original_dunder( + value, + allowed_types=self.allowed_getattr, + allowed_methods=self._getattribute_methods, + allowed_external=self.allowed_getattr_external, + method_name="__getattribute__", + ) + has_original_attr = _has_original_dunder( + value, + allowed_types=self.allowed_getattr, + allowed_methods=self._getattr_methods, + allowed_external=self.allowed_getattr_external, + method_name="__getattr__", + ) + + accept = False + + # Many objects do not have `__getattr__`, this is fine. + if has_original_attr is None and has_original_attribute: + accept = True + else: + # Accept objects without modifications to `__getattr__` and `__getattribute__` + accept = has_original_attr and has_original_attribute + + if accept: + # We still need to check for overriden properties. + + value_class = type(value) + if not hasattr(value_class, attr): + return True + + class_attr_val = getattr(value_class, attr) + is_property = isinstance(class_attr_val, property) + + if not is_property: + return True + + # Properties in allowed types are ok (although we do not include any + # properties in our default allow list currently). + if type(value) in self.allowed_getattr: + return True # pragma: no cover + + # Properties in subclasses of allowed types may be ok if not changed + for module_name, *access_path in self.allowed_getattr_external: + try: + external_class = _get_external(module_name, access_path) + external_class_attr_val = getattr(external_class, attr) + except (KeyError, AttributeError): + return False # pragma: no cover + return class_attr_val == external_class_attr_val + + return False + + def can_get_item(self, value, item): + """Allow accessing `__getiitem__` of allow-listed instances unless it was not modified.""" + return _has_original_dunder( + value, + allowed_types=self.allowed_getitem, + allowed_methods=self._getitem_methods, + allowed_external=self.allowed_getitem_external, + method_name="__getitem__", + ) + + def can_operate(self, dunders: Tuple[str, ...], a, b=None): + objects = [a] + if b is not None: + objects.append(b) + return all( + [ + _has_original_dunder( + obj, + allowed_types=self.allowed_operations, + allowed_methods=self._operator_dunder_methods(dunder), + allowed_external=self.allowed_operations_external, + method_name=dunder, + ) + for dunder in dunders + for obj in objects + ] + ) + + def _operator_dunder_methods(self, dunder: str) -> Set[Callable]: + if dunder not in self._operation_methods_cache: + self._operation_methods_cache[dunder] = self._safe_get_methods( + self.allowed_operations, dunder + ) + return self._operation_methods_cache[dunder] + + @cached_property + def _getitem_methods(self) -> Set[Callable]: + return self._safe_get_methods(self.allowed_getitem, "__getitem__") + + @cached_property + def _getattr_methods(self) -> Set[Callable]: + return self._safe_get_methods(self.allowed_getattr, "__getattr__") + + @cached_property + def _getattribute_methods(self) -> Set[Callable]: + return self._safe_get_methods(self.allowed_getattr, "__getattribute__") + + def _safe_get_methods(self, classes, name) -> Set[Callable]: + return { + method + for class_ in classes + for method in [getattr(class_, name, None)] + if method + } + + +class _DummyNamedTuple(NamedTuple): + """Used internally to retrieve methods of named tuple instance.""" + + +class EvaluationContext(NamedTuple): + #: Local namespace + locals: dict + #: Global namespace + globals: dict + #: Evaluation policy identifier + evaluation: Literal[ + "forbidden", "minimal", "limited", "unsafe", "dangerous" + ] = "forbidden" + #: Whether the evalution of code takes place inside of a subscript. + #: Useful for evaluating ``:-1, 'col'`` in ``df[:-1, 'col']``. + in_subscript: bool = False + + +class _IdentitySubscript: + """Returns the key itself when item is requested via subscript.""" + + def __getitem__(self, key): + return key + + +IDENTITY_SUBSCRIPT = _IdentitySubscript() +SUBSCRIPT_MARKER = "__SUBSCRIPT_SENTINEL__" + + +class GuardRejection(Exception): + """Exception raised when guard rejects evaluation attempt.""" + + pass + + +def guarded_eval(code: str, context: EvaluationContext): + """Evaluate provided code in the evaluation context. + + If evaluation policy given by context is set to ``forbidden`` + no evaluation will be performed; if it is set to ``dangerous`` + standard :func:`eval` will be used; finally, for any other, + policy :func:`eval_node` will be called on parsed AST. + """ + locals_ = context.locals + + if context.evaluation == "forbidden": + raise GuardRejection("Forbidden mode") + + # note: not using `ast.literal_eval` as it does not implement + # getitem at all, for example it fails on simple `[0][1]` + + if context.in_subscript: + # syntatic sugar for ellipsis (:) is only available in susbcripts + # so we need to trick the ast parser into thinking that we have + # a subscript, but we need to be able to later recognise that we did + # it so we can ignore the actual __getitem__ operation + if not code: + return tuple() + locals_ = locals_.copy() + locals_[SUBSCRIPT_MARKER] = IDENTITY_SUBSCRIPT + code = SUBSCRIPT_MARKER + "[" + code + "]" + context = EvaluationContext(**{**context._asdict(), **{"locals": locals_}}) + + if context.evaluation == "dangerous": + return eval(code, context.globals, context.locals) + + expression = ast.parse(code, mode="eval") + + return eval_node(expression, context) + + +BINARY_OP_DUNDERS: Dict[Type[ast.operator], Tuple[str]] = { + ast.Add: ("__add__",), + ast.Sub: ("__sub__",), + ast.Mult: ("__mul__",), + ast.Div: ("__truediv__",), + ast.FloorDiv: ("__floordiv__",), + ast.Mod: ("__mod__",), + ast.Pow: ("__pow__",), + ast.LShift: ("__lshift__",), + ast.RShift: ("__rshift__",), + ast.BitOr: ("__or__",), + ast.BitXor: ("__xor__",), + ast.BitAnd: ("__and__",), + ast.MatMult: ("__matmul__",), +} + +COMP_OP_DUNDERS: Dict[Type[ast.cmpop], Tuple[str, ...]] = { + ast.Eq: ("__eq__",), + ast.NotEq: ("__ne__", "__eq__"), + ast.Lt: ("__lt__", "__gt__"), + ast.LtE: ("__le__", "__ge__"), + ast.Gt: ("__gt__", "__lt__"), + ast.GtE: ("__ge__", "__le__"), + ast.In: ("__contains__",), + # Note: ast.Is, ast.IsNot, ast.NotIn are handled specially +} + +UNARY_OP_DUNDERS: Dict[Type[ast.unaryop], Tuple[str, ...]] = { + ast.USub: ("__neg__",), + ast.UAdd: ("__pos__",), + # we have to check both __inv__ and __invert__! + ast.Invert: ("__invert__", "__inv__"), + ast.Not: ("__not__",), +} + + +def _find_dunder(node_op, dunders) -> Union[Tuple[str, ...], None]: + dunder = None + for op, candidate_dunder in dunders.items(): + if isinstance(node_op, op): + dunder = candidate_dunder + return dunder + + +def eval_node(node: Union[ast.AST, None], context: EvaluationContext): + """Evaluate AST node in provided context. + + Applies evaluation restrictions defined in the context. Currently does not support evaluation of functions with keyword arguments. + + Does not evaluate actions that always have side effects: + + - class definitions (``class sth: ...``) + - function definitions (``def sth: ...``) + - variable assignments (``x = 1``) + - augmented assignments (``x += 1``) + - deletions (``del x``) + + Does not evaluate operations which do not return values: + + - assertions (``assert x``) + - pass (``pass``) + - imports (``import x``) + - control flow: + + - conditionals (``if x:``) except for ternary IfExp (``a if x else b``) + - loops (``for`` and `while``) + - exception handling + + The purpose of this function is to guard against unwanted side-effects; + it does not give guarantees on protection from malicious code execution. + """ + policy = EVALUATION_POLICIES[context.evaluation] + if node is None: + return None + if isinstance(node, ast.Expression): + return eval_node(node.body, context) + if isinstance(node, ast.BinOp): + left = eval_node(node.left, context) + right = eval_node(node.right, context) + dunders = _find_dunder(node.op, BINARY_OP_DUNDERS) + if dunders: + if policy.can_operate(dunders, left, right): + return getattr(left, dunders[0])(right) + else: + raise GuardRejection( + f"Operation (`{dunders}`) for", + type(left), + f"not allowed in {context.evaluation} mode", + ) + if isinstance(node, ast.Compare): + left = eval_node(node.left, context) + all_true = True + negate = False + for op, right in zip(node.ops, node.comparators): + right = eval_node(right, context) + dunder = None + dunders = _find_dunder(op, COMP_OP_DUNDERS) + if not dunders: + if isinstance(op, ast.NotIn): + dunders = COMP_OP_DUNDERS[ast.In] + negate = True + if isinstance(op, ast.Is): + dunder = "is_" + if isinstance(op, ast.IsNot): + dunder = "is_" + negate = True + if not dunder and dunders: + dunder = dunders[0] + if dunder: + a, b = (right, left) if dunder == "__contains__" else (left, right) + if dunder == "is_" or dunders and policy.can_operate(dunders, a, b): + result = getattr(operator, dunder)(a, b) + if negate: + result = not result + if not result: + all_true = False + left = right + else: + raise GuardRejection( + f"Comparison (`{dunder}`) for", + type(left), + f"not allowed in {context.evaluation} mode", + ) + else: + raise ValueError( + f"Comparison `{dunder}` not supported" + ) # pragma: no cover + return all_true + if isinstance(node, ast.Constant): + return node.value + if isinstance(node, ast.Index): + # deprecated since Python 3.9 + return eval_node(node.value, context) # pragma: no cover + if isinstance(node, ast.Tuple): + return tuple(eval_node(e, context) for e in node.elts) + if isinstance(node, ast.List): + return [eval_node(e, context) for e in node.elts] + if isinstance(node, ast.Set): + return {eval_node(e, context) for e in node.elts} + if isinstance(node, ast.Dict): + return dict( + zip( + [eval_node(k, context) for k in node.keys], + [eval_node(v, context) for v in node.values], + ) + ) + if isinstance(node, ast.Slice): + return slice( + eval_node(node.lower, context), + eval_node(node.upper, context), + eval_node(node.step, context), + ) + if isinstance(node, ast.ExtSlice): + # deprecated since Python 3.9 + return tuple([eval_node(dim, context) for dim in node.dims]) # pragma: no cover + if isinstance(node, ast.UnaryOp): + value = eval_node(node.operand, context) + dunders = _find_dunder(node.op, UNARY_OP_DUNDERS) + if dunders: + if policy.can_operate(dunders, value): + return getattr(value, dunders[0])() + else: + raise GuardRejection( + f"Operation (`{dunders}`) for", + type(value), + f"not allowed in {context.evaluation} mode", + ) + if isinstance(node, ast.Subscript): + value = eval_node(node.value, context) + slice_ = eval_node(node.slice, context) + if policy.can_get_item(value, slice_): + return value[slice_] + raise GuardRejection( + "Subscript access (`__getitem__`) for", + type(value), # not joined to avoid calling `repr` + f" not allowed in {context.evaluation} mode", + ) + if isinstance(node, ast.Name): + if policy.allow_locals_access and node.id in context.locals: + return context.locals[node.id] + if policy.allow_globals_access and node.id in context.globals: + return context.globals[node.id] + if policy.allow_builtins_access and hasattr(builtins, node.id): + # note: do not use __builtins__, it is implementation detail of cPython + return getattr(builtins, node.id) + if not policy.allow_globals_access and not policy.allow_locals_access: + raise GuardRejection( + f"Namespace access not allowed in {context.evaluation} mode" + ) + else: + raise NameError(f"{node.id} not found in locals, globals, nor builtins") + if isinstance(node, ast.Attribute): + value = eval_node(node.value, context) + if policy.can_get_attr(value, node.attr): + return getattr(value, node.attr) + raise GuardRejection( + "Attribute access (`__getattr__`) for", + type(value), # not joined to avoid calling `repr` + f"not allowed in {context.evaluation} mode", + ) + if isinstance(node, ast.IfExp): + test = eval_node(node.test, context) + if test: + return eval_node(node.body, context) + else: + return eval_node(node.orelse, context) + if isinstance(node, ast.Call): + func = eval_node(node.func, context) + if policy.can_call(func) and not node.keywords: + args = [eval_node(arg, context) for arg in node.args] + return func(*args) + raise GuardRejection( + "Call for", + func, # not joined to avoid calling `repr` + f"not allowed in {context.evaluation} mode", + ) + raise ValueError("Unhandled node", ast.dump(node)) + + +SUPPORTED_EXTERNAL_GETITEM = { + ("pandas", "core", "indexing", "_iLocIndexer"), + ("pandas", "core", "indexing", "_LocIndexer"), + ("pandas", "DataFrame"), + ("pandas", "Series"), + ("numpy", "ndarray"), + ("numpy", "void"), +} + + +BUILTIN_GETITEM: Set[InstancesHaveGetItem] = { + dict, + str, + bytes, + list, + tuple, + collections.defaultdict, + collections.deque, + collections.OrderedDict, + collections.ChainMap, + collections.UserDict, + collections.UserList, + collections.UserString, + _DummyNamedTuple, + _IdentitySubscript, +} + + +def _list_methods(cls, source=None): + """For use on immutable objects or with methods returning a copy""" + return [getattr(cls, k) for k in (source if source else dir(cls))] + + +dict_non_mutating_methods = ("copy", "keys", "values", "items") +list_non_mutating_methods = ("copy", "index", "count") +set_non_mutating_methods = set(dir(set)) & set(dir(frozenset)) + + +dict_keys: Type[collections.abc.KeysView] = type({}.keys()) +method_descriptor: Any = type(list.copy) + +NUMERICS = {int, float, complex} + +ALLOWED_CALLS = { + bytes, + *_list_methods(bytes), + dict, + *_list_methods(dict, dict_non_mutating_methods), + dict_keys.isdisjoint, + list, + *_list_methods(list, list_non_mutating_methods), + set, + *_list_methods(set, set_non_mutating_methods), + frozenset, + *_list_methods(frozenset), + range, + str, + *_list_methods(str), + tuple, + *_list_methods(tuple), + *NUMERICS, + *[method for numeric_cls in NUMERICS for method in _list_methods(numeric_cls)], + collections.deque, + *_list_methods(collections.deque, list_non_mutating_methods), + collections.defaultdict, + *_list_methods(collections.defaultdict, dict_non_mutating_methods), + collections.OrderedDict, + *_list_methods(collections.OrderedDict, dict_non_mutating_methods), + collections.UserDict, + *_list_methods(collections.UserDict, dict_non_mutating_methods), + collections.UserList, + *_list_methods(collections.UserList, list_non_mutating_methods), + collections.UserString, + *_list_methods(collections.UserString, dir(str)), + collections.Counter, + *_list_methods(collections.Counter, dict_non_mutating_methods), + collections.Counter.elements, + collections.Counter.most_common, +} + +BUILTIN_GETATTR: Set[MayHaveGetattr] = { + *BUILTIN_GETITEM, + set, + frozenset, + object, + type, # `type` handles a lot of generic cases, e.g. numbers as in `int.real`. + *NUMERICS, + dict_keys, + method_descriptor, +} + + +BUILTIN_OPERATIONS = {*BUILTIN_GETATTR} + +EVALUATION_POLICIES = { + "minimal": EvaluationPolicy( + allow_builtins_access=True, + allow_locals_access=False, + allow_globals_access=False, + allow_item_access=False, + allow_attr_access=False, + allowed_calls=set(), + allow_any_calls=False, + allow_all_operations=False, + ), + "limited": SelectivePolicy( + allowed_getitem=BUILTIN_GETITEM, + allowed_getitem_external=SUPPORTED_EXTERNAL_GETITEM, + allowed_getattr=BUILTIN_GETATTR, + allowed_getattr_external={ + # pandas Series/Frame implements custom `__getattr__` + ("pandas", "DataFrame"), + ("pandas", "Series"), + }, + allowed_operations=BUILTIN_OPERATIONS, + allow_builtins_access=True, + allow_locals_access=True, + allow_globals_access=True, + allowed_calls=ALLOWED_CALLS, + ), + "unsafe": EvaluationPolicy( + allow_builtins_access=True, + allow_locals_access=True, + allow_globals_access=True, + allow_attr_access=True, + allow_item_access=True, + allow_any_calls=True, + allow_all_operations=True, + ), +} + + +__all__ = [ + "guarded_eval", + "eval_node", + "GuardRejection", + "EvaluationContext", + "_unbind_method", +] diff --git a/IPython/core/interactiveshell.py b/IPython/core/interactiveshell.py index 21e428b..b1a770b 100644 --- a/IPython/core/interactiveshell.py +++ b/IPython/core/interactiveshell.py @@ -389,6 +389,9 @@ class InteractiveShell(SingletonConfigurable): displayhook_class = Type(DisplayHook) display_pub_class = Type(DisplayPublisher) compiler_class = Type(CachingCompiler) + inspector_class = Type( + oinspect.Inspector, help="Class to use to instantiate the shell inspector" + ).tag(config=True) sphinxify_docstring = Bool(False, help= """ @@ -755,10 +758,12 @@ class InteractiveShell(SingletonConfigurable): @observe('colors') def init_inspector(self, changes=None): # Object inspector - self.inspector = oinspect.Inspector(oinspect.InspectColors, - PyColorize.ANSICodeColors, - self.colors, - self.object_info_string_level) + self.inspector = self.inspector_class( + oinspect.InspectColors, + PyColorize.ANSICodeColors, + self.colors, + self.object_info_string_level, + ) def init_io(self): # implemented in subclasses, TerminalInteractiveShell does call @@ -3138,8 +3143,12 @@ class InteractiveShell(SingletonConfigurable): else: cell = raw_cell + # Do NOT store paste/cpaste magic history + if "get_ipython().run_line_magic(" in cell and "paste" in cell: + store_history = False + # Store raw and processed history - if store_history and raw_cell.strip(" %") != "paste": + if store_history: self.history_manager.store_inputs(self.execution_count, cell, raw_cell) if not silent: self.logger.log(cell, raw_cell) diff --git a/IPython/core/magics/config.py b/IPython/core/magics/config.py index 87fe3ee..9e1cb38 100644 --- a/IPython/core/magics/config.py +++ b/IPython/core/magics/config.py @@ -68,94 +68,22 @@ class ConfigMagics(Magics): To view what is configurable on a given class, just pass the class name:: - In [2]: %config IPCompleter - IPCompleter(Completer) options - ---------------------------- - IPCompleter.backslash_combining_completions= - Enable unicode completions, e.g. \\alpha . Includes completion of latex - commands, unicode names, and expanding unicode characters back to latex - commands. - Current: True - IPCompleter.debug= - Enable debug for the Completer. Mostly print extra information for - experimental jedi integration. + In [2]: %config LoggingMagics + LoggingMagics(Magics) options + --------------------------- + LoggingMagics.quiet= + Suppress output of log state when logging is enabled Current: False - IPCompleter.disable_matchers=... - List of matchers to disable. - The list should contain matcher identifiers (see - :any:`completion_matcher`). - Current: [] - IPCompleter.greedy= - Activate greedy completion - PENDING DEPRECATION. this is now mostly taken care of with Jedi. - This will enable completion on elements of lists, results of function calls, etc., - but can be unsafe because the code is actually evaluated on TAB. - Current: False - IPCompleter.jedi_compute_type_timeout= - Experimental: restrict time (in milliseconds) during which Jedi can compute types. - Set to 0 to stop computing types. Non-zero value lower than 100ms may hurt - performance by preventing jedi to build its cache. - Current: 400 - IPCompleter.limit_to__all__= - DEPRECATED as of version 5.0. - Instruct the completer to use __all__ for the completion - Specifically, when completing on ``object.``. - When True: only those names in obj.__all__ will be included. - When False [default]: the __all__ attribute is ignored - Current: False - IPCompleter.merge_completions= - Whether to merge completion results into a single list - If False, only the completion results from the first non-empty - completer will be returned. - As of version 8.6.0, setting the value to ``False`` is an alias for: - ``IPCompleter.suppress_competing_matchers = True.``. - Current: True - IPCompleter.omit__names= - Instruct the completer to omit private method names - Specifically, when completing on ``object.``. - When 2 [default]: all names that start with '_' will be excluded. - When 1: all 'magic' names (``__foo__``) will be excluded. - When 0: nothing will be excluded. - Choices: any of [0, 1, 2] - Current: 2 - IPCompleter.profile_completions= - If True, emit profiling data for completion subsystem using cProfile. - Current: False - IPCompleter.profiler_output_dir= - Template for path at which to output profile data for completions. - Current: '.completion_profiles' - IPCompleter.suppress_competing_matchers= - Whether to suppress completions from other *Matchers*. - When set to ``None`` (default) the matchers will attempt to auto-detect - whether suppression of other matchers is desirable. For example, at the - beginning of a line followed by `%` we expect a magic completion to be the - only applicable option, and after ``my_dict['`` we usually expect a - completion with an existing dictionary key. - If you want to disable this heuristic and see completions from all matchers, - set ``IPCompleter.suppress_competing_matchers = False``. To disable the - heuristic for specific matchers provide a dictionary mapping: - ``IPCompleter.suppress_competing_matchers = {'IPCompleter.dict_key_matcher': - False}``. - Set ``IPCompleter.suppress_competing_matchers = True`` to limit completions - to the set of matchers with the highest priority; this is equivalent to - ``IPCompleter.merge_completions`` and can be beneficial for performance, but - will sometimes omit relevant candidates from matchers further down the - priority list. - Current: None - IPCompleter.use_jedi= - Experimental: Use Jedi to generate autocompletions. Default to True if jedi - is installed. - Current: True but the real use is in setting values:: - In [3]: %config IPCompleter.greedy = True + In [3]: %config LoggingMagics.quiet = True and these values are read from the user_ns if they are variables:: - In [4]: feeling_greedy=False + In [4]: feeling_quiet=False - In [5]: %config IPCompleter.greedy = feeling_greedy + In [5]: %config LoggingMagics.quiet = feeling_quiet """ from traitlets.config.loader import Config diff --git a/IPython/core/oinspect.py b/IPython/core/oinspect.py index f1c454b..bcaa95c 100644 --- a/IPython/core/oinspect.py +++ b/IPython/core/oinspect.py @@ -16,6 +16,7 @@ __all__ = ['Inspector','InspectColors'] import ast import inspect from inspect import signature +import html import linecache import warnings import os @@ -530,8 +531,8 @@ class Inspector(Colorable): """ defaults = { - 'text/plain': text, - 'text/html': '
' + text + '
' + "text/plain": text, + "text/html": f"
{html.escape(text)}
", } if formatter is None: @@ -542,66 +543,66 @@ class Inspector(Colorable): if not isinstance(formatted, dict): # Handle the deprecated behavior of a formatter returning # a string instead of a mime bundle. - return { - 'text/plain': formatted, - 'text/html': '
' + formatted + '
' - } + return {"text/plain": formatted, "text/html": f"
{formatted}
"} else: return dict(defaults, **formatted) def format_mime(self, bundle): - - text_plain = bundle['text/plain'] - - text = '' - heads, bodies = list(zip(*text_plain)) - _len = max(len(h) for h in heads) - - for head, body in zip(heads, bodies): - body = body.strip('\n') - delim = '\n' if '\n' in body else ' ' - text += self.__head(head+':') + (_len - len(head))*' ' +delim + body +'\n' - - bundle['text/plain'] = text + """Format a mimebundle being created by _make_info_unformatted into a real mimebundle""" + # Format text/plain mimetype + if isinstance(bundle["text/plain"], (list, tuple)): + # bundle['text/plain'] is a list of (head, formatted body) pairs + lines = [] + _len = max(len(h) for h, _ in bundle["text/plain"]) + + for head, body in bundle["text/plain"]: + body = body.strip("\n") + delim = "\n" if "\n" in body else " " + lines.append( + f"{self.__head(head+':')}{(_len - len(head))*' '}{delim}{body}" + ) + + bundle["text/plain"] = "\n".join(lines) + + # Format the text/html mimetype + if isinstance(bundle["text/html"], (list, tuple)): + # bundle['text/html'] is a list of (head, formatted body) pairs + bundle["text/html"] = "\n".join( + (f"

{head}

\n{body}" for (head, body) in bundle["text/html"]) + ) return bundle - def _get_info( - self, obj, oname="", formatter=None, info=None, detail_level=0, omit_sections=() + def _append_info_field( + self, bundle, title: str, key: str, info, omit_sections, formatter ): - """Retrieve an info dict and format it. - - Parameters - ---------- - obj : any - Object to inspect and return info from - oname : str (default: ''): - Name of the variable pointing to `obj`. - formatter : callable - info - already computed information - detail_level : integer - Granularity of detail level, if set to 1, give more information. - omit_sections : container[str] - Titles or keys to omit from output (can be set, tuple, etc., anything supporting `in`) - """ - - info = self.info(obj, oname=oname, info=info, detail_level=detail_level) - - _mime = { - 'text/plain': [], - 'text/html': '', + """Append an info value to the unformatted mimebundle being constructed by _make_info_unformatted""" + if title in omit_sections or key in omit_sections: + return + field = info[key] + if field is not None: + formatted_field = self._mime_format(field, formatter) + bundle["text/plain"].append((title, formatted_field["text/plain"])) + bundle["text/html"].append((title, formatted_field["text/html"])) + + def _make_info_unformatted(self, obj, info, formatter, detail_level, omit_sections): + """Assemble the mimebundle as unformatted lists of information""" + bundle = { + "text/plain": [], + "text/html": [], } - def append_field(bundle, title:str, key:str, formatter=None): - if title in omit_sections or key in omit_sections: - return - field = info[key] - if field is not None: - formatted_field = self._mime_format(field, formatter) - bundle['text/plain'].append((title, formatted_field['text/plain'])) - bundle['text/html'] += '

' + title + '

\n' + formatted_field['text/html'] + '\n' + # A convenience function to simplify calls below + def append_field(bundle, title: str, key: str, formatter=None): + self._append_info_field( + bundle, + title=title, + key=key, + info=info, + omit_sections=omit_sections, + formatter=formatter, + ) def code_formatter(text): return { @@ -609,57 +610,82 @@ class Inspector(Colorable): 'text/html': pylight(text) } - if info['isalias']: - append_field(_mime, 'Repr', 'string_form') + if info["isalias"]: + append_field(bundle, "Repr", "string_form") elif info['ismagic']: if detail_level > 0: - append_field(_mime, 'Source', 'source', code_formatter) + append_field(bundle, "Source", "source", code_formatter) else: - append_field(_mime, 'Docstring', 'docstring', formatter) - append_field(_mime, 'File', 'file') + append_field(bundle, "Docstring", "docstring", formatter) + append_field(bundle, "File", "file") elif info['isclass'] or is_simple_callable(obj): # Functions, methods, classes - append_field(_mime, 'Signature', 'definition', code_formatter) - append_field(_mime, 'Init signature', 'init_definition', code_formatter) - append_field(_mime, 'Docstring', 'docstring', formatter) - if detail_level > 0 and info['source']: - append_field(_mime, 'Source', 'source', code_formatter) + append_field(bundle, "Signature", "definition", code_formatter) + append_field(bundle, "Init signature", "init_definition", code_formatter) + append_field(bundle, "Docstring", "docstring", formatter) + if detail_level > 0 and info["source"]: + append_field(bundle, "Source", "source", code_formatter) else: - append_field(_mime, 'Init docstring', 'init_docstring', formatter) + append_field(bundle, "Init docstring", "init_docstring", formatter) - append_field(_mime, 'File', 'file') - append_field(_mime, 'Type', 'type_name') - append_field(_mime, 'Subclasses', 'subclasses') + append_field(bundle, "File", "file") + append_field(bundle, "Type", "type_name") + append_field(bundle, "Subclasses", "subclasses") else: # General Python objects - append_field(_mime, 'Signature', 'definition', code_formatter) - append_field(_mime, 'Call signature', 'call_def', code_formatter) - append_field(_mime, 'Type', 'type_name') - append_field(_mime, 'String form', 'string_form') + append_field(bundle, "Signature", "definition", code_formatter) + append_field(bundle, "Call signature", "call_def", code_formatter) + append_field(bundle, "Type", "type_name") + append_field(bundle, "String form", "string_form") # Namespace - if info['namespace'] != 'Interactive': - append_field(_mime, 'Namespace', 'namespace') + if info["namespace"] != "Interactive": + append_field(bundle, "Namespace", "namespace") - append_field(_mime, 'Length', 'length') - append_field(_mime, 'File', 'file') + append_field(bundle, "Length", "length") + append_field(bundle, "File", "file") # Source or docstring, depending on detail level and whether # source found. - if detail_level > 0 and info['source']: - append_field(_mime, 'Source', 'source', code_formatter) + if detail_level > 0 and info["source"]: + append_field(bundle, "Source", "source", code_formatter) else: - append_field(_mime, 'Docstring', 'docstring', formatter) + append_field(bundle, "Docstring", "docstring", formatter) + + append_field(bundle, "Class docstring", "class_docstring", formatter) + append_field(bundle, "Init docstring", "init_docstring", formatter) + append_field(bundle, "Call docstring", "call_docstring", formatter) + return bundle - append_field(_mime, 'Class docstring', 'class_docstring', formatter) - append_field(_mime, 'Init docstring', 'init_docstring', formatter) - append_field(_mime, 'Call docstring', 'call_docstring', formatter) + def _get_info( + self, obj, oname="", formatter=None, info=None, detail_level=0, omit_sections=() + ): + """Retrieve an info dict and format it. + + Parameters + ---------- + obj : any + Object to inspect and return info from + oname : str (default: ''): + Name of the variable pointing to `obj`. + formatter : callable + info + already computed information + detail_level : integer + Granularity of detail level, if set to 1, give more information. + omit_sections : container[str] + Titles or keys to omit from output (can be set, tuple, etc., anything supporting `in`) + """ - return self.format_mime(_mime) + info = self.info(obj, oname=oname, info=info, detail_level=detail_level) + bundle = self._make_info_unformatted( + obj, info, formatter, detail_level=detail_level, omit_sections=omit_sections + ) + return self.format_mime(bundle) def pinfo( self, diff --git a/IPython/core/tests/test_completer.py b/IPython/core/tests/test_completer.py index 98ec814..5e8cb35 100644 --- a/IPython/core/tests/test_completer.py +++ b/IPython/core/tests/test_completer.py @@ -24,6 +24,7 @@ from IPython.core.completer import ( provisionalcompleter, match_dict_keys, _deduplicate_completions, + _match_number_in_dict_key_prefix, completion_matcher, SimpleCompletion, CompletionContext, @@ -113,6 +114,17 @@ def greedy_completion(): @contextmanager +def evaluation_policy(evaluation: str): + ip = get_ipython() + evaluation_original = ip.Completer.evaluation + try: + ip.Completer.evaluation = evaluation + yield + finally: + ip.Completer.evaluation = evaluation_original + + +@contextmanager def custom_matchers(matchers): ip = get_ipython() try: @@ -170,7 +182,6 @@ def check_line_split(splitter, test_specs): out = splitter.split_line(line, cursor_pos) assert out == split - def test_line_split(): """Basic line splitter test with default specs.""" sp = completer.CompletionSplitter() @@ -522,10 +533,10 @@ class TestCompleter(unittest.TestCase): def test_greedy_completions(self): """ - Test the capability of the Greedy completer. + Test the capability of the Greedy completer. Most of the test here does not really show off the greedy completer, for proof - each of the text below now pass with Jedi. The greedy completer is capable of more. + each of the text below now pass with Jedi. The greedy completer is capable of more. See the :any:`test_dict_key_completion_contexts` @@ -841,18 +852,45 @@ class TestCompleter(unittest.TestCase): """ delims = " \t\n`!@#$^&*()=+[{]}\\|;:'\",<>?" - keys = ["foo", b"far"] - assert match_dict_keys(keys, "b'", delims=delims) == ("'", 2, ["far"]) - assert match_dict_keys(keys, "b'f", delims=delims) == ("'", 2, ["far"]) - assert match_dict_keys(keys, 'b"', delims=delims) == ('"', 2, ["far"]) - assert match_dict_keys(keys, 'b"f', delims=delims) == ('"', 2, ["far"]) - - assert match_dict_keys(keys, "'", delims=delims) == ("'", 1, ["foo"]) - assert match_dict_keys(keys, "'f", delims=delims) == ("'", 1, ["foo"]) - assert match_dict_keys(keys, '"', delims=delims) == ('"', 1, ["foo"]) - assert match_dict_keys(keys, '"f', delims=delims) == ('"', 1, ["foo"]) + def match(*args, **kwargs): + quote, offset, matches = match_dict_keys(*args, delims=delims, **kwargs) + return quote, offset, list(matches) - match_dict_keys + keys = ["foo", b"far"] + assert match(keys, "b'") == ("'", 2, ["far"]) + assert match(keys, "b'f") == ("'", 2, ["far"]) + assert match(keys, 'b"') == ('"', 2, ["far"]) + assert match(keys, 'b"f') == ('"', 2, ["far"]) + + assert match(keys, "'") == ("'", 1, ["foo"]) + assert match(keys, "'f") == ("'", 1, ["foo"]) + assert match(keys, '"') == ('"', 1, ["foo"]) + assert match(keys, '"f') == ('"', 1, ["foo"]) + + # Completion on first item of tuple + keys = [("foo", 1111), ("foo", 2222), (3333, "bar"), (3333, "test")] + assert match(keys, "'f") == ("'", 1, ["foo"]) + assert match(keys, "33") == ("", 0, ["3333"]) + + # Completion on numbers + keys = [ + 0xDEADBEEF, + 1111, + 1234, + "1999", + 0b10101, + 22, + ] # 0xDEADBEEF = 3735928559; 0b10101 = 21 + assert match(keys, "0xdead") == ("", 0, ["0xdeadbeef"]) + assert match(keys, "1") == ("", 0, ["1111", "1234"]) + assert match(keys, "2") == ("", 0, ["21", "22"]) + assert match(keys, "0b101") == ("", 0, ["0b10101", "0b10110"]) + + # Should yield on variables + assert match(keys, "a_variable") == ("", 0, []) + + # Should pass over invalid literals + assert match(keys, "'' ''") == ("", 0, []) def test_match_dict_keys_tuple(self): """ @@ -860,28 +898,94 @@ class TestCompleter(unittest.TestCase): does return what expected, and does not crash. """ delims = " \t\n`!@#$^&*()=+[{]}\\|;:'\",<>?" - + keys = [("foo", "bar"), ("foo", "oof"), ("foo", b"bar"), ('other', 'test')] + def match(*args, extra=None, **kwargs): + quote, offset, matches = match_dict_keys( + *args, delims=delims, extra_prefix=extra, **kwargs + ) + return quote, offset, list(matches) + # Completion on first key == "foo" - assert match_dict_keys(keys, "'", delims=delims, extra_prefix=("foo",)) == ("'", 1, ["bar", "oof"]) - assert match_dict_keys(keys, "\"", delims=delims, extra_prefix=("foo",)) == ("\"", 1, ["bar", "oof"]) - assert match_dict_keys(keys, "'o", delims=delims, extra_prefix=("foo",)) == ("'", 1, ["oof"]) - assert match_dict_keys(keys, "\"o", delims=delims, extra_prefix=("foo",)) == ("\"", 1, ["oof"]) - assert match_dict_keys(keys, "b'", delims=delims, extra_prefix=("foo",)) == ("'", 2, ["bar"]) - assert match_dict_keys(keys, "b\"", delims=delims, extra_prefix=("foo",)) == ("\"", 2, ["bar"]) - assert match_dict_keys(keys, "b'b", delims=delims, extra_prefix=("foo",)) == ("'", 2, ["bar"]) - assert match_dict_keys(keys, "b\"b", delims=delims, extra_prefix=("foo",)) == ("\"", 2, ["bar"]) + assert match(keys, "'", extra=("foo",)) == ("'", 1, ["bar", "oof"]) + assert match(keys, '"', extra=("foo",)) == ('"', 1, ["bar", "oof"]) + assert match(keys, "'o", extra=("foo",)) == ("'", 1, ["oof"]) + assert match(keys, '"o', extra=("foo",)) == ('"', 1, ["oof"]) + assert match(keys, "b'", extra=("foo",)) == ("'", 2, ["bar"]) + assert match(keys, 'b"', extra=("foo",)) == ('"', 2, ["bar"]) + assert match(keys, "b'b", extra=("foo",)) == ("'", 2, ["bar"]) + assert match(keys, 'b"b', extra=("foo",)) == ('"', 2, ["bar"]) # No Completion - assert match_dict_keys(keys, "'", delims=delims, extra_prefix=("no_foo",)) == ("'", 1, []) - assert match_dict_keys(keys, "'", delims=delims, extra_prefix=("fo",)) == ("'", 1, []) + assert match(keys, "'", extra=("no_foo",)) == ("'", 1, []) + assert match(keys, "'", extra=("fo",)) == ("'", 1, []) + + keys = [("foo1", "foo2", "foo3", "foo4"), ("foo1", "foo2", "bar", "foo4")] + assert match(keys, "'foo", extra=("foo1",)) == ("'", 1, ["foo2"]) + assert match(keys, "'foo", extra=("foo1", "foo2")) == ("'", 1, ["foo3"]) + assert match(keys, "'foo", extra=("foo1", "foo2", "foo3")) == ("'", 1, ["foo4"]) + assert match(keys, "'foo", extra=("foo1", "foo2", "foo3", "foo4")) == ( + "'", + 1, + [], + ) + + keys = [("foo", 1111), ("foo", "2222"), (3333, "bar"), (3333, 4444)] + assert match(keys, "'", extra=("foo",)) == ("'", 1, ["2222"]) + assert match(keys, "", extra=("foo",)) == ("", 0, ["1111", "'2222'"]) + assert match(keys, "'", extra=(3333,)) == ("'", 1, ["bar"]) + assert match(keys, "", extra=(3333,)) == ("", 0, ["'bar'", "4444"]) + assert match(keys, "'", extra=("3333",)) == ("'", 1, []) + assert match(keys, "33") == ("", 0, ["3333"]) + + def test_dict_key_completion_closures(self): + ip = get_ipython() + complete = ip.Completer.complete + ip.Completer.auto_close_dict_keys = True - keys = [('foo1', 'foo2', 'foo3', 'foo4'), ('foo1', 'foo2', 'bar', 'foo4')] - assert match_dict_keys(keys, "'foo", delims=delims, extra_prefix=('foo1',)) == ("'", 1, ["foo2", "foo2"]) - assert match_dict_keys(keys, "'foo", delims=delims, extra_prefix=('foo1', 'foo2')) == ("'", 1, ["foo3"]) - assert match_dict_keys(keys, "'foo", delims=delims, extra_prefix=('foo1', 'foo2', 'foo3')) == ("'", 1, ["foo4"]) - assert match_dict_keys(keys, "'foo", delims=delims, extra_prefix=('foo1', 'foo2', 'foo3', 'foo4')) == ("'", 1, []) + ip.user_ns["d"] = { + # tuple only + ("aa", 11): None, + # tuple and non-tuple + ("bb", 22): None, + "bb": None, + # non-tuple only + "cc": None, + # numeric tuple only + (77, "x"): None, + # numeric tuple and non-tuple + (88, "y"): None, + 88: None, + # numeric non-tuple only + 99: None, + } + + _, matches = complete(line_buffer="d[") + # should append `, ` if matches a tuple only + self.assertIn("'aa', ", matches) + # should not append anything if matches a tuple and an item + self.assertIn("'bb'", matches) + # should append `]` if matches and item only + self.assertIn("'cc']", matches) + + # should append `, ` if matches a tuple only + self.assertIn("77, ", matches) + # should not append anything if matches a tuple and an item + self.assertIn("88", matches) + # should append `]` if matches and item only + self.assertIn("99]", matches) + + _, matches = complete(line_buffer="d['aa', ") + # should restrict matches to those matching tuple prefix + self.assertIn("11]", matches) + self.assertNotIn("'bb'", matches) + self.assertNotIn("'bb', ", matches) + self.assertNotIn("'bb']", matches) + self.assertNotIn("'cc'", matches) + self.assertNotIn("'cc', ", matches) + self.assertNotIn("'cc']", matches) + ip.Completer.auto_close_dict_keys = False def test_dict_key_completion_string(self): """Test dictionary key completion for string keys""" @@ -1038,6 +1142,35 @@ class TestCompleter(unittest.TestCase): self.assertNotIn("foo", matches) self.assertNotIn("bar", matches) + def test_dict_key_completion_numbers(self): + ip = get_ipython() + complete = ip.Completer.complete + + ip.user_ns["d"] = { + 0xDEADBEEF: None, # 3735928559 + 1111: None, + 1234: None, + "1999": None, + 0b10101: None, # 21 + 22: None, + } + _, matches = complete(line_buffer="d[1") + self.assertIn("1111", matches) + self.assertIn("1234", matches) + self.assertNotIn("1999", matches) + self.assertNotIn("'1999'", matches) + + _, matches = complete(line_buffer="d[0xdead") + self.assertIn("0xdeadbeef", matches) + + _, matches = complete(line_buffer="d[2") + self.assertIn("21", matches) + self.assertIn("22", matches) + + _, matches = complete(line_buffer="d[0b101") + self.assertIn("0b10101", matches) + self.assertIn("0b10110", matches) + def test_dict_key_completion_contexts(self): """Test expression contexts in which dict key completion occurs""" ip = get_ipython() @@ -1050,6 +1183,7 @@ class TestCompleter(unittest.TestCase): ip.user_ns["C"] = C ip.user_ns["get"] = lambda: d + ip.user_ns["nested"] = {"x": d} def assert_no_completion(**kwargs): _, matches = complete(**kwargs) @@ -1075,6 +1209,13 @@ class TestCompleter(unittest.TestCase): assert_completion(line_buffer="(d[") assert_completion(line_buffer="C.data[") + # nested dict completion + assert_completion(line_buffer="nested['x'][") + + with evaluation_policy("minimal"): + with pytest.raises(AssertionError): + assert_completion(line_buffer="nested['x'][") + # greedy flag def assert_completion(**kwargs): _, matches = complete(**kwargs) @@ -1162,12 +1303,22 @@ class TestCompleter(unittest.TestCase): _, matches = complete(line_buffer="d['") self.assertIn("my_head", matches) self.assertIn("my_data", matches) - # complete on a nested level - with greedy_completion(): + + def completes_on_nested(): ip.user_ns["d"] = numpy.zeros(2, dtype=dt) _, matches = complete(line_buffer="d[1]['my_head']['") self.assertTrue(any(["my_dt" in m for m in matches])) self.assertTrue(any(["my_df" in m for m in matches])) + # complete on a nested level + with greedy_completion(): + completes_on_nested() + + with evaluation_policy("limited"): + completes_on_nested() + + with evaluation_policy("minimal"): + with pytest.raises(AssertionError): + completes_on_nested() @dec.skip_without("pandas") def test_dataframe_key_completion(self): @@ -1180,6 +1331,17 @@ class TestCompleter(unittest.TestCase): _, matches = complete(line_buffer="d['") self.assertIn("hello", matches) self.assertIn("world", matches) + _, matches = complete(line_buffer="d.loc[:, '") + self.assertIn("hello", matches) + self.assertIn("world", matches) + _, matches = complete(line_buffer="d.loc[1:, '") + self.assertIn("hello", matches) + _, matches = complete(line_buffer="d.loc[1:1, '") + self.assertIn("hello", matches) + _, matches = complete(line_buffer="d.loc[1:1:-1, '") + self.assertIn("hello", matches) + _, matches = complete(line_buffer="d.loc[::, '") + self.assertIn("hello", matches) def test_dict_key_completion_invalids(self): """Smoke test cases dict key completion can't handle""" @@ -1503,3 +1665,38 @@ class TestCompleter(unittest.TestCase): _(["completion_b"]) a_matcher.matcher_priority = 3 _(["completion_a"]) + + +@pytest.mark.parametrize( + "input, expected", + [ + ["1.234", "1.234"], + # should match signed numbers + ["+1", "+1"], + ["-1", "-1"], + ["-1.0", "-1.0"], + ["-1.", "-1."], + ["+1.", "+1."], + [".1", ".1"], + # should not match non-numbers + ["1..", None], + ["..", None], + [".1.", None], + # should match after comma + [",1", "1"], + [", 1", "1"], + [", .1", ".1"], + [", +.1", "+.1"], + # should not match after trailing spaces + [".1 ", None], + # some complex cases + ["0b_0011_1111_0100_1110", "0b_0011_1111_0100_1110"], + ["0xdeadbeef", "0xdeadbeef"], + ["0b_1110_0101", "0b_1110_0101"], + # should not match if in an operation + ["1 + 1", None], + [", 1 + 1", None], + ], +) +def test_match_numeric_literal_for_dict_key(input, expected): + assert _match_number_in_dict_key_prefix(input) == expected diff --git a/IPython/core/tests/test_guarded_eval.py b/IPython/core/tests/test_guarded_eval.py new file mode 100644 index 0000000..905cf3a --- /dev/null +++ b/IPython/core/tests/test_guarded_eval.py @@ -0,0 +1,570 @@ +from contextlib import contextmanager +from typing import NamedTuple +from functools import partial +from IPython.core.guarded_eval import ( + EvaluationContext, + GuardRejection, + guarded_eval, + _unbind_method, +) +from IPython.testing import decorators as dec +import pytest + + +def create_context(evaluation: str, **kwargs): + return EvaluationContext(locals=kwargs, globals={}, evaluation=evaluation) + + +forbidden = partial(create_context, "forbidden") +minimal = partial(create_context, "minimal") +limited = partial(create_context, "limited") +unsafe = partial(create_context, "unsafe") +dangerous = partial(create_context, "dangerous") + +LIMITED_OR_HIGHER = [limited, unsafe, dangerous] +MINIMAL_OR_HIGHER = [minimal, *LIMITED_OR_HIGHER] + + +@contextmanager +def module_not_installed(module: str): + import sys + + try: + to_restore = sys.modules[module] + del sys.modules[module] + except KeyError: + to_restore = None + try: + yield + finally: + sys.modules[module] = to_restore + + +def test_external_not_installed(): + """ + Because attribute check requires checking if object is not of allowed + external type, this tests logic for absence of external module. + """ + + class Custom: + def __init__(self): + self.test = 1 + + def __getattr__(self, key): + return key + + with module_not_installed("pandas"): + context = limited(x=Custom()) + with pytest.raises(GuardRejection): + guarded_eval("x.test", context) + + +@dec.skip_without("pandas") +def test_external_changed_api(monkeypatch): + """Check that the execution rejects if external API changed paths""" + import pandas as pd + + series = pd.Series([1], index=["a"]) + + with monkeypatch.context() as m: + m.delattr(pd, "Series") + context = limited(data=series) + with pytest.raises(GuardRejection): + guarded_eval("data.iloc[0]", context) + + +@dec.skip_without("pandas") +def test_pandas_series_iloc(): + import pandas as pd + + series = pd.Series([1], index=["a"]) + context = limited(data=series) + assert guarded_eval("data.iloc[0]", context) == 1 + + +def test_rejects_custom_properties(): + class BadProperty: + @property + def iloc(self): + return [None] + + series = BadProperty() + context = limited(data=series) + + with pytest.raises(GuardRejection): + guarded_eval("data.iloc[0]", context) + + +@dec.skip_without("pandas") +def test_accepts_non_overriden_properties(): + import pandas as pd + + class GoodProperty(pd.Series): + pass + + series = GoodProperty([1], index=["a"]) + context = limited(data=series) + + assert guarded_eval("data.iloc[0]", context) == 1 + + +@dec.skip_without("pandas") +def test_pandas_series(): + import pandas as pd + + context = limited(data=pd.Series([1], index=["a"])) + assert guarded_eval('data["a"]', context) == 1 + with pytest.raises(KeyError): + guarded_eval('data["c"]', context) + + +@dec.skip_without("pandas") +def test_pandas_bad_series(): + import pandas as pd + + class BadItemSeries(pd.Series): + def __getitem__(self, key): + return "CUSTOM_ITEM" + + class BadAttrSeries(pd.Series): + def __getattr__(self, key): + return "CUSTOM_ATTR" + + bad_series = BadItemSeries([1], index=["a"]) + context = limited(data=bad_series) + + with pytest.raises(GuardRejection): + guarded_eval('data["a"]', context) + with pytest.raises(GuardRejection): + guarded_eval('data["c"]', context) + + # note: here result is a bit unexpected because + # pandas `__getattr__` calls `__getitem__`; + # FIXME - special case to handle it? + assert guarded_eval("data.a", context) == "CUSTOM_ITEM" + + context = unsafe(data=bad_series) + assert guarded_eval('data["a"]', context) == "CUSTOM_ITEM" + + bad_attr_series = BadAttrSeries([1], index=["a"]) + context = limited(data=bad_attr_series) + assert guarded_eval('data["a"]', context) == 1 + with pytest.raises(GuardRejection): + guarded_eval("data.a", context) + + +@dec.skip_without("pandas") +def test_pandas_dataframe_loc(): + import pandas as pd + from pandas.testing import assert_series_equal + + data = pd.DataFrame([{"a": 1}]) + context = limited(data=data) + assert_series_equal(guarded_eval('data.loc[:, "a"]', context), data["a"]) + + +def test_named_tuple(): + class GoodNamedTuple(NamedTuple): + a: str + pass + + class BadNamedTuple(NamedTuple): + a: str + + def __getitem__(self, key): + return None + + good = GoodNamedTuple(a="x") + bad = BadNamedTuple(a="x") + + context = limited(data=good) + assert guarded_eval("data[0]", context) == "x" + + context = limited(data=bad) + with pytest.raises(GuardRejection): + guarded_eval("data[0]", context) + + +def test_dict(): + context = limited(data={"a": 1, "b": {"x": 2}, ("x", "y"): 3}) + assert guarded_eval('data["a"]', context) == 1 + assert guarded_eval('data["b"]', context) == {"x": 2} + assert guarded_eval('data["b"]["x"]', context) == 2 + assert guarded_eval('data["x", "y"]', context) == 3 + + assert guarded_eval("data.keys", context) + + +def test_set(): + context = limited(data={"a", "b"}) + assert guarded_eval("data.difference", context) + + +def test_list(): + context = limited(data=[1, 2, 3]) + assert guarded_eval("data[1]", context) == 2 + assert guarded_eval("data.copy", context) + + +def test_dict_literal(): + context = limited() + assert guarded_eval("{}", context) == {} + assert guarded_eval('{"a": 1}', context) == {"a": 1} + + +def test_list_literal(): + context = limited() + assert guarded_eval("[]", context) == [] + assert guarded_eval('[1, "a"]', context) == [1, "a"] + + +def test_set_literal(): + context = limited() + assert guarded_eval("set()", context) == set() + assert guarded_eval('{"a"}', context) == {"a"} + + +def test_evaluates_if_expression(): + context = limited() + assert guarded_eval("2 if True else 3", context) == 2 + assert guarded_eval("4 if False else 5", context) == 5 + + +def test_object(): + obj = object() + context = limited(obj=obj) + assert guarded_eval("obj.__dir__", context) == obj.__dir__ + + +@pytest.mark.parametrize( + "code,expected", + [ + ["int.numerator", int.numerator], + ["float.is_integer", float.is_integer], + ["complex.real", complex.real], + ], +) +def test_number_attributes(code, expected): + assert guarded_eval(code, limited()) == expected + + +def test_method_descriptor(): + context = limited() + assert guarded_eval("list.copy.__name__", context) == "copy" + + +@pytest.mark.parametrize( + "data,good,bad,expected", + [ + [[1, 2, 3], "data.index(2)", "data.append(4)", 1], + [{"a": 1}, "data.keys().isdisjoint({})", "data.update()", True], + ], +) +def test_evaluates_calls(data, good, bad, expected): + context = limited(data=data) + assert guarded_eval(good, context) == expected + + with pytest.raises(GuardRejection): + guarded_eval(bad, context) + + +@pytest.mark.parametrize( + "code,expected", + [ + ["(1\n+\n1)", 2], + ["list(range(10))[-1:]", [9]], + ["list(range(20))[3:-2:3]", [3, 6, 9, 12, 15]], + ], +) +@pytest.mark.parametrize("context", LIMITED_OR_HIGHER) +def test_evaluates_complex_cases(code, expected, context): + assert guarded_eval(code, context()) == expected + + +@pytest.mark.parametrize( + "code,expected", + [ + ["1", 1], + ["1.0", 1.0], + ["0xdeedbeef", 0xDEEDBEEF], + ["True", True], + ["None", None], + ["{}", {}], + ["[]", []], + ], +) +@pytest.mark.parametrize("context", MINIMAL_OR_HIGHER) +def test_evaluates_literals(code, expected, context): + assert guarded_eval(code, context()) == expected + + +@pytest.mark.parametrize( + "code,expected", + [ + ["-5", -5], + ["+5", +5], + ["~5", -6], + ], +) +@pytest.mark.parametrize("context", LIMITED_OR_HIGHER) +def test_evaluates_unary_operations(code, expected, context): + assert guarded_eval(code, context()) == expected + + +@pytest.mark.parametrize( + "code,expected", + [ + ["1 + 1", 2], + ["3 - 1", 2], + ["2 * 3", 6], + ["5 // 2", 2], + ["5 / 2", 2.5], + ["5**2", 25], + ["2 >> 1", 1], + ["2 << 1", 4], + ["1 | 2", 3], + ["1 & 1", 1], + ["1 & 2", 0], + ], +) +@pytest.mark.parametrize("context", LIMITED_OR_HIGHER) +def test_evaluates_binary_operations(code, expected, context): + assert guarded_eval(code, context()) == expected + + +@pytest.mark.parametrize( + "code,expected", + [ + ["2 > 1", True], + ["2 < 1", False], + ["2 <= 1", False], + ["2 <= 2", True], + ["1 >= 2", False], + ["2 >= 2", True], + ["2 == 2", True], + ["1 == 2", False], + ["1 != 2", True], + ["1 != 1", False], + ["1 < 4 < 3", False], + ["(1 < 4) < 3", True], + ["4 > 3 > 2 > 1", True], + ["4 > 3 > 2 > 9", False], + ["1 < 2 < 3 < 4", True], + ["9 < 2 < 3 < 4", False], + ["1 < 2 > 1 > 0 > -1 < 1", True], + ["1 in [1] in [[1]]", True], + ["1 in [1] in [[2]]", False], + ["1 in [1]", True], + ["0 in [1]", False], + ["1 not in [1]", False], + ["0 not in [1]", True], + ["True is True", True], + ["False is False", True], + ["True is False", False], + ["True is not True", False], + ["False is not True", True], + ], +) +@pytest.mark.parametrize("context", LIMITED_OR_HIGHER) +def test_evaluates_comparisons(code, expected, context): + assert guarded_eval(code, context()) == expected + + +def test_guards_comparisons(): + class GoodEq(int): + pass + + class BadEq(int): + def __eq__(self, other): + assert False + + context = limited(bad=BadEq(1), good=GoodEq(1)) + + with pytest.raises(GuardRejection): + guarded_eval("bad == 1", context) + + with pytest.raises(GuardRejection): + guarded_eval("bad != 1", context) + + with pytest.raises(GuardRejection): + guarded_eval("1 == bad", context) + + with pytest.raises(GuardRejection): + guarded_eval("1 != bad", context) + + assert guarded_eval("good == 1", context) is True + assert guarded_eval("good != 1", context) is False + assert guarded_eval("1 == good", context) is True + assert guarded_eval("1 != good", context) is False + + +def test_guards_unary_operations(): + class GoodOp(int): + pass + + class BadOpInv(int): + def __inv__(self, other): + assert False + + class BadOpInverse(int): + def __inv__(self, other): + assert False + + context = limited(good=GoodOp(1), bad1=BadOpInv(1), bad2=BadOpInverse(1)) + + with pytest.raises(GuardRejection): + guarded_eval("~bad1", context) + + with pytest.raises(GuardRejection): + guarded_eval("~bad2", context) + + +def test_guards_binary_operations(): + class GoodOp(int): + pass + + class BadOp(int): + def __add__(self, other): + assert False + + context = limited(good=GoodOp(1), bad=BadOp(1)) + + with pytest.raises(GuardRejection): + guarded_eval("1 + bad", context) + + with pytest.raises(GuardRejection): + guarded_eval("bad + 1", context) + + assert guarded_eval("good + 1", context) == 2 + assert guarded_eval("1 + good", context) == 2 + + +def test_guards_attributes(): + class GoodAttr(float): + pass + + class BadAttr1(float): + def __getattr__(self, key): + assert False + + class BadAttr2(float): + def __getattribute__(self, key): + assert False + + context = limited(good=GoodAttr(0.5), bad1=BadAttr1(0.5), bad2=BadAttr2(0.5)) + + with pytest.raises(GuardRejection): + guarded_eval("bad1.as_integer_ratio", context) + + with pytest.raises(GuardRejection): + guarded_eval("bad2.as_integer_ratio", context) + + assert guarded_eval("good.as_integer_ratio()", context) == (1, 2) + + +@pytest.mark.parametrize("context", MINIMAL_OR_HIGHER) +def test_access_builtins(context): + assert guarded_eval("round", context()) == round + + +def test_access_builtins_fails(): + context = limited() + with pytest.raises(NameError): + guarded_eval("this_is_not_builtin", context) + + +def test_rejects_forbidden(): + context = forbidden() + with pytest.raises(GuardRejection): + guarded_eval("1", context) + + +def test_guards_locals_and_globals(): + context = EvaluationContext( + locals={"local_a": "a"}, globals={"global_b": "b"}, evaluation="minimal" + ) + + with pytest.raises(GuardRejection): + guarded_eval("local_a", context) + + with pytest.raises(GuardRejection): + guarded_eval("global_b", context) + + +def test_access_locals_and_globals(): + context = EvaluationContext( + locals={"local_a": "a"}, globals={"global_b": "b"}, evaluation="limited" + ) + assert guarded_eval("local_a", context) == "a" + assert guarded_eval("global_b", context) == "b" + + +@pytest.mark.parametrize( + "code", + ["def func(): pass", "class C: pass", "x = 1", "x += 1", "del x", "import ast"], +) +@pytest.mark.parametrize("context", [minimal(), limited(), unsafe()]) +def test_rejects_side_effect_syntax(code, context): + with pytest.raises(SyntaxError): + guarded_eval(code, context) + + +def test_subscript(): + context = EvaluationContext( + locals={}, globals={}, evaluation="limited", in_subscript=True + ) + empty_slice = slice(None, None, None) + assert guarded_eval("", context) == tuple() + assert guarded_eval(":", context) == empty_slice + assert guarded_eval("1:2:3", context) == slice(1, 2, 3) + assert guarded_eval(':, "a"', context) == (empty_slice, "a") + + +def test_unbind_method(): + class X(list): + def index(self, k): + return "CUSTOM" + + x = X() + assert _unbind_method(x.index) is X.index + assert _unbind_method([].index) is list.index + assert _unbind_method(list.index) is None + + +def test_assumption_instance_attr_do_not_matter(): + """This is semi-specified in Python documentation. + + However, since the specification says 'not guaranted + to work' rather than 'is forbidden to work', future + versions could invalidate this assumptions. This test + is meant to catch such a change if it ever comes true. + """ + + class T: + def __getitem__(self, k): + return "a" + + def __getattr__(self, k): + return "a" + + def f(self): + return "b" + + t = T() + t.__getitem__ = f + t.__getattr__ = f + assert t[1] == "a" + assert t[1] == "a" + + +def test_assumption_named_tuples_share_getitem(): + """Check assumption on named tuples sharing __getitem__""" + from typing import NamedTuple + + class A(NamedTuple): + pass + + class B(NamedTuple): + pass + + assert A.__getitem__ == B.__getitem__ diff --git a/IPython/lib/tests/test_pygments.py b/IPython/lib/tests/test_pygments.py new file mode 100644 index 0000000..877b422 --- /dev/null +++ b/IPython/lib/tests/test_pygments.py @@ -0,0 +1,26 @@ +from typing import List + +import pytest +import pygments.lexers +import pygments.lexer + +from IPython.lib.lexers import IPythonConsoleLexer, IPythonLexer, IPython3Lexer + +#: the human-readable names of the IPython lexers with ``entry_points`` +EXPECTED_LEXER_NAMES = [ + cls.name for cls in [IPythonConsoleLexer, IPythonLexer, IPython3Lexer] +] + + +@pytest.fixture +def all_pygments_lexer_names() -> List[str]: + """Get all lexer names registered in pygments.""" + return {l[0] for l in pygments.lexers.get_all_lexers()} + + +@pytest.mark.parametrize("expected_lexer", EXPECTED_LEXER_NAMES) +def test_pygments_entry_points( + expected_lexer: str, all_pygments_lexer_names: List[str] +) -> None: + """Check whether the ``entry_points`` for ``pygments.lexers`` are correct.""" + assert expected_lexer in all_pygments_lexer_names diff --git a/IPython/terminal/magics.py b/IPython/terminal/magics.py index 66d5325..cea53e4 100644 --- a/IPython/terminal/magics.py +++ b/IPython/terminal/magics.py @@ -147,7 +147,7 @@ class TerminalMagics(Magics): sentinel = opts.get('s', u'--') block = '\n'.join(get_pasted_lines(sentinel, quiet=quiet)) - self.store_or_execute(block, name, store_history=False) + self.store_or_execute(block, name, store_history=True) @line_magic def paste(self, parameter_s=''): diff --git a/setup.cfg b/setup.cfg index 226506f..de327ab 100644 --- a/setup.cfg +++ b/setup.cfg @@ -100,17 +100,12 @@ exclude = setupext [options.package_data] +IPython = py.typed IPython.core = profile/README* IPython.core.tests = *.png, *.jpg, daft_extension/*.py IPython.lib.tests = *.wav IPython.testing.plugin = *.txt -[options.entry_points] -pygments.lexers = - ipythonconsole = IPython.lib.lexers:IPythonConsoleLexer - ipython = IPython.lib.lexers:IPythonLexer - ipython3 = IPython.lib.lexers:IPython3Lexer - [velin] ignore_patterns = IPython/core/tests diff --git a/setup.py b/setup.py index 4939ca5..3f7cd6d 100644 --- a/setup.py +++ b/setup.py @@ -139,7 +139,15 @@ setup_args['cmdclass'] = { 'install_scripts_sym': install_scripts_for_symlink, 'unsymlink': unsymlink, } -setup_args["entry_points"] = {"console_scripts": find_entry_points()} + +setup_args["entry_points"] = { + "console_scripts": find_entry_points(), + "pygments.lexers": [ + "ipythonconsole = IPython.lib.lexers:IPythonConsoleLexer", + "ipython = IPython.lib.lexers:IPythonLexer", + "ipython3 = IPython.lib.lexers:IPython3Lexer", + ], +} #--------------------------------------------------------------------------- # Do the actual setup now