Merge branch 'main' into main

IPython/core/guarded_eval.py

@@ -0,0 +1,738 b''
+from typing import (
+    Any,
+    Callable,
+    Dict,
+    Set,
+    Sequence,
+    Tuple,
+    NamedTuple,
+    Type,
+    Literal,
+    Union,
+    TYPE_CHECKING,
+)
+import ast
+import builtins
+import collections
+import operator
+import sys
+from functools import cached_property
+from dataclasses import dataclass, field
+
+from IPython.utils.docs import GENERATING_DOCUMENTATION
+from IPython.utils.decorators import undoc
+
+
+if TYPE_CHECKING or GENERATING_DOCUMENTATION:
+    from typing_extensions import Protocol
+else:
+    # do not require on runtime
+    Protocol = object  # requires Python >=3.8
+
+
+@undoc
+class HasGetItem(Protocol):
+    def __getitem__(self, key) -> None:
+        ...
+
+
+@undoc
+class InstancesHaveGetItem(Protocol):
+    def __call__(self, *args, **kwargs) -> HasGetItem:
+        ...
+
+
+@undoc
+class HasGetAttr(Protocol):
+    def __getattr__(self, key) -> None:
+        ...
+
+
+@undoc
+class DoesNotHaveGetAttr(Protocol):
+    pass
+
+
+# By default `__getattr__` is not explicitly implemented on most objects
+MayHaveGetattr = Union[HasGetAttr, DoesNotHaveGetAttr]
+
+
+def _unbind_method(func: Callable) -> Union[Callable, None]:
+    """Get unbound method for given bound method.
+
+    Returns None if cannot get unbound method, or method is already unbound.
+    """
+    owner = getattr(func, "__self__", None)
+    owner_class = type(owner)
+    name = getattr(func, "__name__", None)
+    instance_dict_overrides = getattr(owner, "__dict__", None)
+    if (
+        owner is not None
+        and name
+        and (
+            not instance_dict_overrides
+            or (instance_dict_overrides and name not in instance_dict_overrides)
+        )
+    ):
+        return getattr(owner_class, name)
+    return None
+
+
+@undoc
+@dataclass
+class EvaluationPolicy:
+    """Definition of evaluation policy."""
+
+    allow_locals_access: bool = False
+    allow_globals_access: bool = False
+    allow_item_access: bool = False
+    allow_attr_access: bool = False
+    allow_builtins_access: bool = False
+    allow_all_operations: bool = False
+    allow_any_calls: bool = False
+    allowed_calls: Set[Callable] = field(default_factory=set)
+
+    def can_get_item(self, value, item):
+        return self.allow_item_access
+
+    def can_get_attr(self, value, attr):
+        return self.allow_attr_access
+
+    def can_operate(self, dunders: Tuple[str, ...], a, b=None):
+        if self.allow_all_operations:
+            return True
+
+    def can_call(self, func):
+        if self.allow_any_calls:
+            return True
+
+        if func in self.allowed_calls:
+            return True
+
+        owner_method = _unbind_method(func)
+
+        if owner_method and owner_method in self.allowed_calls:
+            return True
+
+
+def _get_external(module_name: str, access_path: Sequence[str]):
+    """Get value from external module given a dotted access path.
+
+    Raises:
+    * `KeyError` if module is removed not found, and
+    * `AttributeError` if acess path does not match an exported object
+    """
+    member_type = sys.modules[module_name]
+    for attr in access_path:
+        member_type = getattr(member_type, attr)
+    return member_type
+
+
+def _has_original_dunder_external(
+    value,
+    module_name: str,
+    access_path: Sequence[str],
+    method_name: str,
+):
+    if module_name not in sys.modules:
+        # LBYLB as it is faster
+        return False
+    try:
+        member_type = _get_external(module_name, access_path)
+        value_type = type(value)
+        if type(value) == member_type:
+            return True
+        if method_name == "__getattribute__":
+            # we have to short-circuit here due to an unresolved issue in
+            # `isinstance` implementation: https://bugs.python.org/issue32683
+            return False
+        if isinstance(value, member_type):
+            method = getattr(value_type, method_name, None)
+            member_method = getattr(member_type, method_name, None)
+            if member_method == method:
+                return True
+    except (AttributeError, KeyError):
+        return False
+
+
+def _has_original_dunder(
+    value, allowed_types, allowed_methods, allowed_external, method_name
+):
+    # note: Python ignores `__getattr__`/`__getitem__` on instances,
+    # we only need to check at class level
+    value_type = type(value)
+
+    # strict type check passes → no need to check method
+    if value_type in allowed_types:
+        return True
+
+    method = getattr(value_type, method_name, None)
+
+    if method is None:
+        return None
+
+    if method in allowed_methods:
+        return True
+
+    for module_name, *access_path in allowed_external:
+        if _has_original_dunder_external(value, module_name, access_path, method_name):
+            return True
+
+    return False
+
+
+@undoc
+@dataclass
+class SelectivePolicy(EvaluationPolicy):
+    allowed_getitem: Set[InstancesHaveGetItem] = field(default_factory=set)
+    allowed_getitem_external: Set[Tuple[str, ...]] = field(default_factory=set)
+
+    allowed_getattr: Set[MayHaveGetattr] = field(default_factory=set)
+    allowed_getattr_external: Set[Tuple[str, ...]] = field(default_factory=set)
+
+    allowed_operations: Set = field(default_factory=set)
+    allowed_operations_external: Set[Tuple[str, ...]] = field(default_factory=set)
+
+    _operation_methods_cache: Dict[str, Set[Callable]] = field(
+        default_factory=dict, init=False
+    )
+
+    def can_get_attr(self, value, attr):
+        has_original_attribute = _has_original_dunder(
+            value,
+            allowed_types=self.allowed_getattr,
+            allowed_methods=self._getattribute_methods,
+            allowed_external=self.allowed_getattr_external,
+            method_name="__getattribute__",
+        )
+        has_original_attr = _has_original_dunder(
+            value,
+            allowed_types=self.allowed_getattr,
+            allowed_methods=self._getattr_methods,
+            allowed_external=self.allowed_getattr_external,
+            method_name="__getattr__",
+        )
+
+        accept = False
+
+        # Many objects do not have `__getattr__`, this is fine.
+        if has_original_attr is None and has_original_attribute:
+            accept = True
+        else:
+            # Accept objects without modifications to `__getattr__` and `__getattribute__`
+            accept = has_original_attr and has_original_attribute
+
+        if accept:
+            # We still need to check for overriden properties.
+
+            value_class = type(value)
+            if not hasattr(value_class, attr):
+                return True
+
+            class_attr_val = getattr(value_class, attr)
+            is_property = isinstance(class_attr_val, property)
+
+            if not is_property:
+                return True
+
+            # Properties in allowed types are ok (although we do not include any
+            # properties in our default allow list currently).
+            if type(value) in self.allowed_getattr:
+                return True  # pragma: no cover
+
+            # Properties in subclasses of allowed types may be ok if not changed
+            for module_name, *access_path in self.allowed_getattr_external:
+                try:
+                    external_class = _get_external(module_name, access_path)
+                    external_class_attr_val = getattr(external_class, attr)
+                except (KeyError, AttributeError):
+                    return False  # pragma: no cover
+                return class_attr_val == external_class_attr_val
+
+        return False
+
+    def can_get_item(self, value, item):
+        """Allow accessing `__getiitem__` of allow-listed instances unless it was not modified."""
+        return _has_original_dunder(
+            value,
+            allowed_types=self.allowed_getitem,
+            allowed_methods=self._getitem_methods,
+            allowed_external=self.allowed_getitem_external,
+            method_name="__getitem__",
+        )
+
+    def can_operate(self, dunders: Tuple[str, ...], a, b=None):
+        objects = [a]
+        if b is not None:
+            objects.append(b)
+        return all(
+            [
+                _has_original_dunder(
+                    obj,
+                    allowed_types=self.allowed_operations,
+                    allowed_methods=self._operator_dunder_methods(dunder),
+                    allowed_external=self.allowed_operations_external,
+                    method_name=dunder,
+                )
+                for dunder in dunders
+                for obj in objects
+            ]
+        )
+
+    def _operator_dunder_methods(self, dunder: str) -> Set[Callable]:
+        if dunder not in self._operation_methods_cache:
+            self._operation_methods_cache[dunder] = self._safe_get_methods(
+                self.allowed_operations, dunder
+            )
+        return self._operation_methods_cache[dunder]
+
+    @cached_property
+    def _getitem_methods(self) -> Set[Callable]:
+        return self._safe_get_methods(self.allowed_getitem, "__getitem__")
+
+    @cached_property
+    def _getattr_methods(self) -> Set[Callable]:
+        return self._safe_get_methods(self.allowed_getattr, "__getattr__")
+
+    @cached_property
+    def _getattribute_methods(self) -> Set[Callable]:
+        return self._safe_get_methods(self.allowed_getattr, "__getattribute__")
+
+    def _safe_get_methods(self, classes, name) -> Set[Callable]:
+        return {
+            method
+            for class_ in classes
+            for method in [getattr(class_, name, None)]
+            if method
+        }
+
+
+class _DummyNamedTuple(NamedTuple):
+    """Used internally to retrieve methods of named tuple instance."""
+
+
+class EvaluationContext(NamedTuple):
+    #: Local namespace
+    locals: dict
+    #: Global namespace
+    globals: dict
+    #: Evaluation policy identifier
+    evaluation: Literal[
+        "forbidden", "minimal", "limited", "unsafe", "dangerous"
+    ] = "forbidden"
+    #: Whether the evalution of code takes place inside of a subscript.
+    #: Useful for evaluating ``:-1, 'col'`` in ``df[:-1, 'col']``.
+    in_subscript: bool = False
+
+
+class _IdentitySubscript:
+    """Returns the key itself when item is requested via subscript."""
+
+    def __getitem__(self, key):
+        return key
+
+
+IDENTITY_SUBSCRIPT = _IdentitySubscript()
+SUBSCRIPT_MARKER = "__SUBSCRIPT_SENTINEL__"
+
+
+class GuardRejection(Exception):
+    """Exception raised when guard rejects evaluation attempt."""
+
+    pass
+
+
+def guarded_eval(code: str, context: EvaluationContext):
+    """Evaluate provided code in the evaluation context.
+
+    If evaluation policy given by context is set to ``forbidden``
+    no evaluation will be performed; if it is set to ``dangerous``
+    standard :func:`eval` will be used; finally, for any other,
+    policy :func:`eval_node` will be called on parsed AST.
+    """
+    locals_ = context.locals
+
+    if context.evaluation == "forbidden":
+        raise GuardRejection("Forbidden mode")
+
+    # note: not using `ast.literal_eval` as it does not implement
+    # getitem at all, for example it fails on simple `[0][1]`
+
+    if context.in_subscript:
+        # syntatic sugar for ellipsis (:) is only available in susbcripts
+        # so we need to trick the ast parser into thinking that we have
+        # a subscript, but we need to be able to later recognise that we did
+        # it so we can ignore the actual __getitem__ operation
+        if not code:
+            return tuple()
+        locals_ = locals_.copy()
+        locals_[SUBSCRIPT_MARKER] = IDENTITY_SUBSCRIPT
+        code = SUBSCRIPT_MARKER + "[" + code + "]"
+        context = EvaluationContext(**{**context._asdict(), **{"locals": locals_}})
+
+    if context.evaluation == "dangerous":
+        return eval(code, context.globals, context.locals)
+
+    expression = ast.parse(code, mode="eval")
+
+    return eval_node(expression, context)
+
+
+BINARY_OP_DUNDERS: Dict[Type[ast.operator], Tuple[str]] = {
+    ast.Add: ("__add__",),
+    ast.Sub: ("__sub__",),
+    ast.Mult: ("__mul__",),
+    ast.Div: ("__truediv__",),
+    ast.FloorDiv: ("__floordiv__",),
+    ast.Mod: ("__mod__",),
+    ast.Pow: ("__pow__",),
+    ast.LShift: ("__lshift__",),
+    ast.RShift: ("__rshift__",),
+    ast.BitOr: ("__or__",),
+    ast.BitXor: ("__xor__",),
+    ast.BitAnd: ("__and__",),
+    ast.MatMult: ("__matmul__",),
+}
+
+COMP_OP_DUNDERS: Dict[Type[ast.cmpop], Tuple[str, ...]] = {
+    ast.Eq: ("__eq__",),
+    ast.NotEq: ("__ne__", "__eq__"),
+    ast.Lt: ("__lt__", "__gt__"),
+    ast.LtE: ("__le__", "__ge__"),
+    ast.Gt: ("__gt__", "__lt__"),
+    ast.GtE: ("__ge__", "__le__"),
+    ast.In: ("__contains__",),
+    # Note: ast.Is, ast.IsNot, ast.NotIn are handled specially
+}
+
+UNARY_OP_DUNDERS: Dict[Type[ast.unaryop], Tuple[str, ...]] = {
+    ast.USub: ("__neg__",),
+    ast.UAdd: ("__pos__",),
+    # we have to check both __inv__ and __invert__!
+    ast.Invert: ("__invert__", "__inv__"),
+    ast.Not: ("__not__",),
+}
+
+
+def _find_dunder(node_op, dunders) -> Union[Tuple[str, ...], None]:
+    dunder = None
+    for op, candidate_dunder in dunders.items():
+        if isinstance(node_op, op):
+            dunder = candidate_dunder
+    return dunder
+
+
+def eval_node(node: Union[ast.AST, None], context: EvaluationContext):
+    """Evaluate AST node in provided context.
+
+    Applies evaluation restrictions defined in the context. Currently does not support evaluation of functions with keyword arguments.
+
+    Does not evaluate actions that always have side effects:
+
+    - class definitions (``class sth: ...``)
+    - function definitions (``def sth: ...``)
+    - variable assignments (``x = 1``)
+    - augmented assignments (``x += 1``)
+    - deletions (``del x``)
+
+    Does not evaluate operations which do not return values:
+
+    - assertions (``assert x``)
+    - pass (``pass``)
+    - imports (``import x``)
+    - control flow:
+
+        - conditionals (``if x:``) except for ternary IfExp (``a if x else b``)
+        - loops (``for`` and `while``)
+        - exception handling
+
+    The purpose of this function is to guard against unwanted side-effects;
+    it does not give guarantees on protection from malicious code execution.
+    """
+    policy = EVALUATION_POLICIES[context.evaluation]
+    if node is None:
+        return None
+    if isinstance(node, ast.Expression):
+        return eval_node(node.body, context)
+    if isinstance(node, ast.BinOp):
+        left = eval_node(node.left, context)
+        right = eval_node(node.right, context)
+        dunders = _find_dunder(node.op, BINARY_OP_DUNDERS)
+        if dunders:
+            if policy.can_operate(dunders, left, right):
+                return getattr(left, dunders[0])(right)
+            else:
+                raise GuardRejection(
+                    f"Operation (`{dunders}`) for",
+                    type(left),
+                    f"not allowed in {context.evaluation} mode",
+                )
+    if isinstance(node, ast.Compare):
+        left = eval_node(node.left, context)
+        all_true = True
+        negate = False
+        for op, right in zip(node.ops, node.comparators):
+            right = eval_node(right, context)
+            dunder = None
+            dunders = _find_dunder(op, COMP_OP_DUNDERS)
+            if not dunders:
+                if isinstance(op, ast.NotIn):
+                    dunders = COMP_OP_DUNDERS[ast.In]
+                    negate = True
+                if isinstance(op, ast.Is):
+                    dunder = "is_"
+                if isinstance(op, ast.IsNot):
+                    dunder = "is_"
+                    negate = True
+            if not dunder and dunders:
+                dunder = dunders[0]
+            if dunder:
+                a, b = (right, left) if dunder == "__contains__" else (left, right)
+                if dunder == "is_" or dunders and policy.can_operate(dunders, a, b):
+                    result = getattr(operator, dunder)(a, b)
+                    if negate:
+                        result = not result
+                    if not result:
+                        all_true = False
+                    left = right
+                else:
+                    raise GuardRejection(
+                        f"Comparison (`{dunder}`) for",
+                        type(left),
+                        f"not allowed in {context.evaluation} mode",
+                    )
+            else:
+                raise ValueError(
+                    f"Comparison `{dunder}` not supported"
+                )  # pragma: no cover
+        return all_true
+    if isinstance(node, ast.Constant):
+        return node.value
+    if isinstance(node, ast.Index):
+        # deprecated since Python 3.9
+        return eval_node(node.value, context)  # pragma: no cover
+    if isinstance(node, ast.Tuple):
+        return tuple(eval_node(e, context) for e in node.elts)
+    if isinstance(node, ast.List):
+        return [eval_node(e, context) for e in node.elts]
+    if isinstance(node, ast.Set):
+        return {eval_node(e, context) for e in node.elts}
+    if isinstance(node, ast.Dict):
+        return dict(
+            zip(
+                [eval_node(k, context) for k in node.keys],
+                [eval_node(v, context) for v in node.values],
+            )
+        )
+    if isinstance(node, ast.Slice):
+        return slice(
+            eval_node(node.lower, context),
+            eval_node(node.upper, context),
+            eval_node(node.step, context),
+        )
+    if isinstance(node, ast.ExtSlice):
+        # deprecated since Python 3.9
+        return tuple([eval_node(dim, context) for dim in node.dims])  # pragma: no cover
+    if isinstance(node, ast.UnaryOp):
+        value = eval_node(node.operand, context)
+        dunders = _find_dunder(node.op, UNARY_OP_DUNDERS)
+        if dunders:
+            if policy.can_operate(dunders, value):
+                return getattr(value, dunders[0])()
+            else:
+                raise GuardRejection(
+                    f"Operation (`{dunders}`) for",
+                    type(value),
+                    f"not allowed in {context.evaluation} mode",
+                )
+    if isinstance(node, ast.Subscript):
+        value = eval_node(node.value, context)
+        slice_ = eval_node(node.slice, context)
+        if policy.can_get_item(value, slice_):
+            return value[slice_]
+        raise GuardRejection(
+            "Subscript access (`__getitem__`) for",
+            type(value),  # not joined to avoid calling `repr`
+            f" not allowed in {context.evaluation} mode",
+        )
+    if isinstance(node, ast.Name):
+        if policy.allow_locals_access and node.id in context.locals:
+            return context.locals[node.id]
+        if policy.allow_globals_access and node.id in context.globals:
+            return context.globals[node.id]
+        if policy.allow_builtins_access and hasattr(builtins, node.id):
+            # note: do not use __builtins__, it is implementation detail of cPython
+            return getattr(builtins, node.id)
+        if not policy.allow_globals_access and not policy.allow_locals_access:
+            raise GuardRejection(
+                f"Namespace access not allowed in {context.evaluation} mode"
+            )
+        else:
+            raise NameError(f"{node.id} not found in locals, globals, nor builtins")
+    if isinstance(node, ast.Attribute):
+        value = eval_node(node.value, context)
+        if policy.can_get_attr(value, node.attr):
+            return getattr(value, node.attr)
+        raise GuardRejection(
+            "Attribute access (`__getattr__`) for",
+            type(value),  # not joined to avoid calling `repr`
+            f"not allowed in {context.evaluation} mode",
+        )
+    if isinstance(node, ast.IfExp):
+        test = eval_node(node.test, context)
+        if test:
+            return eval_node(node.body, context)
+        else:
+            return eval_node(node.orelse, context)
+    if isinstance(node, ast.Call):
+        func = eval_node(node.func, context)
+        if policy.can_call(func) and not node.keywords:
+            args = [eval_node(arg, context) for arg in node.args]
+            return func(*args)
+        raise GuardRejection(
+            "Call for",
+            func,  # not joined to avoid calling `repr`
+            f"not allowed in {context.evaluation} mode",
+        )
+    raise ValueError("Unhandled node", ast.dump(node))
+
+
+SUPPORTED_EXTERNAL_GETITEM = {
+    ("pandas", "core", "indexing", "_iLocIndexer"),
+    ("pandas", "core", "indexing", "_LocIndexer"),
+    ("pandas", "DataFrame"),
+    ("pandas", "Series"),
+    ("numpy", "ndarray"),
+    ("numpy", "void"),
+}
+
+
+BUILTIN_GETITEM: Set[InstancesHaveGetItem] = {
+    dict,
+    str,
+    bytes,
+    list,
+    tuple,
+    collections.defaultdict,
+    collections.deque,
+    collections.OrderedDict,
+    collections.ChainMap,
+    collections.UserDict,
+    collections.UserList,
+    collections.UserString,
+    _DummyNamedTuple,
+    _IdentitySubscript,
+}
+
+
+def _list_methods(cls, source=None):
+    """For use on immutable objects or with methods returning a copy"""
+    return [getattr(cls, k) for k in (source if source else dir(cls))]
+
+
+dict_non_mutating_methods = ("copy", "keys", "values", "items")
+list_non_mutating_methods = ("copy", "index", "count")
+set_non_mutating_methods = set(dir(set)) & set(dir(frozenset))
+
+
+dict_keys: Type[collections.abc.KeysView] = type({}.keys())
+method_descriptor: Any = type(list.copy)
+
+NUMERICS = {int, float, complex}
+
+ALLOWED_CALLS = {
+    bytes,
+    *_list_methods(bytes),
+    dict,
+    *_list_methods(dict, dict_non_mutating_methods),
+    dict_keys.isdisjoint,
+    list,
+    *_list_methods(list, list_non_mutating_methods),
+    set,
+    *_list_methods(set, set_non_mutating_methods),
+    frozenset,
+    *_list_methods(frozenset),
+    range,
+    str,
+    *_list_methods(str),
+    tuple,
+    *_list_methods(tuple),
+    *NUMERICS,
+    *[method for numeric_cls in NUMERICS for method in _list_methods(numeric_cls)],
+    collections.deque,
+    *_list_methods(collections.deque, list_non_mutating_methods),
+    collections.defaultdict,
+    *_list_methods(collections.defaultdict, dict_non_mutating_methods),
+    collections.OrderedDict,
+    *_list_methods(collections.OrderedDict, dict_non_mutating_methods),
+    collections.UserDict,
+    *_list_methods(collections.UserDict, dict_non_mutating_methods),
+    collections.UserList,
+    *_list_methods(collections.UserList, list_non_mutating_methods),
+    collections.UserString,
+    *_list_methods(collections.UserString, dir(str)),
+    collections.Counter,
+    *_list_methods(collections.Counter, dict_non_mutating_methods),
+    collections.Counter.elements,
+    collections.Counter.most_common,
+}
+
+BUILTIN_GETATTR: Set[MayHaveGetattr] = {
+    *BUILTIN_GETITEM,
+    set,
+    frozenset,
+    object,
+    type,  # `type` handles a lot of generic cases, e.g. numbers as in `int.real`.
+    *NUMERICS,
+    dict_keys,
+    method_descriptor,
+}
+
+
+BUILTIN_OPERATIONS = {*BUILTIN_GETATTR}
+
+EVALUATION_POLICIES = {
+    "minimal": EvaluationPolicy(
+        allow_builtins_access=True,
+        allow_locals_access=False,
+        allow_globals_access=False,
+        allow_item_access=False,
+        allow_attr_access=False,
+        allowed_calls=set(),
+        allow_any_calls=False,
+        allow_all_operations=False,
+    ),
+    "limited": SelectivePolicy(
+        allowed_getitem=BUILTIN_GETITEM,
+        allowed_getitem_external=SUPPORTED_EXTERNAL_GETITEM,
+        allowed_getattr=BUILTIN_GETATTR,
+        allowed_getattr_external={
+            # pandas Series/Frame implements custom `__getattr__`
+            ("pandas", "DataFrame"),
+            ("pandas", "Series"),
+        },
+        allowed_operations=BUILTIN_OPERATIONS,
+        allow_builtins_access=True,
+        allow_locals_access=True,
+        allow_globals_access=True,
+        allowed_calls=ALLOWED_CALLS,
+    ),
+    "unsafe": EvaluationPolicy(
+        allow_builtins_access=True,
+        allow_locals_access=True,
+        allow_globals_access=True,
+        allow_attr_access=True,
+        allow_item_access=True,
+        allow_any_calls=True,
+        allow_all_operations=True,
+    ),
+}
+
+
+__all__ = [
+    "guarded_eval",
+    "eval_node",
+    "GuardRejection",
+    "EvaluationContext",
+    "_unbind_method",
+]

IPython/core/tests/test_guarded_eval.py

@@ -0,0 +1,570 b''
+from contextlib import contextmanager
+from typing import NamedTuple
+from functools import partial
+from IPython.core.guarded_eval import (
+    EvaluationContext,
+    GuardRejection,
+    guarded_eval,
+    _unbind_method,
+)
+from IPython.testing import decorators as dec
+import pytest
+
+
+def create_context(evaluation: str, **kwargs):
+    return EvaluationContext(locals=kwargs, globals={}, evaluation=evaluation)
+
+
+forbidden = partial(create_context, "forbidden")
+minimal = partial(create_context, "minimal")
+limited = partial(create_context, "limited")
+unsafe = partial(create_context, "unsafe")
+dangerous = partial(create_context, "dangerous")
+
+LIMITED_OR_HIGHER = [limited, unsafe, dangerous]
+MINIMAL_OR_HIGHER = [minimal, *LIMITED_OR_HIGHER]
+
+
+@contextmanager
+def module_not_installed(module: str):
+    import sys
+
+    try:
+        to_restore = sys.modules[module]
+        del sys.modules[module]
+    except KeyError:
+        to_restore = None
+    try:
+        yield
+    finally:
+        sys.modules[module] = to_restore
+
+
+def test_external_not_installed():
+    """
+    Because attribute check requires checking if object is not of allowed
+    external type, this tests logic for absence of external module.
+    """
+
+    class Custom:
+        def __init__(self):
+            self.test = 1
+
+        def __getattr__(self, key):
+            return key
+
+    with module_not_installed("pandas"):
+        context = limited(x=Custom())
+        with pytest.raises(GuardRejection):
+            guarded_eval("x.test", context)
+
+
[e-mail]_without("pandas")
+def test_external_changed_api(monkeypatch):
+    """Check that the execution rejects if external API changed paths"""
+    import pandas as pd
+
+    series = pd.Series([1], index=["a"])
+
+    with monkeypatch.context() as m:
+        m.delattr(pd, "Series")
+        context = limited(data=series)
+        with pytest.raises(GuardRejection):
+            guarded_eval("data.iloc[0]", context)
+
+
[e-mail]_without("pandas")
+def test_pandas_series_iloc():
+    import pandas as pd
+
+    series = pd.Series([1], index=["a"])
+    context = limited(data=series)
+    assert guarded_eval("data.iloc[0]", context) == 1
+
+
+def test_rejects_custom_properties():
+    class BadProperty:
+        @property
+        def iloc(self):
+            return [None]
+
+    series = BadProperty()
+    context = limited(data=series)
+
+    with pytest.raises(GuardRejection):
+        guarded_eval("data.iloc[0]", context)
+
+
[e-mail]_without("pandas")
+def test_accepts_non_overriden_properties():
+    import pandas as pd
+
+    class GoodProperty(pd.Series):
+        pass
+
+    series = GoodProperty([1], index=["a"])
+    context = limited(data=series)
+
+    assert guarded_eval("data.iloc[0]", context) == 1
+
+
[e-mail]_without("pandas")
+def test_pandas_series():
+    import pandas as pd
+
+    context = limited(data=pd.Series([1], index=["a"]))
+    assert guarded_eval('data["a"]', context) == 1
+    with pytest.raises(KeyError):
+        guarded_eval('data["c"]', context)
+
+
[e-mail]_without("pandas")
+def test_pandas_bad_series():
+    import pandas as pd
+
+    class BadItemSeries(pd.Series):
+        def __getitem__(self, key):
+            return "CUSTOM_ITEM"
+
+    class BadAttrSeries(pd.Series):
+        def __getattr__(self, key):
+            return "CUSTOM_ATTR"
+
+    bad_series = BadItemSeries([1], index=["a"])
+    context = limited(data=bad_series)
+
+    with pytest.raises(GuardRejection):
+        guarded_eval('data["a"]', context)
+    with pytest.raises(GuardRejection):
+        guarded_eval('data["c"]', context)
+
+    # note: here result is a bit unexpected because
+    # pandas `__getattr__` calls `__getitem__`;
+    # FIXME - special case to handle it?
+    assert guarded_eval("data.a", context) == "CUSTOM_ITEM"
+
+    context = unsafe(data=bad_series)
+    assert guarded_eval('data["a"]', context) == "CUSTOM_ITEM"
+
+    bad_attr_series = BadAttrSeries([1], index=["a"])
+    context = limited(data=bad_attr_series)
+    assert guarded_eval('data["a"]', context) == 1
+    with pytest.raises(GuardRejection):
+        guarded_eval("data.a", context)
+
+
[e-mail]_without("pandas")
+def test_pandas_dataframe_loc():
+    import pandas as pd
+    from pandas.testing import assert_series_equal
+
+    data = pd.DataFrame([{"a": 1}])
+    context = limited(data=data)
+    assert_series_equal(guarded_eval('data.loc[:, "a"]', context), data["a"])
+
+
+def test_named_tuple():
+    class GoodNamedTuple(NamedTuple):
+        a: str
+        pass
+
+    class BadNamedTuple(NamedTuple):
+        a: str
+
+        def __getitem__(self, key):
+            return None
+
+    good = GoodNamedTuple(a="x")
+    bad = BadNamedTuple(a="x")
+
+    context = limited(data=good)
+    assert guarded_eval("data[0]", context) == "x"
+
+    context = limited(data=bad)
+    with pytest.raises(GuardRejection):
+        guarded_eval("data[0]", context)
+
+
+def test_dict():
+    context = limited(data={"a": 1, "b": {"x": 2}, ("x", "y"): 3})
+    assert guarded_eval('data["a"]', context) == 1
+    assert guarded_eval('data["b"]', context) == {"x": 2}
+    assert guarded_eval('data["b"]["x"]', context) == 2
+    assert guarded_eval('data["x", "y"]', context) == 3
+
+    assert guarded_eval("data.keys", context)
+
+
+def test_set():
+    context = limited(data={"a", "b"})
+    assert guarded_eval("data.difference", context)
+
+
+def test_list():
+    context = limited(data=[1, 2, 3])
+    assert guarded_eval("data[1]", context) == 2
+    assert guarded_eval("data.copy", context)
+
+
+def test_dict_literal():
+    context = limited()
+    assert guarded_eval("{}", context) == {}
+    assert guarded_eval('{"a": 1}', context) == {"a": 1}
+
+
+def test_list_literal():
+    context = limited()
+    assert guarded_eval("[]", context) == []
+    assert guarded_eval('[1, "a"]', context) == [1, "a"]
+
+
+def test_set_literal():
+    context = limited()
+    assert guarded_eval("set()", context) == set()
+    assert guarded_eval('{"a"}', context) == {"a"}
+
+
+def test_evaluates_if_expression():
+    context = limited()
+    assert guarded_eval("2 if True else 3", context) == 2
+    assert guarded_eval("4 if False else 5", context) == 5
+
+
+def test_object():
+    obj = object()
+    context = limited(obj=obj)
+    assert guarded_eval("obj.__dir__", context) == obj.__dir__
+
+
[e-mail](
+    "code,expected",
+    [
+        ["int.numerator", int.numerator],
+        ["float.is_integer", float.is_integer],
+        ["complex.real", complex.real],
+    ],
+)
+def test_number_attributes(code, expected):
+    assert guarded_eval(code, limited()) == expected
+
+
+def test_method_descriptor():
+    context = limited()
+    assert guarded_eval("list.copy.__name__", context) == "copy"
+
+
[e-mail](
+    "data,good,bad,expected",
+    [
+        [[1, 2, 3], "data.index(2)", "data.append(4)", 1],
+        [{"a": 1}, "data.keys().isdisjoint({})", "data.update()", True],
+    ],
+)
+def test_evaluates_calls(data, good, bad, expected):
+    context = limited(data=data)
+    assert guarded_eval(good, context) == expected
+
+    with pytest.raises(GuardRejection):
+        guarded_eval(bad, context)
+
+
[e-mail](
+    "code,expected",
+    [
+        ["(1\n+\n1)", 2],
+        ["list(range(10))[-1:]", [9]],
+        ["list(range(20))[3:-2:3]", [3, 6, 9, 12, 15]],
+    ],
+)
[e-mail]("context", LIMITED_OR_HIGHER)
+def test_evaluates_complex_cases(code, expected, context):
+    assert guarded_eval(code, context()) == expected
+
+
[e-mail](
+    "code,expected",
+    [
+        ["1", 1],
+        ["1.0", 1.0],
+        ["0xdeedbeef", 0xDEEDBEEF],
+        ["True", True],
+        ["None", None],
+        ["{}", {}],
+        ["[]", []],
+    ],
+)
[e-mail]("context", MINIMAL_OR_HIGHER)
+def test_evaluates_literals(code, expected, context):
+    assert guarded_eval(code, context()) == expected
+
+
[e-mail](
+    "code,expected",
+    [
+        ["-5", -5],
+        ["+5", +5],
+        ["~5", -6],
+    ],
+)
[e-mail]("context", LIMITED_OR_HIGHER)
+def test_evaluates_unary_operations(code, expected, context):
+    assert guarded_eval(code, context()) == expected
+
+
[e-mail](
+    "code,expected",
+    [
+        ["1 + 1", 2],
+        ["3 - 1", 2],
+        ["2 * 3", 6],
+        ["5 // 2", 2],
+        ["5 / 2", 2.5],
+        ["5**2", 25],
+        ["2 >> 1", 1],
+        ["2 << 1", 4],
+        ["1 | 2", 3],
+        ["1 & 1", 1],
+        ["1 & 2", 0],
+    ],
+)
[e-mail]("context", LIMITED_OR_HIGHER)
+def test_evaluates_binary_operations(code, expected, context):
+    assert guarded_eval(code, context()) == expected
+
+
[e-mail](
+    "code,expected",
+    [
+        ["2 > 1", True],
+        ["2 < 1", False],
+        ["2 <= 1", False],
+        ["2 <= 2", True],
+        ["1 >= 2", False],
+        ["2 >= 2", True],
+        ["2 == 2", True],
+        ["1 == 2", False],
+        ["1 != 2", True],
+        ["1 != 1", False],
+        ["1 < 4 < 3", False],
+        ["(1 < 4) < 3", True],
+        ["4 > 3 > 2 > 1", True],
+        ["4 > 3 > 2 > 9", False],
+        ["1 < 2 < 3 < 4", True],
+        ["9 < 2 < 3 < 4", False],
+        ["1 < 2 > 1 > 0 > -1 < 1", True],
+        ["1 in [1] in [[1]]", True],
+        ["1 in [1] in [[2]]", False],
+        ["1 in [1]", True],
+        ["0 in [1]", False],
+        ["1 not in [1]", False],
+        ["0 not in [1]", True],
+        ["True is True", True],
+        ["False is False", True],
+        ["True is False", False],
+        ["True is not True", False],
+        ["False is not True", True],
+    ],
+)
[e-mail]("context", LIMITED_OR_HIGHER)
+def test_evaluates_comparisons(code, expected, context):
+    assert guarded_eval(code, context()) == expected
+
+
+def test_guards_comparisons():
+    class GoodEq(int):
+        pass
+
+    class BadEq(int):
+        def __eq__(self, other):
+            assert False
+
+    context = limited(bad=BadEq(1), good=GoodEq(1))
+
+    with pytest.raises(GuardRejection):
+        guarded_eval("bad == 1", context)
+
+    with pytest.raises(GuardRejection):
+        guarded_eval("bad != 1", context)
+
+    with pytest.raises(GuardRejection):
+        guarded_eval("1 == bad", context)
+
+    with pytest.raises(GuardRejection):
+        guarded_eval("1 != bad", context)
+
+    assert guarded_eval("good == 1", context) is True
+    assert guarded_eval("good != 1", context) is False
+    assert guarded_eval("1 == good", context) is True
+    assert guarded_eval("1 != good", context) is False
+
+
+def test_guards_unary_operations():
+    class GoodOp(int):
+        pass
+
+    class BadOpInv(int):
+        def __inv__(self, other):
+            assert False
+
+    class BadOpInverse(int):
+        def __inv__(self, other):
+            assert False
+
+    context = limited(good=GoodOp(1), bad1=BadOpInv(1), bad2=BadOpInverse(1))
+
+    with pytest.raises(GuardRejection):
+        guarded_eval("~bad1", context)
+
+    with pytest.raises(GuardRejection):
+        guarded_eval("~bad2", context)
+
+
+def test_guards_binary_operations():
+    class GoodOp(int):
+        pass
+
+    class BadOp(int):
+        def __add__(self, other):
+            assert False
+
+    context = limited(good=GoodOp(1), bad=BadOp(1))
+
+    with pytest.raises(GuardRejection):
+        guarded_eval("1 + bad", context)
+
+    with pytest.raises(GuardRejection):
+        guarded_eval("bad + 1", context)
+
+    assert guarded_eval("good + 1", context) == 2
+    assert guarded_eval("1 + good", context) == 2
+
+
+def test_guards_attributes():
+    class GoodAttr(float):
+        pass
+
+    class BadAttr1(float):
+        def __getattr__(self, key):
+            assert False
+
+    class BadAttr2(float):
+        def __getattribute__(self, key):
+            assert False
+
+    context = limited(good=GoodAttr(0.5), bad1=BadAttr1(0.5), bad2=BadAttr2(0.5))
+
+    with pytest.raises(GuardRejection):
+        guarded_eval("bad1.as_integer_ratio", context)
+
+    with pytest.raises(GuardRejection):
+        guarded_eval("bad2.as_integer_ratio", context)
+
+    assert guarded_eval("good.as_integer_ratio()", context) == (1, 2)
+
+
[e-mail]("context", MINIMAL_OR_HIGHER)
+def test_access_builtins(context):
+    assert guarded_eval("round", context()) == round
+
+
+def test_access_builtins_fails():
+    context = limited()
+    with pytest.raises(NameError):
+        guarded_eval("this_is_not_builtin", context)
+
+
+def test_rejects_forbidden():
+    context = forbidden()
+    with pytest.raises(GuardRejection):
+        guarded_eval("1", context)
+
+
+def test_guards_locals_and_globals():
+    context = EvaluationContext(
+        locals={"local_a": "a"}, globals={"global_b": "b"}, evaluation="minimal"
+    )
+
+    with pytest.raises(GuardRejection):
+        guarded_eval("local_a", context)
+
+    with pytest.raises(GuardRejection):
+        guarded_eval("global_b", context)
+
+
+def test_access_locals_and_globals():
+    context = EvaluationContext(
+        locals={"local_a": "a"}, globals={"global_b": "b"}, evaluation="limited"
+    )
+    assert guarded_eval("local_a", context) == "a"
+    assert guarded_eval("global_b", context) == "b"
+
+
[e-mail](
+    "code",
+    ["def func(): pass", "class C: pass", "x = 1", "x += 1", "del x", "import ast"],
+)
[e-mail]("context", [minimal(), limited(), unsafe()])
+def test_rejects_side_effect_syntax(code, context):
+    with pytest.raises(SyntaxError):
+        guarded_eval(code, context)
+
+
+def test_subscript():
+    context = EvaluationContext(
+        locals={}, globals={}, evaluation="limited", in_subscript=True
+    )
+    empty_slice = slice(None, None, None)
+    assert guarded_eval("", context) == tuple()
+    assert guarded_eval(":", context) == empty_slice
+    assert guarded_eval("1:2:3", context) == slice(1, 2, 3)
+    assert guarded_eval(':, "a"', context) == (empty_slice, "a")
+
+
+def test_unbind_method():
+    class X(list):
+        def index(self, k):
+            return "CUSTOM"
+
+    x = X()
+    assert _unbind_method(x.index) is X.index
+    assert _unbind_method([].index) is list.index
+    assert _unbind_method(list.index) is None
+
+
+def test_assumption_instance_attr_do_not_matter():
+    """This is semi-specified in Python documentation.
+
+    However, since the specification says 'not guaranted
+    to work' rather than 'is forbidden to work', future
+    versions could invalidate this assumptions. This test
+    is meant to catch such a change if it ever comes true.
+    """
+
+    class T:
+        def __getitem__(self, k):
+            return "a"
+
+        def __getattr__(self, k):
+            return "a"
+
+    def f(self):
+        return "b"
+
+    t = T()
+    t.__getitem__ = f
+    t.__getattr__ = f
+    assert t[1] == "a"
+    assert t[1] == "a"
+
+
+def test_assumption_named_tuples_share_getitem():
+    """Check assumption on named tuples sharing __getitem__"""
+    from typing import NamedTuple
+
+    class A(NamedTuple):
+        pass
+
+    class B(NamedTuple):
+        pass
+
+    assert A.__getitem__ == B.__getitem__

IPython/lib/tests/test_pygments.py

@@ -0,0 +1,26 b''
+from typing import List
+
+import pytest
+import pygments.lexers
+import pygments.lexer
+
+from IPython.lib.lexers import IPythonConsoleLexer, IPythonLexer, IPython3Lexer
+
+#: the human-readable names of the IPython lexers with ``entry_points``
+EXPECTED_LEXER_NAMES = [
+    cls.name for cls in [IPythonConsoleLexer, IPythonLexer, IPython3Lexer]
+]
+
+
[e-mail]
+def all_pygments_lexer_names() -> List[str]:
+    """Get all lexer names registered in pygments."""
+    return {l[0] for l in pygments.lexers.get_all_lexers()}
+
+
[e-mail]("expected_lexer", EXPECTED_LEXER_NAMES)
+def test_pygments_entry_points(
+    expected_lexer: str, all_pygments_lexer_names: List[str]
+) -> None:
+    """Check whether the ``entry_points`` for ``pygments.lexers`` are correct."""
+    assert expected_lexer in all_pygments_lexer_names

.github/workflows/mypy.yml

@@ -15,7 +15,7 b' jobs:'
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.8]
+        python-version: ["3.x"]
 
     steps:
     - uses: actions/checkout@v3
@@ -31,6 +31,8 b' jobs:'
       run: |
         mypy -p IPython.terminal
         mypy -p IPython.core.magics
+        mypy -p IPython.core.guarded_eval
+        mypy -p IPython.core.completer
     - name: Lint with pyflakes
       run: |
         flake8  IPython/core/magics/script.py

.github/workflows/test.yml

@@ -19,7 +19,7 b' jobs:'
       fail-fast: false
       matrix:
         os: [ubuntu-latest, windows-latest]
-        python-version: ["3.8", "3.9", "3.10"]
+        python-version: ["3.8", "3.9", "3.10", "3.11"]
         deps: [test_extra]
         # Test all on ubuntu, test ends on macos
         include:
@@ -27,15 +27,15 b' jobs:'
             python-version: "3.8"
             deps: test_extra
           - os: macos-latest
-            python-version: "3.10"
+            python-version: "3.11"
             deps: test_extra
           # Tests minimal dependencies set
           - os: ubuntu-latest
-            python-version: "3.10"
+            python-version: "3.11"
             deps: test
           # Tests latest development Python version
           - os: ubuntu-latest
-            python-version: "3.11-dev"
+            python-version: "3.12-dev"
             deps: test
           # Installing optional dependencies stuff takes ages on PyPy
           - os: ubuntu-latest

IPython/__init__.py

@@ -1,3 +1,4 b''
+# PYTHON_ARGCOMPLETE_OK
 """
 IPython: tools for interactive and parallel computing in Python.
 

IPython/__main__.py

@@ -1,3 +1,4 b''
+# PYTHON_ARGCOMPLETE_OK
 # encoding: utf-8
 """Terminal-based IPython entry point.
 """

IPython/core/application.py

@@ -123,9 +123,8 b' class ProfileAwareConfigLoader(PyFileConfigLoader):'
         return super(ProfileAwareConfigLoader, self).load_subconfig(fname, path=path)
 
 class BaseIPythonApplication(Application):
-
-    name = u'ipython'
-    description = Unicode(u'IPython: an enhanced interactive Python shell.')
+    name = "ipython"
+    description = "IPython: an enhanced interactive Python shell."
     version = Unicode(release.version)
 
     aliases = base_aliases

IPython/core/completer.py

@@ -50,7 +50,7 b' Backward latex completion'
 
 It is sometime challenging to know how to type a character, if you are using
 IPython, or any compatible frontend you can prepend backslash to the character
-and press ``<tab>`` to expand it to its latex form.
+and press :kbd:`Tab` to expand it to its latex form.
 
 .. code::
 
@@ -59,7 +59,8 b' and press ``<tab>`` to expand it to its latex form.'
 
 
 Both forward and backward completions can be deactivated by setting the
-``Completer.backslash_combining_completions`` option to ``False``.
+:std:configtrait:`Completer.backslash_combining_completions` option to
+``False``.
 
 
 Experimental
@@ -95,7 +96,7 b' having to execute any code:'
    ... myvar[1].bi<tab>
 
 Tab completion will be able to infer that ``myvar[1]`` is a real number without
-executing any code unlike the previously available ``IPCompleter.greedy``
+executing almost any code unlike the deprecated :any:`IPCompleter.greedy`
 option.
 
 Be sure to update :any:`jedi` to the latest stable version or to try the
@@ -166,7 +167,7 b' this can be achieved by adding a list of identifiers of matchers which'
 should not be suppressed to ``MatcherResult`` under ``do_not_suppress`` key.
 
 The suppression behaviour can is user-configurable via
-:any:`IPCompleter.suppress_competing_matchers`.
+:std:configtrait:`IPCompleter.suppress_competing_matchers`.
 """
 
 
@@ -178,6 +179,7 b' The suppression behaviour can is user-configurable via'
 
 from __future__ import annotations
 import builtins as builtin_mod
+import enum
 import glob
 import inspect
 import itertools
@@ -186,14 +188,16 b' import os'
 import re
 import string
 import sys
+import tokenize
 import time
 import unicodedata
 import uuid
 import warnings
+from ast import literal_eval
+from collections import defaultdict
 from contextlib import contextmanager
 from dataclasses import dataclass
 from functools import cached_property, partial
-from importlib import import_module
 from types import SimpleNamespace
 from typing import (
     Iterable,
@@ -204,14 +208,15 b' from typing import ('
     Any,
     Sequence,
     Dict,
-    NamedTuple,
-    Pattern,
     Optional,
     TYPE_CHECKING,
     Set,
+    Sized,
+    TypeVar,
     Literal,
 )
 
+from IPython.core.guarded_eval import guarded_eval, EvaluationContext
 from IPython.core.error import TryNext
 from IPython.core.inputtransformer2 import ESC_MAGIC
 from IPython.core.latex_symbols import latex_symbols, reverse_latex_symbol
@@ -231,7 +236,6 b' from traitlets import ('
     Unicode,
     Dict as DictTrait,
     Union as UnionTrait,
-    default,
     observe,
 )
 from traitlets.config.configurable import Configurable
@@ -252,12 +256,13 b' except ImportError:'
     JEDI_INSTALLED = False
 
 
-if TYPE_CHECKING or GENERATING_DOCUMENTATION:
+if TYPE_CHECKING or GENERATING_DOCUMENTATION and sys.version_info >= (3, 11):
     from typing import cast
-    from typing_extensions import TypedDict, NotRequired, Protocol, TypeAlias
+    from typing_extensions import TypedDict, NotRequired, Protocol, TypeAlias, TypeGuard
 else:
+    from typing import Generic
 
-    def cast(obj, type_):
+    def cast(type_, obj):
         """Workaround for `TypeError: MatcherAPIv2() takes no arguments`"""
         return obj
 
@@ -266,6 +271,7 b' else:'
     TypedDict = Dict  # by extension of `NotRequired` requires 3.11 too
     Protocol = object  # requires Python >=3.8
     TypeAlias = Any  # requires Python >=3.10
+    TypeGuard = Generic  # requires Python >=3.10
 if GENERATING_DOCUMENTATION:
     from typing import TypedDict
 
@@ -279,7 +285,7 b' if GENERATING_DOCUMENTATION:'
 # write this). With below range we cover them all, with a density of ~67%
 # biggest next gap we consider only adds up about 1% density and there are 600
 # gaps that would need hard coding.
-_UNICODE_RANGES = [(32, 0x3134b), (0xe0001, 0xe01f0)]
+_UNICODE_RANGES = [(32, 0x323B0), (0xE0001, 0xE01F0)]
 
 # Public API
 __all__ = ["Completer", "IPCompleter"]
@@ -296,6 +302,9 b' MATCHES_LIMIT = 500'
 # Completion type reported when no type can be inferred.
 _UNKNOWN_TYPE = "<unknown>"
 
+# sentinel value to signal lack of a match
+not_found = object()
+
 class ProvisionalCompleterWarning(FutureWarning):
     """
     Exception raise by an experimental feature in this module.
@@ -466,8 +475,9 b' class _FakeJediCompletion:'
         self.complete = name
         self.type = 'crashed'
         self.name_with_symbols = name
-        self.signature = ''
-        self._origin = 'fake'
+        self.signature = ""
+        self._origin = "fake"
+        self.text = "crashed"
 
     def __repr__(self):
         return '<Fake completion object jedi has crashed>'
@@ -503,11 +513,23 b' class Completion:'
 
     __slots__ = ['start', 'end', 'text', 'type', 'signature', '_origin']
 
-    def __init__(self, start: int, end: int, text: str, *, type: str=None, _origin='', signature='') -> None:
-        warnings.warn("``Completion`` is a provisional API (as of IPython 6.0). "
-                      "It may change without warnings. "
-                      "Use in corresponding context manager.",
-                      category=ProvisionalCompleterWarning, stacklevel=2)
+    def __init__(
+        self,
+        start: int,
+        end: int,
+        text: str,
+        *,
+        type: Optional[str] = None,
+        _origin="",
+        signature="",
+    ) -> None:
+        warnings.warn(
+            "``Completion`` is a provisional API (as of IPython 6.0). "
+            "It may change without warnings. "
+            "Use in corresponding context manager.",
+            category=ProvisionalCompleterWarning,
+            stacklevel=2,
+        )
 
         self.start = start
         self.end = end
@@ -520,7 +542,7 b' class Completion:'
         return '<Completion start=%s end=%s text=%r type=%r, signature=%r,>' % \
                 (self.start, self.end, self.text, self.type or '?', self.signature or '?')
 
-    def __eq__(self, other)->Bool:
+    def __eq__(self, other) -> bool:
         """
         Equality and hash do not hash the type (as some completer may not be
         able to infer the type), but are use to (partially) de-duplicate
@@ -554,7 +576,7 b' class SimpleCompletion:'
 
     __slots__ = ["text", "type"]
 
-    def __init__(self, text: str, *, type: str = None):
+    def __init__(self, text: str, *, type: Optional[str] = None):
         self.text = text
         self.type = type
 
@@ -588,14 +610,18 b' class SimpleMatcherResult(_MatcherResultBase, TypedDict):'
     # in order to get __orig_bases__ for documentation
 
     #: List of candidate completions
-    completions: Sequence[SimpleCompletion]
+    completions: Sequence[SimpleCompletion] | Iterator[SimpleCompletion]
 
 
 class _JediMatcherResult(_MatcherResultBase):
     """Matching result returned by Jedi (will be processed differently)"""
 
     #: list of candidate completions
-    completions: Iterable[_JediCompletionLike]
+    completions: Iterator[_JediCompletionLike]
+
+
+AnyMatcherCompletion = Union[_JediCompletionLike, SimpleCompletion]
+AnyCompletion = TypeVar("AnyCompletion", AnyMatcherCompletion, Completion)
 
 
 @dataclass
@@ -642,16 +668,21 b' MatcherResult = Union[SimpleMatcherResult, _JediMatcherResult]'
 
 
 class _MatcherAPIv1Base(Protocol):
-    def __call__(self, text: str) -> list[str]:
+    def __call__(self, text: str) -> List[str]:
         """Call signature."""
+        ...
+
+    #: Used to construct the default matcher identifier
+    __qualname__: str
 
 
 class _MatcherAPIv1Total(_MatcherAPIv1Base, Protocol):
     #: API version
     matcher_api_version: Optional[Literal[1]]
 
-    def __call__(self, text: str) -> list[str]:
+    def __call__(self, text: str) -> List[str]:
         """Call signature."""
+        ...
 
 
 #: Protocol describing Matcher API v1.
@@ -666,26 +697,61 b' class MatcherAPIv2(Protocol):'
 
     def __call__(self, context: CompletionContext) -> MatcherResult:
         """Call signature."""
+        ...
+
+    #: Used to construct the default matcher identifier
+    __qualname__: str
 
 
 Matcher: TypeAlias = Union[MatcherAPIv1, MatcherAPIv2]
 
 
+def _is_matcher_v1(matcher: Matcher) -> TypeGuard[MatcherAPIv1]:
+    api_version = _get_matcher_api_version(matcher)
+    return api_version == 1
+
+
+def _is_matcher_v2(matcher: Matcher) -> TypeGuard[MatcherAPIv2]:
+    api_version = _get_matcher_api_version(matcher)
+    return api_version == 2
+
+
+def _is_sizable(value: Any) -> TypeGuard[Sized]:
+    """Determines whether objects is sizable"""
+    return hasattr(value, "__len__")
+
+
+def _is_iterator(value: Any) -> TypeGuard[Iterator]:
+    """Determines whether objects is sizable"""
+    return hasattr(value, "__next__")
+
+
 def has_any_completions(result: MatcherResult) -> bool:
     """Check if any result includes any completions."""
-    if hasattr(result["completions"], "__len__"):
-        return len(result["completions"]) != 0
-    try:
-        old_iterator = result["completions"]
-        first = next(old_iterator)
-        result["completions"] = itertools.chain([first], old_iterator)
-        return True
-    except StopIteration:
-        return False
+    completions = result["completions"]
+    if _is_sizable(completions):
+        return len(completions) != 0
+    if _is_iterator(completions):
+        try:
+            old_iterator = completions
+            first = next(old_iterator)
+            result["completions"] = cast(
+                Iterator[SimpleCompletion],
+                itertools.chain([first], old_iterator),
+            )
+            return True
+        except StopIteration:
+            return False
+    raise ValueError(
+        "Completions returned by matcher need to be an Iterator or a Sizable"
+    )
 
 
 def completion_matcher(
-    *, priority: float = None, identifier: str = None, api_version: int = 1
+    *,
+    priority: Optional[float] = None,
+    identifier: Optional[str] = None,
+    api_version: int = 1,
 ):
     """Adds attributes describing the matcher.
 
@@ -708,14 +774,14 b' def completion_matcher('
     """
 
     def wrapper(func: Matcher):
-        func.matcher_priority = priority or 0
-        func.matcher_identifier = identifier or func.__qualname__
-        func.matcher_api_version = api_version
+        func.matcher_priority = priority or 0  # type: ignore
+        func.matcher_identifier = identifier or func.__qualname__  # type: ignore
+        func.matcher_api_version = api_version  # type: ignore
         if TYPE_CHECKING:
             if api_version == 1:
-                func = cast(func, MatcherAPIv1)
+                func = cast(MatcherAPIv1, func)
             elif api_version == 2:
-                func = cast(func, MatcherAPIv2)
+                func = cast(MatcherAPIv2, func)
         return func
 
     return wrapper
@@ -902,12 +968,44 b' class CompletionSplitter(object):'
 
 class Completer(Configurable):
 
-    greedy = Bool(False,
-        help="""Activate greedy completion
-        PENDING DEPRECATION. this is now mostly taken care of with Jedi.
+    greedy = Bool(
+        False,
+        help="""Activate greedy completion.
+
+        .. deprecated:: 8.8
+            Use :std:configtrait:`Completer.evaluation` and :std:configtrait:`Completer.auto_close_dict_keys` instead.
+
+        When enabled in IPython 8.8 or newer, changes configuration as follows:
 
-        This will enable completion on elements of lists, results of function calls, etc.,
-        but can be unsafe because the code is actually evaluated on TAB.
+        - ``Completer.evaluation = 'unsafe'``
+        - ``Completer.auto_close_dict_keys = True``
+        """,
+    ).tag(config=True)
+
+    evaluation = Enum(
+        ("forbidden", "minimal", "limited", "unsafe", "dangerous"),
+        default_value="limited",
+        help="""Policy for code evaluation under completion.
+
+        Successive options allow to enable more eager evaluation for better
+        completion suggestions, including for nested dictionaries, nested lists,
+        or even results of function calls.
+        Setting ``unsafe`` or higher can lead to evaluation of arbitrary user
+        code on :kbd:`Tab` with potentially unwanted or dangerous side effects.
+
+        Allowed values are:
+
+        - ``forbidden``: no evaluation of code is permitted,
+        - ``minimal``: evaluation of literals and access to built-in namespace;
+          no item/attribute evaluationm no access to locals/globals,
+          no evaluation of any operations or comparisons.
+        - ``limited``: access to all namespaces, evaluation of hard-coded methods
+          (for example: :any:`dict.keys`, :any:`object.__getattr__`,
+          :any:`object.__getitem__`) on allow-listed objects (for example:
+          :any:`dict`, :any:`list`, :any:`tuple`, ``pandas.Series``),
+        - ``unsafe``: evaluation of all methods and function calls but not of
+          syntax with side-effects like `del x`,
+        - ``dangerous``: completely arbitrary evaluation.
         """,
     ).tag(config=True)
 
@@ -931,6 +1029,18 b' class Completer(Configurable):'
              "Includes completion of latex commands, unicode names, and expanding "
              "unicode characters back to latex commands.").tag(config=True)
 
+    auto_close_dict_keys = Bool(
+        False,
+        help="""
+        Enable auto-closing dictionary keys.
+
+        When enabled string keys will be suffixed with a final quote
+        (matching the opening quote), tuple keys will also receive a
+        separating comma if needed, and keys which are final will
+        receive a closing bracket (``]``).
+        """,
+    ).tag(config=True)
+
     def __init__(self, namespace=None, global_namespace=None, **kwargs):
         """Create a new completer for the command line.
 
@@ -1029,28 +1139,16 b' class Completer(Configurable):'
         with a __getattr__ hook is evaluated.
 
         """
+        m2 = re.match(r"(.+)\.(\w*)$", self.line_buffer)
+        if not m2:
+            return []
+        expr, attr = m2.group(1, 2)
 
-        # Another option, seems to work great. Catches things like ''.<tab>
-        m = re.match(r"(\S+(\.\w+)*)\.(\w*)$", text)
+        obj = self._evaluate_expr(expr)
 
-        if m:
-            expr, attr = m.group(1, 3)
-        elif self.greedy:
-            m2 = re.match(r"(.+)\.(\w*)$", self.line_buffer)
-            if not m2:
-                return []
-            expr, attr = m2.group(1,2)
-        else:
+        if obj is not_found:
             return []
 
-        try:
-            obj = eval(expr, self.namespace)
-        except:
-            try:
-                obj = eval(expr, self.global_namespace)
-            except:
-                return []
-
         if self.limit_to__all__ and hasattr(obj, '__all__'):
             words = get__all__entries(obj)
         else:
@@ -1068,8 +1166,31 b' class Completer(Configurable):'
             pass
         # Build match list to return
         n = len(attr)
-        return [u"%s.%s" % (expr, w) for w in words if w[:n] == attr ]
+        return ["%s.%s" % (expr, w) for w in words if w[:n] == attr]
 
+    def _evaluate_expr(self, expr):
+        obj = not_found
+        done = False
+        while not done and expr:
+            try:
+                obj = guarded_eval(
+                    expr,
+                    EvaluationContext(
+                        globals=self.global_namespace,
+                        locals=self.namespace,
+                        evaluation=self.evaluation,
+                    ),
+                )
+                done = True
+            except Exception as e:
+                if self.debug:
+                    print("Evaluation exception", e)
+                # trim the expression to remove any invalid prefix
+                # e.g. user starts `(d[`, so we get `expr = '(d'`,
+                # where parenthesis is not closed.
+                # TODO: make this faster by reusing parts of the computation?
+                expr = expr[1:]
+        return obj
 
 def get__all__entries(obj):
     """returns the strings in the __all__ attribute"""
@@ -1081,8 +1202,82 b' def get__all__entries(obj):'
     return [w for w in words if isinstance(w, str)]
 
 
-def match_dict_keys(keys: List[Union[str, bytes, Tuple[Union[str, bytes]]]], prefix: str, delims: str,
-                    extra_prefix: Optional[Tuple[str, bytes]]=None) -> Tuple[str, int, List[str]]:
+class _DictKeyState(enum.Flag):
+    """Represent state of the key match in context of other possible matches.
+
+    - given `d1 = {'a': 1}` completion on `d1['<tab>` will yield `{'a': END_OF_ITEM}` as there is no tuple.
+    - given `d2 = {('a', 'b'): 1}`: `d2['a', '<tab>` will yield `{'b': END_OF_TUPLE}` as there is no tuple members to add beyond `'b'`.
+    - given `d3 = {('a', 'b'): 1}`: `d3['<tab>` will yield `{'a': IN_TUPLE}` as `'a'` can be added.
+    - given `d4 = {'a': 1, ('a', 'b'): 2}`: `d4['<tab>` will yield `{'a': END_OF_ITEM & END_OF_TUPLE}`
+    """
+
+    BASELINE = 0
+    END_OF_ITEM = enum.auto()
+    END_OF_TUPLE = enum.auto()
+    IN_TUPLE = enum.auto()
+
+
+def _parse_tokens(c):
+    """Parse tokens even if there is an error."""
+    tokens = []
+    token_generator = tokenize.generate_tokens(iter(c.splitlines()).__next__)
+    while True:
+        try:
+            tokens.append(next(token_generator))
+        except tokenize.TokenError:
+            return tokens
+        except StopIteration:
+            return tokens
+
+
+def _match_number_in_dict_key_prefix(prefix: str) -> Union[str, None]:
+    """Match any valid Python numeric literal in a prefix of dictionary keys.
+
+    References:
+    - https://docs.python.org/3/reference/lexical_analysis.html#numeric-literals
+    - https://docs.python.org/3/library/tokenize.html
+    """
+    if prefix[-1].isspace():
+        # if user typed a space we do not have anything to complete
+        # even if there was a valid number token before
+        return None
+    tokens = _parse_tokens(prefix)
+    rev_tokens = reversed(tokens)
+    skip_over = {tokenize.ENDMARKER, tokenize.NEWLINE}
+    number = None
+    for token in rev_tokens:
+        if token.type in skip_over:
+            continue
+        if number is None:
+            if token.type == tokenize.NUMBER:
+                number = token.string
+                continue
+            else:
+                # we did not match a number
+                return None
+        if token.type == tokenize.OP:
+            if token.string == ",":
+                break
+            if token.string in {"+", "-"}:
+                number = token.string + number
+        else:
+            return None
+    return number
+
+
+_INT_FORMATS = {
+    "0b": bin,
+    "0o": oct,
+    "0x": hex,
+}
+
+
+def match_dict_keys(
+    keys: List[Union[str, bytes, Tuple[Union[str, bytes], ...]]],
+    prefix: str,
+    delims: str,
+    extra_prefix: Optional[Tuple[Union[str, bytes], ...]] = None,
+) -> Tuple[str, int, Dict[str, _DictKeyState]]:
     """Used by dict_key_matches, matching the prefix to a list of keys
 
     Parameters
@@ -1102,47 +1297,89 b' def match_dict_keys(keys: List[Union[str, bytes, Tuple[Union[str, bytes]]]], pre'
     A tuple of three elements: ``quote``, ``token_start``, ``matched``, with
     ``quote`` being the quote that need to be used to close current string.
     ``token_start`` the position where the replacement should start occurring,
-    ``matches`` a list of replacement/completion
-
+    ``matches`` a dictionary of replacement/completion keys on keys and values
+        indicating whether the state.
     """
     prefix_tuple = extra_prefix if extra_prefix else ()
-    Nprefix = len(prefix_tuple)
+
+    prefix_tuple_size = sum(
+        [
+            # for pandas, do not count slices as taking space
+            not isinstance(k, slice)
+            for k in prefix_tuple
+        ]
+    )
+    text_serializable_types = (str, bytes, int, float, slice)
+
     def filter_prefix_tuple(key):
         # Reject too short keys
-        if len(key) <= Nprefix:
+        if len(key) <= prefix_tuple_size:
             return False
-        # Reject keys with non str/bytes in it
+        # Reject keys which cannot be serialised to text
         for k in key:
-            if not isinstance(k, (str, bytes)):
+            if not isinstance(k, text_serializable_types):
                 return False
         # Reject keys that do not match the prefix
         for k, pt in zip(key, prefix_tuple):
-            if k != pt:
+            if k != pt and not isinstance(pt, slice):
                 return False
         # All checks passed!
         return True
 
-    filtered_keys:List[Union[str,bytes]] = []
-    def _add_to_filtered_keys(key):
-        if isinstance(key, (str, bytes)):
-            filtered_keys.append(key)
+    filtered_key_is_final: Dict[
+        Union[str, bytes, int, float], _DictKeyState
+    ] = defaultdict(lambda: _DictKeyState.BASELINE)
 
     for k in keys:
+        # If at least one of the matches is not final, mark as undetermined.
+        # This can happen with `d = {111: 'b', (111, 222): 'a'}` where
+        # `111` appears final on first match but is not final on the second.
+
         if isinstance(k, tuple):
             if filter_prefix_tuple(k):
-                _add_to_filtered_keys(k[Nprefix])
+                key_fragment = k[prefix_tuple_size]
+                filtered_key_is_final[key_fragment] |= (
+                    _DictKeyState.END_OF_TUPLE
+                    if len(k) == prefix_tuple_size + 1
+                    else _DictKeyState.IN_TUPLE
+                )
+        elif prefix_tuple_size > 0:
+            # we are completing a tuple but this key is not a tuple,
+            # so we should ignore it
+            pass
         else:
-            _add_to_filtered_keys(k)
+            if isinstance(k, text_serializable_types):
+                filtered_key_is_final[k] |= _DictKeyState.END_OF_ITEM
+
+    filtered_keys = filtered_key_is_final.keys()
 
     if not prefix:
-        return '', 0, [repr(k) for k in filtered_keys]
-    quote_match = re.search('["\']', prefix)
-    assert quote_match is not None # silence mypy
-    quote = quote_match.group()
-    try:
-        prefix_str = eval(prefix + quote, {})
-    except Exception:
-        return '', 0, []
+        return "", 0, {repr(k): v for k, v in filtered_key_is_final.items()}
+
+    quote_match = re.search("(?:\"|')", prefix)
+    is_user_prefix_numeric = False
+
+    if quote_match:
+        quote = quote_match.group()
+        valid_prefix = prefix + quote
+        try:
+            prefix_str = literal_eval(valid_prefix)
+        except Exception:
+            return "", 0, {}
+    else:
+        # If it does not look like a string, let's assume
+        # we are dealing with a number or variable.
+        number_match = _match_number_in_dict_key_prefix(prefix)
+
+        # We do not want the key matcher to suggest variable names so we yield:
+        if number_match is None:
+            # The alternative would be to assume that user forgort the quote
+            # and if the substring matches, suggest adding it at the start.
+            return "", 0, {}
+
+        prefix_str = number_match
+        is_user_prefix_numeric = True
+        quote = ""
 
     pattern = '[^' + ''.join('\\' + c for c in delims) + ']*$'
     token_match = re.search(pattern, prefix, re.UNICODE)
@@ -1150,17 +1387,36 b' def match_dict_keys(keys: List[Union[str, bytes, Tuple[Union[str, bytes]]]], pre'
     token_start = token_match.start()
     token_prefix = token_match.group()
 
-    matched:List[str] = []
+    matched: Dict[str, _DictKeyState] = {}
+
+    str_key: Union[str, bytes]
+
     for key in filtered_keys:
+        if isinstance(key, (int, float)):
+            # User typed a number but this key is not a number.
+            if not is_user_prefix_numeric:
+                continue
+            str_key = str(key)
+            if isinstance(key, int):
+                int_base = prefix_str[:2].lower()
+                # if user typed integer using binary/oct/hex notation:
+                if int_base in _INT_FORMATS:
+                    int_format = _INT_FORMATS[int_base]
+                    str_key = int_format(key)
+        else:
+            # User typed a string but this key is a number.
+            if is_user_prefix_numeric:
+                continue
+            str_key = key
         try:
-            if not key.startswith(prefix_str):
+            if not str_key.startswith(prefix_str):
                 continue
-        except (AttributeError, TypeError, UnicodeError):
+        except (AttributeError, TypeError, UnicodeError) as e:
             # Python 3+ TypeError on b'a'.startswith('a') or vice-versa
             continue
 
         # reformat remainder of key to begin with prefix
-        rem = key[len(prefix_str):]
+        rem = str_key[len(prefix_str) :]
         # force repr wrapped in '
         rem_repr = repr(rem + '"') if isinstance(rem, str) else repr(rem + b'"')
         rem_repr = rem_repr[1 + rem_repr.index("'"):-2]
@@ -1171,7 +1427,9 b' def match_dict_keys(keys: List[Union[str, bytes, Tuple[Union[str, bytes]]]], pre'
             rem_repr = rem_repr.replace('"', '\\"')
 
         # then reinsert prefix from start of token
-        matched.append('%s%s' % (token_prefix, rem_repr))
+        match = "%s%s" % (token_prefix, rem_repr)
+
+        matched[match] = filtered_key_is_final[key]
     return quote, token_start, matched
 
 
@@ -1237,11 +1495,14 b' def position_to_cursor(text:str, offset:int)->Tuple[int, int]:'
     return line, col
 
 
-def _safe_isinstance(obj, module, class_name):
+def _safe_isinstance(obj, module, class_name, *attrs):
     """Checks if obj is an instance of module.class_name if loaded
     """
-    return (module in sys.modules and
-            isinstance(obj, getattr(import_module(module), class_name)))
+    if module in sys.modules:
+        m = sys.modules[module]
+        for attr in [class_name, *attrs]:
+            m = getattr(m, attr)
+        return isinstance(obj, m)
 
 
 @context_matcher()
@@ -1394,10 +1655,59 b' def _make_signature(completion)-> str:'
 _CompleteResult = Dict[str, MatcherResult]
 
 
+DICT_MATCHER_REGEX = re.compile(
+    r"""(?x)
+(  # match dict-referring - or any get item object - expression
+    .+
+)
+\[   # open bracket
+\s*  # and optional whitespace
+# Capture any number of serializable objects (e.g. "a", "b", 'c')
+# and slices
+((?:(?:
+    (?: # closed string
+        [uUbB]?  # string prefix (r not handled)
+        (?:
+            '(?:[^']|(?<!\\)\\')*'
+        |
+            "(?:[^"]|(?<!\\)\\")*"
+        )
+    )
+    |
+        # capture integers and slices
+        (?:[-+]?\d+)?(?::(?:[-+]?\d+)?){0,2}
+    |
+        # integer in bin/hex/oct notation
+        0[bBxXoO]_?(?:\w|\d)+
+    )
+    \s*,\s*
+)*)
+((?:
+    (?: # unclosed string
+        [uUbB]?  # string prefix (r not handled)
+        (?:
+            '(?:[^']|(?<!\\)\\')*
+            |
+            "(?:[^"]|(?<!\\)\\")*
+        )
+    )
+    |
+        # unfinished integer
+        (?:[-+]?\d+)
+    |
+        # integer in bin/hex/oct notation
+        0[bBxXoO]_?(?:\w|\d)+
+    )
+)?
+$
+"""
+)
+
+
 def _convert_matcher_v1_result_to_v2(
     matches: Sequence[str],
     type: str,
-    fragment: str = None,
+    fragment: Optional[str] = None,
     suppress_if_matches: bool = False,
 ) -> SimpleMatcherResult:
     """Utility to help with transition"""
@@ -1407,27 +1717,29 b' def _convert_matcher_v1_result_to_v2('
     }
     if fragment is not None:
         result["matched_fragment"] = fragment
-    return result
+    return cast(SimpleMatcherResult, result)
 
 
 class IPCompleter(Completer):
     """Extension of the completer class with IPython-specific features"""
 
-    __dict_key_regexps: Optional[Dict[bool,Pattern]] = None
-
     @observe('greedy')
     def _greedy_changed(self, change):
         """update the splitter and readline delims when greedy is changed"""
-        if change['new']:
+        if change["new"]:
+            self.evaluation = "unsafe"
+            self.auto_close_dict_keys = True
             self.splitter.delims = GREEDY_DELIMS
         else:
+            self.evaluation = "limited"
+            self.auto_close_dict_keys = False
             self.splitter.delims = DELIMS
 
     dict_keys_only = Bool(
         False,
         help="""
         Whether to show dict key matches only.
-        
+
         (disables all matchers except for `IPCompleter.dict_key_matcher`).
         """,
     )
@@ -1607,7 +1919,7 b' class IPCompleter(Completer):'
 
         if not self.backslash_combining_completions:
             for matcher in self._backslash_combining_matchers:
-                self.disable_matchers.append(matcher.matcher_identifier)
+                self.disable_matchers.append(_get_matcher_id(matcher))
 
         if not self.merge_completions:
             self.suppress_competing_matchers = True
@@ -1897,7 +2209,7 b' class IPCompleter(Completer):'
 
     def _jedi_matches(
         self, cursor_column: int, cursor_line: int, text: str
-    ) -> Iterable[_JediCompletionLike]:
+    ) -> Iterator[_JediCompletionLike]:
         """
         Return a list of :any:`jedi.api.Completion`s object from a ``text`` and
         cursor position.
@@ -1963,15 +2275,23 b' class IPCompleter(Completer):'
                 print("Error detecting if completing a non-finished string :", e, '|')
 
         if not try_jedi:
-            return []
+            return iter([])
         try:
             return filter(completion_filter, interpreter.complete(column=cursor_column, line=cursor_line + 1))
         except Exception as e:
             if self.debug:
-                return [_FakeJediCompletion('Oops Jedi has crashed, please report a bug with the following:\n"""\n%s\ns"""' % (e))]
+                return iter(
+                    [
+                        _FakeJediCompletion(
+                            'Oops Jedi has crashed, please report a bug with the following:\n"""\n%s\ns"""'
+                            % (e)
+                        )
+                    ]
+                )
             else:
-                return []
+                return iter([])
 
+    @completion_matcher(api_version=1)
     def python_matches(self, text: str) -> Iterable[str]:
         """Match attributes or global python names"""
         if "." in text:
@@ -2149,12 +2469,16 b' class IPCompleter(Completer):'
             return method()
 
         # Special case some common in-memory dict-like types
-        if isinstance(obj, dict) or\
-           _safe_isinstance(obj, 'pandas', 'DataFrame'):
+        if isinstance(obj, dict) or _safe_isinstance(obj, "pandas", "DataFrame"):
             try:
                 return list(obj.keys())
             except Exception:
                 return []
+        elif _safe_isinstance(obj, "pandas", "core", "indexing", "_LocIndexer"):
+            try:
+                return list(obj.obj.keys())
+            except Exception:
+                return []
         elif _safe_isinstance(obj, 'numpy', 'ndarray') or\
              _safe_isinstance(obj, 'numpy', 'void'):
             return obj.dtype.names or []
@@ -2175,74 +2499,49 b' class IPCompleter(Completer):'
             You can use :meth:`dict_key_matcher` instead.
         """
 
-        if self.__dict_key_regexps is not None:
-            regexps = self.__dict_key_regexps
-        else:
-            dict_key_re_fmt = r'''(?x)
-            (  # match dict-referring expression wrt greedy setting
-                %s
-            )
-            \[   # open bracket
-            \s*  # and optional whitespace
-            # Capture any number of str-like objects (e.g. "a", "b", 'c')
-            ((?:[uUbB]?  # string prefix (r not handled)
-                (?:
-                    '(?:[^']|(?<!\\)\\')*'
-                |
-                    "(?:[^"]|(?<!\\)\\")*"
-                )
-                \s*,\s*
-            )*)
-            ([uUbB]?  # string prefix (r not handled)
-                (?:   # unclosed string
-                    '(?:[^']|(?<!\\)\\')*
-                |
-                    "(?:[^"]|(?<!\\)\\")*
-                )
-            )?
-            $
-            '''
-            regexps = self.__dict_key_regexps = {
-                False: re.compile(dict_key_re_fmt % r'''
-                                  # identifiers separated by .
-                                  (?!\d)\w+
-                                  (?:\.(?!\d)\w+)*
-                                  '''),
-                True: re.compile(dict_key_re_fmt % '''
-                                 .+
-                                 ''')
-            }
+        # Short-circuit on closed dictionary (regular expression would
+        # not match anyway, but would take quite a while).
+        if self.text_until_cursor.strip().endswith("]"):
+            return []
 
-        match = regexps[self.greedy].search(self.text_until_cursor)
+        match = DICT_MATCHER_REGEX.search(self.text_until_cursor)
 
         if match is None:
             return []
 
-        expr, prefix0, prefix = match.groups()
-        try:
-            obj = eval(expr, self.namespace)
-        except Exception:
-            try:
-                obj = eval(expr, self.global_namespace)
-            except Exception:
-                return []
+        expr, prior_tuple_keys, key_prefix = match.groups()
+
+        obj = self._evaluate_expr(expr)
+
+        if obj is not_found:
+            return []
 
         keys = self._get_keys(obj)
         if not keys:
             return keys
 
-        extra_prefix = eval(prefix0) if prefix0 != '' else None
+        tuple_prefix = guarded_eval(
+            prior_tuple_keys,
+            EvaluationContext(
+                globals=self.global_namespace,
+                locals=self.namespace,
+                evaluation=self.evaluation,
+                in_subscript=True,
+            ),
+        )
 
-        closing_quote, token_offset, matches = match_dict_keys(keys, prefix, self.splitter.delims, extra_prefix=extra_prefix)
+        closing_quote, token_offset, matches = match_dict_keys(
+            keys, key_prefix, self.splitter.delims, extra_prefix=tuple_prefix
+        )
         if not matches:
-            return matches
+            return []
 
         # get the cursor position of
         # - the text being completed
         # - the start of the key text
         # - the start of the completion
         text_start = len(self.text_until_cursor) - len(text)
-        if prefix:
+        if key_prefix:
             key_start = match.start(3)
             completion_start = key_start + token_offset
         else:
@@ -2254,26 +2553,61 b' class IPCompleter(Completer):'
         else:
             leading = text[text_start:completion_start]
 
-        # the index of the `[` character
-        bracket_idx = match.end(1)
-
         # append closing quote and bracket as appropriate
         # this is *not* appropriate if the opening quote or bracket is outside
-        # the text given to this method
-        suf = ''
-        continuation = self.line_buffer[len(self.text_until_cursor):]
-        if key_start > text_start and closing_quote:
-            # quotes were opened inside text, maybe close them
-            if continuation.startswith(closing_quote):
-                continuation = continuation[len(closing_quote):]
-            else:
-                suf += closing_quote
-        if bracket_idx > text_start:
-            # brackets were opened inside text, maybe close them
-            if not continuation.startswith(']'):
-                suf += ']'
+        # the text given to this method, e.g. `d["""a\nt
+        can_close_quote = False
+        can_close_bracket = False
+
+        continuation = self.line_buffer[len(self.text_until_cursor) :].strip()
 
-        return [leading + k + suf for k in matches]
+        if continuation.startswith(closing_quote):
+            # do not close if already closed, e.g. `d['a<tab>'`
+            continuation = continuation[len(closing_quote) :]
+        else:
+            can_close_quote = True
+
+        continuation = continuation.strip()
+
+        # e.g. `pandas.DataFrame` has different tuple indexer behaviour,
+        # handling it is out of scope, so let's avoid appending suffixes.
+        has_known_tuple_handling = isinstance(obj, dict)
+
+        can_close_bracket = (
+            not continuation.startswith("]") and self.auto_close_dict_keys
+        )
+        can_close_tuple_item = (
+            not continuation.startswith(",")
+            and has_known_tuple_handling
+            and self.auto_close_dict_keys
+        )
+        can_close_quote = can_close_quote and self.auto_close_dict_keys
+
+        # fast path if closing qoute should be appended but not suffix is allowed
+        if not can_close_quote and not can_close_bracket and closing_quote:
+            return [leading + k for k in matches]
+
+        results = []
+
+        end_of_tuple_or_item = _DictKeyState.END_OF_TUPLE | _DictKeyState.END_OF_ITEM
+
+        for k, state_flag in matches.items():
+            result = leading + k
+            if can_close_quote and closing_quote:
+                result += closing_quote
+
+            if state_flag == end_of_tuple_or_item:
+                # We do not know which suffix to add,
+                # e.g. both tuple item and string
+                # match this item.
+                pass
+
+            if state_flag in end_of_tuple_or_item and can_close_bracket:
+                result += "]"
+            if state_flag == _DictKeyState.IN_TUPLE and can_close_tuple_item:
+                result += ", "
+            results.append(result)
+        return results
 
     @context_matcher()
     def unicode_name_matcher(self, context: CompletionContext):
@@ -2516,17 +2850,23 b' class IPCompleter(Completer):'
 
         jedi_matcher_id = _get_matcher_id(self._jedi_matcher)
 
+        def is_non_jedi_result(
+            result: MatcherResult, identifier: str
+        ) -> TypeGuard[SimpleMatcherResult]:
+            return identifier != jedi_matcher_id
+
         results = self._complete(
             full_text=full_text, cursor_line=cursor_line, cursor_pos=cursor_column
         )
+
         non_jedi_results: Dict[str, SimpleMatcherResult] = {
             identifier: result
             for identifier, result in results.items()
-            if identifier != jedi_matcher_id
+            if is_non_jedi_result(result, identifier)
         }
 
         jedi_matches = (
-            cast(results[jedi_matcher_id], _JediMatcherResult)["completions"]
+            cast(_JediMatcherResult, results[jedi_matcher_id])["completions"]
             if jedi_matcher_id in results
             else ()
         )
@@ -2581,8 +2921,8 b' class IPCompleter(Completer):'
                 signature="",
             )
 
-        ordered = []
-        sortable = []
+        ordered: List[Completion] = []
+        sortable: List[Completion] = []
 
         for origin, result in non_jedi_results.items():
             matched_text = result["matched_fragment"]
@@ -2672,8 +3012,8 b' class IPCompleter(Completer):'
         abort_if_offset_changes: bool,
     ):
 
-        sortable = []
-        ordered = []
+        sortable: List[AnyMatcherCompletion] = []
+        ordered: List[AnyMatcherCompletion] = []
         most_recent_fragment = None
         for identifier, result in results.items():
             if identifier in skip_matchers:
@@ -2772,11 +3112,11 b' class IPCompleter(Completer):'
         )
 
         # Start with a clean slate of completions
-        results = {}
+        results: Dict[str, MatcherResult] = {}
 
         jedi_matcher_id = _get_matcher_id(self._jedi_matcher)
 
-        suppressed_matchers = set()
+        suppressed_matchers: Set[str] = set()
 
         matchers = {
             _get_matcher_id(matcher): matcher
@@ -2786,7 +3126,6 b' class IPCompleter(Completer):'
         }
 
         for matcher_id, matcher in matchers.items():
-            api_version = _get_matcher_api_version(matcher)
             matcher_id = _get_matcher_id(matcher)
 
             if matcher_id in self.disable_matchers:
@@ -2798,14 +3137,16 b' class IPCompleter(Completer):'
             if matcher_id in suppressed_matchers:
                 continue
 
+            result: MatcherResult
             try:
-                if api_version == 1:
+                if _is_matcher_v1(matcher):
                     result = _convert_matcher_v1_result_to_v2(
                         matcher(text), type=_UNKNOWN_TYPE
                     )
-                elif api_version == 2:
-                    result = cast(matcher, MatcherAPIv2)(context)
+                elif _is_matcher_v2(matcher):
+                    result = matcher(context)
                 else:
+                    api_version = _get_matcher_api_version(matcher)
                     raise ValueError(f"Unsupported API version {api_version}")
             except:
                 # Show the ugly traceback if the matcher causes an
@@ -2817,7 +3158,9 b' class IPCompleter(Completer):'
             result["matched_fragment"] = result.get("matched_fragment", context.token)
 
             if not suppressed_matchers:
-                suppression_recommended = result.get("suppress", False)
+                suppression_recommended: Union[bool, Set[str]] = result.get(
+                    "suppress", False
+                )
 
                 suppression_config = (
                     self.suppress_competing_matchers.get(matcher_id, None)
@@ -2830,10 +3173,12 b' class IPCompleter(Completer):'
                 ) and has_any_completions(result)
 
                 if should_suppress:
-                    suppression_exceptions = result.get("do_not_suppress", set())
-                    try:
+                    suppression_exceptions: Set[str] = result.get(
+                        "do_not_suppress", set()
+                    )
+                    if isinstance(suppression_recommended, Iterable):
                         to_suppress = set(suppression_recommended)
-                    except TypeError:
+                    else:
                         to_suppress = set(matchers)
                     suppressed_matchers = to_suppress - suppression_exceptions
 
@@ -2860,9 +3205,9 b' class IPCompleter(Completer):'
 
     @staticmethod
     def _deduplicate(
-        matches: Sequence[SimpleCompletion],
-    ) -> Iterable[SimpleCompletion]:
-        filtered_matches = {}
+        matches: Sequence[AnyCompletion],
+    ) -> Iterable[AnyCompletion]:
+        filtered_matches: Dict[str, AnyCompletion] = {}
         for match in matches:
             text = match.text
             if (
@@ -2874,7 +3219,7 b' class IPCompleter(Completer):'
         return filtered_matches.values()
 
     @staticmethod
-    def _sort(matches: Sequence[SimpleCompletion]):
+    def _sort(matches: Sequence[AnyCompletion]):
         return sorted(matches, key=lambda x: completions_sorting_key(x.text))
 
     @context_matcher()

IPython/core/interactiveshell.py

@@ -389,6 +389,9 b' class InteractiveShell(SingletonConfigurable):'
     displayhook_class = Type(DisplayHook)
     display_pub_class = Type(DisplayPublisher)
     compiler_class = Type(CachingCompiler)
+    inspector_class = Type(
+        oinspect.Inspector, help="Class to use to instantiate the shell inspector"
+    ).tag(config=True)
 
     sphinxify_docstring = Bool(False, help=
         """
@@ -755,10 +758,12 b' class InteractiveShell(SingletonConfigurable):'
     @observe('colors')
     def init_inspector(self, changes=None):
         # Object inspector
-        self.inspector = oinspect.Inspector(oinspect.InspectColors,
-                                            PyColorize.ANSICodeColors,
-                                            self.colors,
-                                            self.object_info_string_level)
+        self.inspector = self.inspector_class(
+            oinspect.InspectColors,
+            PyColorize.ANSICodeColors,
+            self.colors,
+            self.object_info_string_level,
+        )
 
     def init_io(self):
         # implemented in subclasses, TerminalInteractiveShell does call
@@ -3154,8 +3159,12 b' class InteractiveShell(SingletonConfigurable):'
             else:
                 cell = raw_cell
 
+        # Do NOT store paste/cpaste magic history
+        if "get_ipython().run_line_magic(" in cell and "paste" in cell:
+            store_history = False
+
         # Store raw and processed history
-        if store_history and raw_cell.strip(" %") != "paste":
+        if store_history:
             self.history_manager.store_inputs(self.execution_count, cell, raw_cell)
         if not silent:
             self.logger.log(cell, raw_cell)

IPython/core/magics/config.py

@@ -68,94 +68,22 b' class ConfigMagics(Magics):'
         To view what is configurable on a given class, just pass the class
         name::
 
-            In [2]: %config IPCompleter
-            IPCompleter(Completer) options
-            ----------------------------
-            IPCompleter.backslash_combining_completions=<Bool>
-                Enable unicode completions, e.g. \\alpha<tab> . Includes completion of latex
-                commands, unicode names, and expanding unicode characters back to latex
-                commands.
-                Current: True
-            IPCompleter.debug=<Bool>
-                Enable debug for the Completer. Mostly print extra information for
-                experimental jedi integration.
+            In [2]: %config LoggingMagics
+            LoggingMagics(Magics) options
+            ---------------------------
+            LoggingMagics.quiet=<Bool>
+                Suppress output of log state when logging is enabled
                 Current: False
-            IPCompleter.disable_matchers=<list-item-1>...
-                List of matchers to disable.
-                        The list should contain matcher identifiers (see
-                :any:`completion_matcher`).
-                Current: []
-            IPCompleter.greedy=<Bool>
-                Activate greedy completion
-                        PENDING DEPRECATION. this is now mostly taken care of with Jedi.
-                        This will enable completion on elements of lists, results of function calls, etc.,
-                        but can be unsafe because the code is actually evaluated on TAB.
-                Current: False
-            IPCompleter.jedi_compute_type_timeout=<Int>
-                Experimental: restrict time (in milliseconds) during which Jedi can compute types.
-                        Set to 0 to stop computing types. Non-zero value lower than 100ms may hurt
-                        performance by preventing jedi to build its cache.
-                Current: 400
-            IPCompleter.limit_to__all__=<Bool>
-                DEPRECATED as of version 5.0.
-                Instruct the completer to use __all__ for the completion
-                Specifically, when completing on ``object.<tab>``.
-                When True: only those names in obj.__all__ will be included.
-                When False [default]: the __all__ attribute is ignored
-                Current: False
-            IPCompleter.merge_completions=<Bool>
-                Whether to merge completion results into a single list
-                        If False, only the completion results from the first non-empty
-                        completer will be returned.
-                        As of version 8.6.0, setting the value to ``False`` is an alias for:
-                        ``IPCompleter.suppress_competing_matchers = True.``.
-                Current: True
-            IPCompleter.omit__names=<Enum>
-                Instruct the completer to omit private method names
-                        Specifically, when completing on ``object.<tab>``.
-                        When 2 [default]: all names that start with '_' will be excluded.
-                        When 1: all 'magic' names (``__foo__``) will be excluded.
-                        When 0: nothing will be excluded.
-                Choices: any of [0, 1, 2]
-                Current: 2
-            IPCompleter.profile_completions=<Bool>
-                If True, emit profiling data for completion subsystem using cProfile.
-                Current: False
-            IPCompleter.profiler_output_dir=<Unicode>
-                Template for path at which to output profile data for completions.
-                Current: '.completion_profiles'
-            IPCompleter.suppress_competing_matchers=<Union>
-                Whether to suppress completions from other *Matchers*.
-                When set to ``None`` (default) the matchers will attempt to auto-detect
-                whether suppression of other matchers is desirable. For example, at the
-                beginning of a line followed by `%` we expect a magic completion to be the
-                only applicable option, and after ``my_dict['`` we usually expect a
-                completion with an existing dictionary key.
-                If you want to disable this heuristic and see completions from all matchers,
-                set ``IPCompleter.suppress_competing_matchers = False``. To disable the
-                heuristic for specific matchers provide a dictionary mapping:
-                ``IPCompleter.suppress_competing_matchers = {'IPCompleter.dict_key_matcher':
-                False}``.
-                Set ``IPCompleter.suppress_competing_matchers = True`` to limit completions
-                to the set of matchers with the highest priority; this is equivalent to
-                ``IPCompleter.merge_completions`` and can be beneficial for performance, but
-                will sometimes omit relevant candidates from matchers further down the
-                priority list.
-                Current: None
-            IPCompleter.use_jedi=<Bool>
-                Experimental: Use Jedi to generate autocompletions. Default to True if jedi
-                is installed.
-                Current: True
 
         but the real use is in setting values::
 
-            In [3]: %config IPCompleter.greedy = True
+            In [3]: %config LoggingMagics.quiet = True
 
         and these values are read from the user_ns if they are variables::
 
-            In [4]: feeling_greedy=False
+            In [4]: feeling_quiet=False
 
-            In [5]: %config IPCompleter.greedy = feeling_greedy
+            In [5]: %config LoggingMagics.quiet = feeling_quiet
 
         """
         from traitlets.config.loader import Config

IPython/core/magics/script.py

@@ -210,7 +210,7 b' class ScriptMagics(Magics):'
 
         async def _handle_stream(stream, stream_arg, file_object):
             while True:
-                line = (await stream.readline()).decode("utf8")
+                line = (await stream.readline()).decode("utf8", errors="replace")
                 if not line:
                     break
                 if stream_arg:

IPython/core/oinspect.py

@@ -16,6 +16,7 b" __all__ = ['Inspector','InspectColors']"
 import ast
 import inspect
 from inspect import signature
+import html
 import linecache
 import warnings
 import os
@@ -530,8 +531,8 b' class Inspector(Colorable):'
 
         """
         defaults = {
-            'text/plain': text,
-            'text/html': '<pre>' + text + '</pre>'
+            "text/plain": text,
+            "text/html": f"<pre>{html.escape(text)}</pre>",
         }
 
         if formatter is None:
@@ -542,66 +543,66 b' class Inspector(Colorable):'
             if not isinstance(formatted, dict):
                 # Handle the deprecated behavior of a formatter returning
                 # a string instead of a mime bundle.
-                return {
-                    'text/plain': formatted,
-                    'text/html': '<pre>' + formatted + '</pre>'
-                }
+                return {"text/plain": formatted, "text/html": f"<pre>{formatted}</pre>"}
 
             else:
                 return dict(defaults, **formatted)
 
 
     def format_mime(self, bundle):
-
-        text_plain = bundle['text/plain']
-
-        text = ''
-        heads, bodies = list(zip(*text_plain))
-        _len = max(len(h) for h in heads)
-
-        for head, body in zip(heads, bodies):
-            body = body.strip('\n')
-            delim = '\n' if '\n' in body else ' '
-            text += self.__head(head+':') + (_len - len(head))*' ' +delim + body +'\n'
-
-        bundle['text/plain'] = text
+        """Format a mimebundle being created by _make_info_unformatted into a real mimebundle"""
+        # Format text/plain mimetype
+        if isinstance(bundle["text/plain"], (list, tuple)):
+            # bundle['text/plain'] is a list of (head, formatted body) pairs
+            lines = []
+            _len = max(len(h) for h, _ in bundle["text/plain"])
+
+            for head, body in bundle["text/plain"]:
+                body = body.strip("\n")
+                delim = "\n" if "\n" in body else " "
+                lines.append(
+                    f"{self.__head(head+':')}{(_len - len(head))*' '}{delim}{body}"
+                )
+
+            bundle["text/plain"] = "\n".join(lines)
+
+        # Format the text/html mimetype
+        if isinstance(bundle["text/html"], (list, tuple)):
+            # bundle['text/html'] is a list of (head, formatted body) pairs
+            bundle["text/html"] = "\n".join(
+                (f"<h1>{head}</h1>\n{body}" for (head, body) in bundle["text/html"])
+            )
         return bundle
 
-    def _get_info(
-        self, obj, oname="", formatter=None, info=None, detail_level=0, omit_sections=()
+    def _append_info_field(
+        self, bundle, title: str, key: str, info, omit_sections, formatter
     ):
-        """Retrieve an info dict and format it.
-
-        Parameters
-        ----------
-        obj : any
-            Object to inspect and return info from
-        oname : str (default: ''):
-            Name of the variable pointing to `obj`.
-        formatter : callable
-        info
-            already computed information
-        detail_level : integer
-            Granularity of detail level, if set to 1, give more information.
-        omit_sections : container[str]
-            Titles or keys to omit from output (can be set, tuple, etc., anything supporting `in`)
-        """
-
-        info = self.info(obj, oname=oname, info=info, detail_level=detail_level)
-
-        _mime = {
-            'text/plain': [],
-            'text/html': '',
+        """Append an info value to the unformatted mimebundle being constructed by _make_info_unformatted"""
+        if title in omit_sections or key in omit_sections:
+            return
+        field = info[key]
+        if field is not None:
+            formatted_field = self._mime_format(field, formatter)
+            bundle["text/plain"].append((title, formatted_field["text/plain"]))
+            bundle["text/html"].append((title, formatted_field["text/html"]))
+
+    def _make_info_unformatted(self, obj, info, formatter, detail_level, omit_sections):
+        """Assemble the mimebundle as unformatted lists of information"""
+        bundle = {
+            "text/plain": [],
+            "text/html": [],
         }
 
-        def append_field(bundle, title:str, key:str, formatter=None):
-            if title in omit_sections or key in omit_sections:
-                return
-            field = info[key]
-            if field is not None:
-                formatted_field = self._mime_format(field, formatter)
-                bundle['text/plain'].append((title, formatted_field['text/plain']))
-                bundle['text/html'] += '<h1>' + title + '</h1>\n' + formatted_field['text/html'] + '\n'
+        # A convenience function to simplify calls below
+        def append_field(bundle, title: str, key: str, formatter=None):
+            self._append_info_field(
+                bundle,
+                title=title,
+                key=key,
+                info=info,
+                omit_sections=omit_sections,
+                formatter=formatter,
+            )
 
         def code_formatter(text):
             return {
@@ -609,57 +610,82 b' class Inspector(Colorable):'
                 'text/html': pylight(text)
             }
 
-        if info['isalias']:
-            append_field(_mime, 'Repr', 'string_form')
+        if info["isalias"]:
+            append_field(bundle, "Repr", "string_form")
 
         elif info['ismagic']:
             if detail_level > 0:
-                append_field(_mime, 'Source', 'source', code_formatter)
+                append_field(bundle, "Source", "source", code_formatter)
             else:
-                append_field(_mime, 'Docstring', 'docstring', formatter)
-            append_field(_mime, 'File', 'file')
+                append_field(bundle, "Docstring", "docstring", formatter)
+            append_field(bundle, "File", "file")
 
         elif info['isclass'] or is_simple_callable(obj):
             # Functions, methods, classes
-            append_field(_mime, 'Signature', 'definition', code_formatter)
-            append_field(_mime, 'Init signature', 'init_definition', code_formatter)
-            append_field(_mime, 'Docstring', 'docstring', formatter)
-            if detail_level > 0 and info['source']:
-                append_field(_mime, 'Source', 'source', code_formatter)
+            append_field(bundle, "Signature", "definition", code_formatter)
+            append_field(bundle, "Init signature", "init_definition", code_formatter)
+            append_field(bundle, "Docstring", "docstring", formatter)
+            if detail_level > 0 and info["source"]:
+                append_field(bundle, "Source", "source", code_formatter)
             else:
-                append_field(_mime, 'Init docstring', 'init_docstring', formatter)
+                append_field(bundle, "Init docstring", "init_docstring", formatter)
 
-            append_field(_mime, 'File', 'file')
-            append_field(_mime, 'Type', 'type_name')
-            append_field(_mime, 'Subclasses', 'subclasses')
+            append_field(bundle, "File", "file")
+            append_field(bundle, "Type", "type_name")
+            append_field(bundle, "Subclasses", "subclasses")
 
         else:
             # General Python objects
-            append_field(_mime, 'Signature', 'definition', code_formatter)
-            append_field(_mime, 'Call signature', 'call_def', code_formatter)
-            append_field(_mime, 'Type', 'type_name')
-            append_field(_mime, 'String form', 'string_form')
+            append_field(bundle, "Signature", "definition", code_formatter)
+            append_field(bundle, "Call signature", "call_def", code_formatter)
+            append_field(bundle, "Type", "type_name")
+            append_field(bundle, "String form", "string_form")
 
             # Namespace
-            if info['namespace'] != 'Interactive':
-                append_field(_mime, 'Namespace', 'namespace')
+            if info["namespace"] != "Interactive":
+                append_field(bundle, "Namespace", "namespace")
 
-            append_field(_mime, 'Length', 'length')
-            append_field(_mime, 'File', 'file')
+            append_field(bundle, "Length", "length")
+            append_field(bundle, "File", "file")
 
             # Source or docstring, depending on detail level and whether
             # source found.
-            if detail_level > 0 and info['source']:
-                append_field(_mime, 'Source', 'source', code_formatter)
+            if detail_level > 0 and info["source"]:
+                append_field(bundle, "Source", "source", code_formatter)
             else:
-                append_field(_mime, 'Docstring', 'docstring', formatter)
+                append_field(bundle, "Docstring", "docstring", formatter)
+
+            append_field(bundle, "Class docstring", "class_docstring", formatter)
+            append_field(bundle, "Init docstring", "init_docstring", formatter)
+            append_field(bundle, "Call docstring", "call_docstring", formatter)
+        return bundle
 
-            append_field(_mime, 'Class docstring', 'class_docstring', formatter)
-            append_field(_mime, 'Init docstring', 'init_docstring', formatter)
-            append_field(_mime, 'Call docstring', 'call_docstring', formatter)
 
+    def _get_info(
+        self, obj, oname="", formatter=None, info=None, detail_level=0, omit_sections=()
+    ):
+        """Retrieve an info dict and format it.
+
+        Parameters
+        ----------
+        obj : any
+            Object to inspect and return info from
+        oname : str (default: ''):
+            Name of the variable pointing to `obj`.
+        formatter : callable
+        info
+            already computed information
+        detail_level : integer
+            Granularity of detail level, if set to 1, give more information.
+        omit_sections : container[str]
+            Titles or keys to omit from output (can be set, tuple, etc., anything supporting `in`)
+        """
 
-        return self.format_mime(_mime)
+        info = self.info(obj, oname=oname, info=info, detail_level=detail_level)
+        bundle = self._make_info_unformatted(
+            obj, info, formatter, detail_level=detail_level, omit_sections=omit_sections
+        )
+        return self.format_mime(bundle)
 
     def pinfo(
         self,

IPython/core/release.py

@@ -16,7 +16,7 b''
 # release.  'dev' as a _version_extra string means this is a development
 # version
 _version_major = 8
-_version_minor = 8
+_version_minor = 9
 _version_patch = 0
 _version_extra = ".dev"
 # _version_extra = "rc1"

IPython/core/tests/test_completer.py

@@ -24,6 +24,7 b' from IPython.core.completer import ('
     provisionalcompleter,
     match_dict_keys,
     _deduplicate_completions,
+    _match_number_in_dict_key_prefix,
     completion_matcher,
     SimpleCompletion,
     CompletionContext,
@@ -98,7 +99,7 b' def test_unicode_range():'
     assert len_exp == len_test, message
 
     # fail if new unicode symbols have been added. 
-    assert len_exp <= 138552, message
+    assert len_exp <= 143041, message
 
 
 @contextmanager
@@ -113,6 +114,17 b' def greedy_completion():'
 
 
 @contextmanager
+def evaluation_policy(evaluation: str):
+    ip = get_ipython()
+    evaluation_original = ip.Completer.evaluation
+    try:
+        ip.Completer.evaluation = evaluation
+        yield
+    finally:
+        ip.Completer.evaluation = evaluation_original
+
+
+@contextmanager
 def custom_matchers(matchers):
     ip = get_ipython()
     try:
@@ -170,7 +182,6 b' def check_line_split(splitter, test_specs):'
         out = splitter.split_line(line, cursor_pos)
         assert out == split
 
-
 def test_line_split():
     """Basic line splitter test with default specs."""
     sp = completer.CompletionSplitter()
@@ -522,10 +533,10 b' class TestCompleter(unittest.TestCase):'
 
     def test_greedy_completions(self):
         """
-        Test the capability of the Greedy completer. 
+        Test the capability of the Greedy completer.
 
         Most of the test here does not really show off the greedy completer, for proof
-        each of the text below now pass with Jedi. The greedy completer is capable of more. 
+        each of the text below now pass with Jedi. The greedy completer is capable of more.
 
         See the :any:`test_dict_key_completion_contexts`
 
@@ -841,18 +852,45 b' class TestCompleter(unittest.TestCase):'
         """
         delims = " \t\n`!@#$^&*()=+[{]}\\|;:'\",<>?"
 
-        keys = ["foo", b"far"]
-        assert match_dict_keys(keys, "b'", delims=delims) == ("'", 2, ["far"])
-        assert match_dict_keys(keys, "b'f", delims=delims) == ("'", 2, ["far"])
-        assert match_dict_keys(keys, 'b"', delims=delims) == ('"', 2, ["far"])
-        assert match_dict_keys(keys, 'b"f', delims=delims) == ('"', 2, ["far"])
-
-        assert match_dict_keys(keys, "'", delims=delims) == ("'", 1, ["foo"])
-        assert match_dict_keys(keys, "'f", delims=delims) == ("'", 1, ["foo"])
-        assert match_dict_keys(keys, '"', delims=delims) == ('"', 1, ["foo"])
-        assert match_dict_keys(keys, '"f', delims=delims) == ('"', 1, ["foo"])
+        def match(*args, **kwargs):
+            quote, offset, matches = match_dict_keys(*args, delims=delims, **kwargs)
+            return quote, offset, list(matches)
 
-        match_dict_keys
+        keys = ["foo", b"far"]
+        assert match(keys, "b'") == ("'", 2, ["far"])
+        assert match(keys, "b'f") == ("'", 2, ["far"])
+        assert match(keys, 'b"') == ('"', 2, ["far"])
+        assert match(keys, 'b"f') == ('"', 2, ["far"])
+
+        assert match(keys, "'") == ("'", 1, ["foo"])
+        assert match(keys, "'f") == ("'", 1, ["foo"])
+        assert match(keys, '"') == ('"', 1, ["foo"])
+        assert match(keys, '"f') == ('"', 1, ["foo"])
+
+        # Completion on first item of tuple
+        keys = [("foo", 1111), ("foo", 2222), (3333, "bar"), (3333, "test")]
+        assert match(keys, "'f") == ("'", 1, ["foo"])
+        assert match(keys, "33") == ("", 0, ["3333"])
+
+        # Completion on numbers
+        keys = [
+            0xDEADBEEF,
+            1111,
+            1234,
+            "1999",
+            0b10101,
+            22,
+        ]  # 0xDEADBEEF = 3735928559; 0b10101 = 21
+        assert match(keys, "0xdead") == ("", 0, ["0xdeadbeef"])
+        assert match(keys, "1") == ("", 0, ["1111", "1234"])
+        assert match(keys, "2") == ("", 0, ["21", "22"])
+        assert match(keys, "0b101") == ("", 0, ["0b10101", "0b10110"])
+
+        # Should yield on variables
+        assert match(keys, "a_variable") == ("", 0, [])
+
+        # Should pass over invalid literals
+        assert match(keys, "'' ''") == ("", 0, [])
 
     def test_match_dict_keys_tuple(self):
         """
@@ -860,28 +898,94 b' class TestCompleter(unittest.TestCase):'
         does return what expected, and does not crash.
         """
         delims = " \t\n`!@#$^&*()=+[{]}\\|;:'\",<>?"
-        
+
         keys = [("foo", "bar"), ("foo", "oof"), ("foo", b"bar"), ('other', 'test')]
 
+        def match(*args, extra=None, **kwargs):
+            quote, offset, matches = match_dict_keys(
+                *args, delims=delims, extra_prefix=extra, **kwargs
+            )
+            return quote, offset, list(matches)
+
         # Completion on first key == "foo"
-        assert match_dict_keys(keys, "'", delims=delims, extra_prefix=("foo",)) == ("'", 1, ["bar", "oof"])
-        assert match_dict_keys(keys, "\"", delims=delims, extra_prefix=("foo",)) == ("\"", 1, ["bar", "oof"])
-        assert match_dict_keys(keys, "'o", delims=delims, extra_prefix=("foo",)) == ("'", 1, ["oof"])
-        assert match_dict_keys(keys, "\"o", delims=delims, extra_prefix=("foo",)) == ("\"", 1, ["oof"])
-        assert match_dict_keys(keys, "b'", delims=delims, extra_prefix=("foo",)) == ("'", 2, ["bar"])
-        assert match_dict_keys(keys, "b\"", delims=delims, extra_prefix=("foo",)) == ("\"", 2, ["bar"])
-        assert match_dict_keys(keys, "b'b", delims=delims, extra_prefix=("foo",)) == ("'", 2, ["bar"])
-        assert match_dict_keys(keys, "b\"b", delims=delims, extra_prefix=("foo",)) == ("\"", 2, ["bar"])
+        assert match(keys, "'", extra=("foo",)) == ("'", 1, ["bar", "oof"])
+        assert match(keys, '"', extra=("foo",)) == ('"', 1, ["bar", "oof"])
+        assert match(keys, "'o", extra=("foo",)) == ("'", 1, ["oof"])
+        assert match(keys, '"o', extra=("foo",)) == ('"', 1, ["oof"])
+        assert match(keys, "b'", extra=("foo",)) == ("'", 2, ["bar"])
+        assert match(keys, 'b"', extra=("foo",)) == ('"', 2, ["bar"])
+        assert match(keys, "b'b", extra=("foo",)) == ("'", 2, ["bar"])
+        assert match(keys, 'b"b', extra=("foo",)) == ('"', 2, ["bar"])
 
         # No Completion
-        assert match_dict_keys(keys, "'", delims=delims, extra_prefix=("no_foo",)) == ("'", 1, [])
-        assert match_dict_keys(keys, "'", delims=delims, extra_prefix=("fo",)) == ("'", 1, [])
+        assert match(keys, "'", extra=("no_foo",)) == ("'", 1, [])
+        assert match(keys, "'", extra=("fo",)) == ("'", 1, [])
+
+        keys = [("foo1", "foo2", "foo3", "foo4"), ("foo1", "foo2", "bar", "foo4")]
+        assert match(keys, "'foo", extra=("foo1",)) == ("'", 1, ["foo2"])
+        assert match(keys, "'foo", extra=("foo1", "foo2")) == ("'", 1, ["foo3"])
+        assert match(keys, "'foo", extra=("foo1", "foo2", "foo3")) == ("'", 1, ["foo4"])
+        assert match(keys, "'foo", extra=("foo1", "foo2", "foo3", "foo4")) == (
+            "'",
+            1,
+            [],
+        )
+
+        keys = [("foo", 1111), ("foo", "2222"), (3333, "bar"), (3333, 4444)]
+        assert match(keys, "'", extra=("foo",)) == ("'", 1, ["2222"])
+        assert match(keys, "", extra=("foo",)) == ("", 0, ["1111", "'2222'"])
+        assert match(keys, "'", extra=(3333,)) == ("'", 1, ["bar"])
+        assert match(keys, "", extra=(3333,)) == ("", 0, ["'bar'", "4444"])
+        assert match(keys, "'", extra=("3333",)) == ("'", 1, [])
+        assert match(keys, "33") == ("", 0, ["3333"])
+
+    def test_dict_key_completion_closures(self):
+        ip = get_ipython()
+        complete = ip.Completer.complete
+        ip.Completer.auto_close_dict_keys = True
 
-        keys = [('foo1', 'foo2', 'foo3', 'foo4'), ('foo1', 'foo2', 'bar', 'foo4')]
-        assert match_dict_keys(keys, "'foo", delims=delims, extra_prefix=('foo1',)) == ("'", 1, ["foo2", "foo2"])
-        assert match_dict_keys(keys, "'foo", delims=delims, extra_prefix=('foo1', 'foo2')) == ("'", 1, ["foo3"])
-        assert match_dict_keys(keys, "'foo", delims=delims, extra_prefix=('foo1', 'foo2', 'foo3')) == ("'", 1, ["foo4"])
-        assert match_dict_keys(keys, "'foo", delims=delims, extra_prefix=('foo1', 'foo2', 'foo3', 'foo4')) == ("'", 1, [])
+        ip.user_ns["d"] = {
+            # tuple only
+            ("aa", 11): None,
+            # tuple and non-tuple
+            ("bb", 22): None,
+            "bb": None,
+            # non-tuple only
+            "cc": None,
+            # numeric tuple only
+            (77, "x"): None,
+            # numeric tuple and non-tuple
+            (88, "y"): None,
+            88: None,
+            # numeric non-tuple only
+            99: None,
+        }
+
+        _, matches = complete(line_buffer="d[")
+        # should append `, ` if matches a tuple only
+        self.assertIn("'aa', ", matches)
+        # should not append anything if matches a tuple and an item
+        self.assertIn("'bb'", matches)
+        # should append `]` if matches and item only
+        self.assertIn("'cc']", matches)
+
+        # should append `, ` if matches a tuple only
+        self.assertIn("77, ", matches)
+        # should not append anything if matches a tuple and an item
+        self.assertIn("88", matches)
+        # should append `]` if matches and item only
+        self.assertIn("99]", matches)
+
+        _, matches = complete(line_buffer="d['aa', ")
+        # should restrict matches to those matching tuple prefix
+        self.assertIn("11]", matches)
+        self.assertNotIn("'bb'", matches)
+        self.assertNotIn("'bb', ", matches)
+        self.assertNotIn("'bb']", matches)
+        self.assertNotIn("'cc'", matches)
+        self.assertNotIn("'cc', ", matches)
+        self.assertNotIn("'cc']", matches)
+        ip.Completer.auto_close_dict_keys = False
 
     def test_dict_key_completion_string(self):
         """Test dictionary key completion for string keys"""
@@ -1038,6 +1142,35 b' class TestCompleter(unittest.TestCase):'
         self.assertNotIn("foo", matches)
         self.assertNotIn("bar", matches)
 
+    def test_dict_key_completion_numbers(self):
+        ip = get_ipython()
+        complete = ip.Completer.complete
+
+        ip.user_ns["d"] = {
+            0xDEADBEEF: None,  # 3735928559
+            1111: None,
+            1234: None,
+            "1999": None,
+            0b10101: None,  # 21
+            22: None,
+        }
+        _, matches = complete(line_buffer="d[1")
+        self.assertIn("1111", matches)
+        self.assertIn("1234", matches)
+        self.assertNotIn("1999", matches)
+        self.assertNotIn("'1999'", matches)
+
+        _, matches = complete(line_buffer="d[0xdead")
+        self.assertIn("0xdeadbeef", matches)
+
+        _, matches = complete(line_buffer="d[2")
+        self.assertIn("21", matches)
+        self.assertIn("22", matches)
+
+        _, matches = complete(line_buffer="d[0b101")
+        self.assertIn("0b10101", matches)
+        self.assertIn("0b10110", matches)
+
     def test_dict_key_completion_contexts(self):
         """Test expression contexts in which dict key completion occurs"""
         ip = get_ipython()
@@ -1050,6 +1183,7 b' class TestCompleter(unittest.TestCase):'
 
         ip.user_ns["C"] = C
         ip.user_ns["get"] = lambda: d
+        ip.user_ns["nested"] = {"x": d}
 
         def assert_no_completion(**kwargs):
             _, matches = complete(**kwargs)
@@ -1075,6 +1209,13 b' class TestCompleter(unittest.TestCase):'
         assert_completion(line_buffer="(d[")
         assert_completion(line_buffer="C.data[")
 
+        # nested dict completion
+        assert_completion(line_buffer="nested['x'][")
+
+        with evaluation_policy("minimal"):
+            with pytest.raises(AssertionError):
+                assert_completion(line_buffer="nested['x'][")
+
         # greedy flag
         def assert_completion(**kwargs):
             _, matches = complete(**kwargs)
@@ -1162,12 +1303,22 b' class TestCompleter(unittest.TestCase):'
         _, matches = complete(line_buffer="d['")
         self.assertIn("my_head", matches)
         self.assertIn("my_data", matches)
-        # complete on a nested level
-        with greedy_completion():
+
+        def completes_on_nested():
             ip.user_ns["d"] = numpy.zeros(2, dtype=dt)
             _, matches = complete(line_buffer="d[1]['my_head']['")
             self.assertTrue(any(["my_dt" in m for m in matches]))
             self.assertTrue(any(["my_df" in m for m in matches]))
+        # complete on a nested level
+        with greedy_completion():
+            completes_on_nested()
+
+        with evaluation_policy("limited"):
+            completes_on_nested()
+
+        with evaluation_policy("minimal"):
+            with pytest.raises(AssertionError):
+                completes_on_nested()
 
     @dec.skip_without("pandas")
     def test_dataframe_key_completion(self):
@@ -1180,6 +1331,17 b' class TestCompleter(unittest.TestCase):'
         _, matches = complete(line_buffer="d['")
         self.assertIn("hello", matches)
         self.assertIn("world", matches)
+        _, matches = complete(line_buffer="d.loc[:, '")
+        self.assertIn("hello", matches)
+        self.assertIn("world", matches)
+        _, matches = complete(line_buffer="d.loc[1:, '")
+        self.assertIn("hello", matches)
+        _, matches = complete(line_buffer="d.loc[1:1, '")
+        self.assertIn("hello", matches)
+        _, matches = complete(line_buffer="d.loc[1:1:-1, '")
+        self.assertIn("hello", matches)
+        _, matches = complete(line_buffer="d.loc[::, '")
+        self.assertIn("hello", matches)
 
     def test_dict_key_completion_invalids(self):
         """Smoke test cases dict key completion can't handle"""
@@ -1503,3 +1665,38 b' class TestCompleter(unittest.TestCase):'
             _(["completion_b"])
             a_matcher.matcher_priority = 3
             _(["completion_a"])
+
+
[e-mail](
+    "input, expected",
+    [
+        ["1.234", "1.234"],
+        # should match signed numbers
+        ["+1", "+1"],
+        ["-1", "-1"],
+        ["-1.0", "-1.0"],
+        ["-1.", "-1."],
+        ["+1.", "+1."],
+        [".1", ".1"],
+        # should not match non-numbers
+        ["1..", None],
+        ["..", None],
+        [".1.", None],
+        # should match after comma
+        [",1", "1"],
+        [", 1", "1"],
+        [", .1", ".1"],
+        [", +.1", "+.1"],
+        # should not match after trailing spaces
+        [".1 ", None],
+        # some complex cases
+        ["0b_0011_1111_0100_1110", "0b_0011_1111_0100_1110"],
+        ["0xdeadbeef", "0xdeadbeef"],
+        ["0b_1110_0101", "0b_1110_0101"],
+        # should not match if in an operation
+        ["1 + 1", None],
+        [", 1 + 1", None],
+    ],
+)
+def test_match_numeric_literal_for_dict_key(input, expected):
+    assert _match_number_in_dict_key_prefix(input) == expected

IPython/extensions/tests/test_autoreload.py

@@ -367,7 +367,8 b' class TestAutoreload(Fixture):'
         self.shell.run_code("assert func2() == 'changed'")
         self.shell.run_code("t = Test(); assert t.new_func() == 'changed'")
         self.shell.run_code("assert number == 1")
-        self.shell.run_code("assert TestEnum.B.value == 'added'")
+        if sys.version_info < (3, 12):
+            self.shell.run_code("assert TestEnum.B.value == 'added'")
 
         # ----------- TEST IMPORT FROM MODULE --------------------------
 

IPython/lib/tests/test_lexers.py

@@ -4,11 +4,14 b''
 # Distributed under the terms of the Modified BSD License.
 
 from unittest import TestCase
+from pygments import __version__ as pygments_version
 from pygments.token import Token
 from pygments.lexers import BashLexer
 
 from .. import lexers
 
+pyg214 = tuple(int(x) for x in pygments_version.split(".")[:2]) >= (2, 14)
+
 
 class TestLexers(TestCase):
     """Collection of lexers tests"""
@@ -18,25 +21,26 b' class TestLexers(TestCase):'
 
     def testIPythonLexer(self):
         fragment = '!echo $HOME\n'
-        tokens = [
+        bash_tokens = [
             (Token.Operator, '!'),
         ]
-        tokens.extend(self.bash_lexer.get_tokens(fragment[1:]))
-        self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
+        bash_tokens.extend(self.bash_lexer.get_tokens(fragment[1:]))
+        ipylex_token = list(self.lexer.get_tokens(fragment))
+        assert bash_tokens[:-1] == ipylex_token[:-1]
 
-        fragment_2 = '!' + fragment
+        fragment_2 = "!" + fragment
         tokens_2 = [
             (Token.Operator, '!!'),
-        ] + tokens[1:]
-        self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2)))
+        ] + bash_tokens[1:]
+        assert tokens_2[:-1] == list(self.lexer.get_tokens(fragment_2))[:-1]
 
         fragment_2 = '\t %%!\n' + fragment[1:]
         tokens_2 = [
             (Token.Text, '\t '),
             (Token.Operator, '%%!'),
             (Token.Text, '\n'),
-        ] + tokens[1:]
-        self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2)))
+        ] + bash_tokens[1:]
+        assert tokens_2 == list(self.lexer.get_tokens(fragment_2))
 
         fragment_2 = 'x = ' + fragment
         tokens_2 = [
@@ -44,8 +48,8 b' class TestLexers(TestCase):'
             (Token.Text, ' '),
             (Token.Operator, '='),
             (Token.Text, ' '),
-        ] + tokens
-        self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2)))
+        ] + bash_tokens
+        assert tokens_2[:-1] == list(self.lexer.get_tokens(fragment_2))[:-1]
 
         fragment_2 = 'x, = ' + fragment
         tokens_2 = [
@@ -54,8 +58,8 b' class TestLexers(TestCase):'
             (Token.Text, ' '),
             (Token.Operator, '='),
             (Token.Text, ' '),
-        ] + tokens
-        self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2)))
+        ] + bash_tokens
+        assert tokens_2[:-1] == list(self.lexer.get_tokens(fragment_2))[:-1]
 
         fragment_2 = 'x, = %sx ' + fragment[1:]
         tokens_2 = [
@@ -67,8 +71,10 b' class TestLexers(TestCase):'
             (Token.Operator, '%'),
             (Token.Keyword, 'sx'),
             (Token.Text, ' '),
-        ] + tokens[1:]
-        self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2)))
+        ] + bash_tokens[1:]
+        if tokens_2[7] == (Token.Text, " ") and pyg214:  # pygments 2.14+
+            tokens_2[7] = (Token.Text.Whitespace, " ")
+        assert tokens_2[:-1] == list(self.lexer.get_tokens(fragment_2))[:-1]
 
         fragment_2 = 'f = %R function () {}\n'
         tokens_2 = [
@@ -80,7 +86,7 b' class TestLexers(TestCase):'
             (Token.Keyword, 'R'),
             (Token.Text, ' function () {}\n'),
         ]
-        self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2)))
+        assert tokens_2 == list(self.lexer.get_tokens(fragment_2))
 
         fragment_2 = '\t%%xyz\n$foo\n'
         tokens_2 = [
@@ -89,7 +95,7 b' class TestLexers(TestCase):'
             (Token.Keyword, 'xyz'),
             (Token.Text, '\n$foo\n'),
         ]
-        self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2)))
+        assert tokens_2 == list(self.lexer.get_tokens(fragment_2))
 
         fragment_2 = '%system?\n'
         tokens_2 = [
@@ -98,7 +104,7 b' class TestLexers(TestCase):'
             (Token.Operator, '?'),
             (Token.Text, '\n'),
         ]
-        self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2)))
+        assert tokens_2[:-1] == list(self.lexer.get_tokens(fragment_2))[:-1]
 
         fragment_2 = 'x != y\n'
         tokens_2 = [
@@ -109,7 +115,7 b' class TestLexers(TestCase):'
             (Token.Name, 'y'),
             (Token.Text, '\n'),
         ]
-        self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2)))
+        assert tokens_2[:-1] == list(self.lexer.get_tokens(fragment_2))[:-1]
 
         fragment_2 = ' ?math.sin\n'
         tokens_2 = [
@@ -118,7 +124,7 b' class TestLexers(TestCase):'
             (Token.Text, 'math.sin'),
             (Token.Text, '\n'),
         ]
-        self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2)))
+        assert tokens_2[:-1] == list(self.lexer.get_tokens(fragment_2))[:-1]
 
         fragment = ' *int*?\n'
         tokens = [
@@ -126,7 +132,7 b' class TestLexers(TestCase):'
             (Token.Operator, '?'),
             (Token.Text, '\n'),
         ]
-        self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
+        assert tokens == list(self.lexer.get_tokens(fragment))
 
         fragment = '%%writefile -a foo.py\nif a == b:\n    pass'
         tokens = [
@@ -145,7 +151,9 b' class TestLexers(TestCase):'
             (Token.Keyword, 'pass'),
             (Token.Text, '\n'),
         ]
-        self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
+        if tokens[10] == (Token.Text, "\n") and pyg214:  # pygments 2.14+
+            tokens[10] = (Token.Text.Whitespace, "\n")
+        assert tokens[:-1] == list(self.lexer.get_tokens(fragment))[:-1]
 
         fragment = '%%timeit\nmath.sin(0)'
         tokens = [
@@ -173,4 +181,4 b' class TestLexers(TestCase):'
             (Token.Punctuation, '>'),
             (Token.Text, '\n'),
         ]
-        self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
+        assert tokens == list(self.lexer.get_tokens(fragment))

IPython/terminal/magics.py

@@ -147,7 +147,7 b' class TerminalMagics(Magics):'
 
         sentinel = opts.get('s', u'--')
         block = '\n'.join(get_pasted_lines(sentinel, quiet=quiet))
-        self.store_or_execute(block, name, store_history=False)
+        self.store_or_execute(block, name, store_history=True)
 
     @line_magic
     def paste(self, parameter_s=''):

IPython/terminal/shortcuts.py

@@ -68,10 +68,15 b' def create_ipython_shortcuts(shell):'
         reformat_text_before_cursor(event.current_buffer, event.current_buffer.document, shell)
         event.current_buffer.validate_and_handle()
 
-    kb.add('escape', 'enter', filter=(has_focus(DEFAULT_BUFFER)
-                            & ~has_selection
-                            & insert_mode
-                                      ))(reformat_and_execute)
+    @Condition
+    def ebivim():
+        return shell.emacs_bindings_in_vi_insert_mode
+
+    kb.add(
+        "escape",
+        "enter",
+        filter=(has_focus(DEFAULT_BUFFER) & ~has_selection & insert_mode & ebivim),
+    )(reformat_and_execute)
 
     kb.add("c-\\")(quit)
 
@@ -333,10 +338,6 b' def create_ipython_shortcuts(shell):'
     if sys.platform == "win32":
         kb.add("c-v", filter=(has_focus(DEFAULT_BUFFER) & ~vi_mode))(win_paste)
 
-    @Condition
-    def ebivim():
-        return shell.emacs_bindings_in_vi_insert_mode
-
     focused_insert_vi = has_focus(DEFAULT_BUFFER) & vi_insert_mode
 
     @kb.add("end", filter=has_focus(DEFAULT_BUFFER) & (ebivim | ~vi_insert_mode))

docs/source/whatsnew/version8.rst

@@ -2,6 +2,44 b''
  8.x Series
 ============
 
+.. _version 8.8.0:
+
+IPython 8.8.0
+-------------
+
+First release of IPython in 2023 as there was no release at the end of
+December.
+
+This is an unusually big release (relatively speaking) with more than 15 Pull
+Requests merge.
+
+Of particular interest are:
+
+ - :ghpull:`13852` that replace the greedy completer and improve
+   completion, in particular for dictionary keys.
+ - :ghpull:`13858` that adds ``py.typed`` to ``setup.cfg`` to make sure it is
+   bundled in wheels.
+ - :ghpull:`13869` that implements tab completions for IPython options in the
+   shell when using `argcomplete <https://github.com/kislyuk/argcomplete>`. I
+   believe this also needs a recent version of Traitlets.
+ - :ghpull:`13865` makes the ``inspector`` class of `InteractiveShell`
+   configurable.
+ - :ghpull:`13880` that remove minor-version entrypoints as the minor version
+   entry points that would be included in the wheel would be the one of the
+   Python version that was used to build the ``whl`` file.
+
+In no particular order, the rest of the changes update the test suite to be
+compatible with Pygments 2.14, various docfixes, testing on more recent python
+versions and various updates.
+
+As usual you can find the full list of PRs on GitHub under `the 8.8 milestone
+<https://github.com/ipython/ipython/milestone/110>`__.
+
+Many thanks to @krassowski for the many PRs and @jasongrout for reviewing and
+merging contributions.
+
+Thanks to the `D. E. Shaw group <https://deshaw.com/>`__ for sponsoring
+work on IPython and related libraries.
 
 .. _version 8.7.0:
 
@@ -138,7 +176,7 b' Here is a non exhaustive list of changes that have been implemented for IPython'
  - Fix paste magic on wayland. :ghpull:`13671`
  - show maxlen in deque's repr. :ghpull:`13648`
 
-Restore line numbers for Input 
+Restore line numbers for Input
 ------------------------------
 
 Line number information in tracebacks from input are restored.
@@ -269,7 +307,7 b' Thanks to the `D. E. Shaw group <https://deshaw.com/>`__ for sponsoring'
 work on IPython and related libraries.
 
 .. _version 8.1.1:
-   
+
 IPython 8.1.1
 -------------
 
@@ -403,10 +441,10 b' The 8.x branch started diverging from its predecessor around IPython 7.12'
 (January 2020).
 
 This release contains 250+ pull requests, in addition to many of the features
-and backports that have made it to the 7.x branch. Please see the 
+and backports that have made it to the 7.x branch. Please see the
 `8.0 milestone <https://github.com/ipython/ipython/milestone/73?closed=1>`__ for the full list of pull requests.
 
-Please feel free to send pull requests to updates those notes after release, 
+Please feel free to send pull requests to updates those notes after release,
 I have likely forgotten a few things reviewing 250+ PRs.
 
 Dependencies changes/downstream packaging
@@ -421,7 +459,7 b' looking for help to do so.'
  - minimal Python is now 3.8
  - ``nose`` is not a testing requirement anymore
  - ``pytest`` replaces nose.
- - ``iptest``/``iptest3`` cli entrypoints do not exists anymore. 
+ - ``iptest``/``iptest3`` cli entrypoints do not exists anymore.
  - minimum officially support ``numpy`` version has been bumped, but this should
    not have much effect on packaging.
 
@@ -443,7 +481,7 b' deprecation warning:'
  - Please add **since which version** something is deprecated.
 
 As a side note, it is much easier to conditionally compare version
-numbers rather than using ``try/except`` when functionality changes with a version. 
+numbers rather than using ``try/except`` when functionality changes with a version.
 
 I won't list all the removed features here, but modules like ``IPython.kernel``,
 which was just a shim module around ``ipykernel`` for the past 8 years, have been
@@ -475,7 +513,7 b' by mypy.'
 Featured changes
 ----------------
 
-Here is a features list of changes in IPython 8.0. This is of course non-exhaustive. 
+Here is a features list of changes in IPython 8.0. This is of course non-exhaustive.
 Please note as well that many features have been added in the 7.x branch as well
 (and hence why you want to read the 7.x what's new notes), in particular
 features contributed by QuantStack (with respect to debugger protocol and Xeus
@@ -523,7 +561,7 b' The error traceback is now correctly formatted, showing the cell number in which'
 
     ZeroDivisionError: division by zero
 
-The ``stack_data`` package has been integrated, which provides smarter information in the traceback; 
+The ``stack_data`` package has been integrated, which provides smarter information in the traceback;
 in particular it will highlight the AST node where an error occurs which can help to quickly narrow down errors.
 
 For example in the following snippet::
@@ -563,7 +601,7 b' and IPython 8.0 is capable of telling you where the index error occurs::'
     ----> 3     return x[0][i][0]
                        ^^^^^^^
 
-The corresponding locations marked here with ``^`` will show up highlighted in 
+The corresponding locations marked here with ``^`` will show up highlighted in
 the terminal and notebooks.
 
 Finally, a colon ``::`` and line number is appended after a filename in
@@ -760,7 +798,7 b' Previously, this was not the case for the Vi-mode prompts::'
 This is now fixed, and Vi prompt prefixes - ``[ins]`` and ``[nav]`` -  are
 skipped just as the normal ``In`` would be.
 
-IPython shell can be started in the Vi mode using ``ipython --TerminalInteractiveShell.editing_mode=vi``, 
+IPython shell can be started in the Vi mode using ``ipython --TerminalInteractiveShell.editing_mode=vi``,
 You should be able to change mode dynamically with ``%config TerminalInteractiveShell.editing_mode='vi'``
 
 Empty History Ranges
@@ -787,8 +825,8 b' when followed with :kbd:`F2`), send it to `dpaste.org <http://dpast.org>`_ using'
 
 Windows timing implementation: Switch to process_time
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-Timing on Windows, for example with ``%%time``,  was changed from being based on ``time.perf_counter`` 
-(which counted time even when the process was sleeping) to being based on ``time.process_time`` instead 
+Timing on Windows, for example with ``%%time``,  was changed from being based on ``time.perf_counter``
+(which counted time even when the process was sleeping) to being based on ``time.process_time`` instead
 (which only counts CPU time). This brings it closer to the behavior on Linux. See :ghpull:`12984`.
 
 Miscellaneous
@@ -813,7 +851,7 b' Re-added support for XDG config directories'
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 XDG support through the years comes and goes. There is a tension between having
-an identical location for configuration in all platforms versus having simple instructions. 
+an identical location for configuration in all platforms versus having simple instructions.
 After initial failures a couple of years ago, IPython was modified to automatically migrate XDG
 config files back into ``~/.ipython``. That migration code has now been removed.
 IPython now checks the XDG locations, so if you _manually_ move your config
@@ -841,7 +879,7 b' Removing support for older Python versions'
 
 
 We are removing support for Python up through 3.7, allowing internal code to use the more
-efficient ``pathlib`` and to make better use of type annotations. 
+efficient ``pathlib`` and to make better use of type annotations.
 
 .. image:: ../_images/8.0/pathlib_pathlib_everywhere.jpg
    :alt: "Meme image of Toy Story with Woody and Buzz, with the text 'pathlib, pathlib everywhere'"

setup.cfg

@@ -100,17 +100,12 b' exclude ='
     setupext
 
 [options.package_data]
+IPython = py.typed
 IPython.core = profile/README*
 IPython.core.tests = *.png, *.jpg, daft_extension/*.py
 IPython.lib.tests = *.wav
 IPython.testing.plugin = *.txt
 
-[options.entry_points]
-pygments.lexers =
-    ipythonconsole = IPython.lib.lexers:IPythonConsoleLexer
-    ipython = IPython.lib.lexers:IPythonLexer
-    ipython3 = IPython.lib.lexers:IPython3Lexer
-
 [velin]
 ignore_patterns =
    IPython/core/tests

setup.py

@@ -139,7 +139,15 b" setup_args['cmdclass'] = {"
     'install_scripts_sym': install_scripts_for_symlink,
     'unsymlink': unsymlink,
 }
-setup_args["entry_points"] = {"console_scripts": find_entry_points()}
+
+setup_args["entry_points"] = {
+    "console_scripts": find_entry_points(),
+    "pygments.lexers": [
+        "ipythonconsole = IPython.lib.lexers:IPythonConsoleLexer",
+        "ipython = IPython.lib.lexers:IPythonLexer",
+        "ipython3 = IPython.lib.lexers:IPython3Lexer",
+    ],
+}
 
 #---------------------------------------------------------------------------
 # Do the actual setup now

setupbase.py

@@ -211,20 +211,15 b' def find_entry_points():'
     use, our own build_scripts_entrypt class below parses these and builds
     command line scripts.
 
-    Each of our entry points gets a plain name, e.g. ipython, a name
-    suffixed with the Python major version number, e.g. ipython3, and
-    a name suffixed with the Python major.minor version number, eg. ipython3.8.
+    Each of our entry points gets a plain name, e.g. ipython, and a name
+    suffixed with the Python major version number, e.g. ipython3.
     """
     ep = [
             'ipython%s = IPython:start_ipython',
         ]
     major_suffix = str(sys.version_info[0])
-    minor_suffix = ".".join([str(sys.version_info[0]), str(sys.version_info[1])])
-    return (
-        [e % "" for e in ep]
-        + [e % major_suffix for e in ep]
-        + [e % minor_suffix for e in ep]
-    )
+    return [e % "" for e in ep] + [e % major_suffix for e in ep]
+
 
 class install_lib_symlink(Command):
     user_options = [

tools/release_helper.sh

@@ -2,15 +2,6 b''
 # when releasing with bash, simple source it to get asked questions. 
 
 # misc check before starting
-
-python -c 'import keyring'
-python -c 'import twine'
-python -c 'import sphinx'
-python -c 'import sphinx_rtd_theme'
-python -c 'import pytest'
-python -c 'import build'
-
-
 BLACK=$(tput setaf 1)
 RED=$(tput setaf 1)
 GREEN=$(tput setaf 2)
@@ -22,6 +13,22 b' WHITE=$(tput setaf 7)'
 NOR=$(tput sgr0)
 
 
+echo "Checking all tools are installed..."
+
+python -c 'import keyring'
+python -c 'import twine'
+python -c 'import sphinx'
+python -c 'import sphinx_rtd_theme'
+python -c 'import pytest'
+python -c 'import build'
+# those are necessary fo building the docs
+echo "Checking imports for docs"
+python -c 'import numpy'
+python -c 'import matplotlib'
+
+
+
+
 echo "Will use $BLUE'$EDITOR'$NOR to edit files when necessary"
 echo -n "PREV_RELEASE (X.y.z) [$PREV_RELEASE]: "
 read input