Merge branch 'main' into shaperilio/autoreload-verbosity

IPython/core/guarded_eval.py

@@ -0,0 +1,738 b''
+from typing import (
+    Any,
+    Callable,
+    Dict,
+    Set,
+    Sequence,
+    Tuple,
+    NamedTuple,
+    Type,
+    Literal,
+    Union,
+    TYPE_CHECKING,
+)
+import ast
+import builtins
+import collections
+import operator
+import sys
+from functools import cached_property
+from dataclasses import dataclass, field
+
+from IPython.utils.docs import GENERATING_DOCUMENTATION
+from IPython.utils.decorators import undoc
+
+
+if TYPE_CHECKING or GENERATING_DOCUMENTATION:
+    from typing_extensions import Protocol
+else:
+    # do not require on runtime
+    Protocol = object  # requires Python >=3.8
+
+
+@undoc
+class HasGetItem(Protocol):
+    def __getitem__(self, key) -> None:
+        ...
+
+
+@undoc
+class InstancesHaveGetItem(Protocol):
+    def __call__(self, *args, **kwargs) -> HasGetItem:
+        ...
+
+
+@undoc
+class HasGetAttr(Protocol):
+    def __getattr__(self, key) -> None:
+        ...
+
+
+@undoc
+class DoesNotHaveGetAttr(Protocol):
+    pass
+
+
+# By default `__getattr__` is not explicitly implemented on most objects
+MayHaveGetattr = Union[HasGetAttr, DoesNotHaveGetAttr]
+
+
+def _unbind_method(func: Callable) -> Union[Callable, None]:
+    """Get unbound method for given bound method.
+
+    Returns None if cannot get unbound method, or method is already unbound.
+    """
+    owner = getattr(func, "__self__", None)
+    owner_class = type(owner)
+    name = getattr(func, "__name__", None)
+    instance_dict_overrides = getattr(owner, "__dict__", None)
+    if (
+        owner is not None
+        and name
+        and (
+            not instance_dict_overrides
+            or (instance_dict_overrides and name not in instance_dict_overrides)
+        )
+    ):
+        return getattr(owner_class, name)
+    return None
+
+
+@undoc
+@dataclass
+class EvaluationPolicy:
+    """Definition of evaluation policy."""
+
+    allow_locals_access: bool = False
+    allow_globals_access: bool = False
+    allow_item_access: bool = False
+    allow_attr_access: bool = False
+    allow_builtins_access: bool = False
+    allow_all_operations: bool = False
+    allow_any_calls: bool = False
+    allowed_calls: Set[Callable] = field(default_factory=set)
+
+    def can_get_item(self, value, item):
+        return self.allow_item_access
+
+    def can_get_attr(self, value, attr):
+        return self.allow_attr_access
+
+    def can_operate(self, dunders: Tuple[str, ...], a, b=None):
+        if self.allow_all_operations:
+            return True
+
+    def can_call(self, func):
+        if self.allow_any_calls:
+            return True
+
+        if func in self.allowed_calls:
+            return True
+
+        owner_method = _unbind_method(func)
+
+        if owner_method and owner_method in self.allowed_calls:
+            return True
+
+
+def _get_external(module_name: str, access_path: Sequence[str]):
+    """Get value from external module given a dotted access path.
+
+    Raises:
+    * `KeyError` if module is removed not found, and
+    * `AttributeError` if acess path does not match an exported object
+    """
+    member_type = sys.modules[module_name]
+    for attr in access_path:
+        member_type = getattr(member_type, attr)
+    return member_type
+
+
+def _has_original_dunder_external(
+    value,
+    module_name: str,
+    access_path: Sequence[str],
+    method_name: str,
+):
+    if module_name not in sys.modules:
+        # LBYLB as it is faster
+        return False
+    try:
+        member_type = _get_external(module_name, access_path)
+        value_type = type(value)
+        if type(value) == member_type:
+            return True
+        if method_name == "__getattribute__":
+            # we have to short-circuit here due to an unresolved issue in
+            # `isinstance` implementation: https://bugs.python.org/issue32683
+            return False
+        if isinstance(value, member_type):
+            method = getattr(value_type, method_name, None)
+            member_method = getattr(member_type, method_name, None)
+            if member_method == method:
+                return True
+    except (AttributeError, KeyError):
+        return False
+
+
+def _has_original_dunder(
+    value, allowed_types, allowed_methods, allowed_external, method_name
+):
+    # note: Python ignores `__getattr__`/`__getitem__` on instances,
+    # we only need to check at class level
+    value_type = type(value)
+
+    # strict type check passes → no need to check method
+    if value_type in allowed_types:
+        return True
+
+    method = getattr(value_type, method_name, None)
+
+    if method is None:
+        return None
+
+    if method in allowed_methods:
+        return True
+
+    for module_name, *access_path in allowed_external:
+        if _has_original_dunder_external(value, module_name, access_path, method_name):
+            return True
+
+    return False
+
+
+@undoc
+@dataclass
+class SelectivePolicy(EvaluationPolicy):
+    allowed_getitem: Set[InstancesHaveGetItem] = field(default_factory=set)
+    allowed_getitem_external: Set[Tuple[str, ...]] = field(default_factory=set)
+
+    allowed_getattr: Set[MayHaveGetattr] = field(default_factory=set)
+    allowed_getattr_external: Set[Tuple[str, ...]] = field(default_factory=set)
+
+    allowed_operations: Set = field(default_factory=set)
+    allowed_operations_external: Set[Tuple[str, ...]] = field(default_factory=set)
+
+    _operation_methods_cache: Dict[str, Set[Callable]] = field(
+        default_factory=dict, init=False
+    )
+
+    def can_get_attr(self, value, attr):
+        has_original_attribute = _has_original_dunder(
+            value,
+            allowed_types=self.allowed_getattr,
+            allowed_methods=self._getattribute_methods,
+            allowed_external=self.allowed_getattr_external,
+            method_name="__getattribute__",
+        )
+        has_original_attr = _has_original_dunder(
+            value,
+            allowed_types=self.allowed_getattr,
+            allowed_methods=self._getattr_methods,
+            allowed_external=self.allowed_getattr_external,
+            method_name="__getattr__",
+        )
+
+        accept = False
+
+        # Many objects do not have `__getattr__`, this is fine.
+        if has_original_attr is None and has_original_attribute:
+            accept = True
+        else:
+            # Accept objects without modifications to `__getattr__` and `__getattribute__`
+            accept = has_original_attr and has_original_attribute
+
+        if accept:
+            # We still need to check for overriden properties.
+
+            value_class = type(value)
+            if not hasattr(value_class, attr):
+                return True
+
+            class_attr_val = getattr(value_class, attr)
+            is_property = isinstance(class_attr_val, property)
+
+            if not is_property:
+                return True
+
+            # Properties in allowed types are ok (although we do not include any
+            # properties in our default allow list currently).
+            if type(value) in self.allowed_getattr:
+                return True  # pragma: no cover
+
+            # Properties in subclasses of allowed types may be ok if not changed
+            for module_name, *access_path in self.allowed_getattr_external:
+                try:
+                    external_class = _get_external(module_name, access_path)
+                    external_class_attr_val = getattr(external_class, attr)
+                except (KeyError, AttributeError):
+                    return False  # pragma: no cover
+                return class_attr_val == external_class_attr_val
+
+        return False
+
+    def can_get_item(self, value, item):
+        """Allow accessing `__getiitem__` of allow-listed instances unless it was not modified."""
+        return _has_original_dunder(
+            value,
+            allowed_types=self.allowed_getitem,
+            allowed_methods=self._getitem_methods,
+            allowed_external=self.allowed_getitem_external,
+            method_name="__getitem__",
+        )
+
+    def can_operate(self, dunders: Tuple[str, ...], a, b=None):
+        objects = [a]
+        if b is not None:
+            objects.append(b)
+        return all(
+            [
+                _has_original_dunder(
+                    obj,
+                    allowed_types=self.allowed_operations,
+                    allowed_methods=self._operator_dunder_methods(dunder),
+                    allowed_external=self.allowed_operations_external,
+                    method_name=dunder,
+                )
+                for dunder in dunders
+                for obj in objects
+            ]
+        )
+
+    def _operator_dunder_methods(self, dunder: str) -> Set[Callable]:
+        if dunder not in self._operation_methods_cache:
+            self._operation_methods_cache[dunder] = self._safe_get_methods(
+                self.allowed_operations, dunder
+            )
+        return self._operation_methods_cache[dunder]
+
+    @cached_property
+    def _getitem_methods(self) -> Set[Callable]:
+        return self._safe_get_methods(self.allowed_getitem, "__getitem__")
+
+    @cached_property
+    def _getattr_methods(self) -> Set[Callable]:
+        return self._safe_get_methods(self.allowed_getattr, "__getattr__")
+
+    @cached_property
+    def _getattribute_methods(self) -> Set[Callable]:
+        return self._safe_get_methods(self.allowed_getattr, "__getattribute__")
+
+    def _safe_get_methods(self, classes, name) -> Set[Callable]:
+        return {
+            method
+            for class_ in classes
+            for method in [getattr(class_, name, None)]
+            if method
+        }
+
+
+class _DummyNamedTuple(NamedTuple):
+    """Used internally to retrieve methods of named tuple instance."""
+
+
+class EvaluationContext(NamedTuple):
+    #: Local namespace
+    locals: dict
+    #: Global namespace
+    globals: dict
+    #: Evaluation policy identifier
+    evaluation: Literal[
+        "forbidden", "minimal", "limited", "unsafe", "dangerous"
+    ] = "forbidden"
+    #: Whether the evalution of code takes place inside of a subscript.
+    #: Useful for evaluating ``:-1, 'col'`` in ``df[:-1, 'col']``.
+    in_subscript: bool = False
+
+
+class _IdentitySubscript:
+    """Returns the key itself when item is requested via subscript."""
+
+    def __getitem__(self, key):
+        return key
+
+
+IDENTITY_SUBSCRIPT = _IdentitySubscript()
+SUBSCRIPT_MARKER = "__SUBSCRIPT_SENTINEL__"
+
+
+class GuardRejection(Exception):
+    """Exception raised when guard rejects evaluation attempt."""
+
+    pass
+
+
+def guarded_eval(code: str, context: EvaluationContext):
+    """Evaluate provided code in the evaluation context.
+
+    If evaluation policy given by context is set to ``forbidden``
+    no evaluation will be performed; if it is set to ``dangerous``
+    standard :func:`eval` will be used; finally, for any other,
+    policy :func:`eval_node` will be called on parsed AST.
+    """
+    locals_ = context.locals
+
+    if context.evaluation == "forbidden":
+        raise GuardRejection("Forbidden mode")
+
+    # note: not using `ast.literal_eval` as it does not implement
+    # getitem at all, for example it fails on simple `[0][1]`
+
+    if context.in_subscript:
+        # syntatic sugar for ellipsis (:) is only available in susbcripts
+        # so we need to trick the ast parser into thinking that we have
+        # a subscript, but we need to be able to later recognise that we did
+        # it so we can ignore the actual __getitem__ operation
+        if not code:
+            return tuple()
+        locals_ = locals_.copy()
+        locals_[SUBSCRIPT_MARKER] = IDENTITY_SUBSCRIPT
+        code = SUBSCRIPT_MARKER + "[" + code + "]"
+        context = EvaluationContext(**{**context._asdict(), **{"locals": locals_}})
+
+    if context.evaluation == "dangerous":
+        return eval(code, context.globals, context.locals)
+
+    expression = ast.parse(code, mode="eval")
+
+    return eval_node(expression, context)
+
+
+BINARY_OP_DUNDERS: Dict[Type[ast.operator], Tuple[str]] = {
+    ast.Add: ("__add__",),
+    ast.Sub: ("__sub__",),
+    ast.Mult: ("__mul__",),
+    ast.Div: ("__truediv__",),
+    ast.FloorDiv: ("__floordiv__",),
+    ast.Mod: ("__mod__",),
+    ast.Pow: ("__pow__",),
+    ast.LShift: ("__lshift__",),
+    ast.RShift: ("__rshift__",),
+    ast.BitOr: ("__or__",),
+    ast.BitXor: ("__xor__",),
+    ast.BitAnd: ("__and__",),
+    ast.MatMult: ("__matmul__",),
+}
+
+COMP_OP_DUNDERS: Dict[Type[ast.cmpop], Tuple[str, ...]] = {
+    ast.Eq: ("__eq__",),
+    ast.NotEq: ("__ne__", "__eq__"),
+    ast.Lt: ("__lt__", "__gt__"),
+    ast.LtE: ("__le__", "__ge__"),
+    ast.Gt: ("__gt__", "__lt__"),
+    ast.GtE: ("__ge__", "__le__"),
+    ast.In: ("__contains__",),
+    # Note: ast.Is, ast.IsNot, ast.NotIn are handled specially
+}
+
+UNARY_OP_DUNDERS: Dict[Type[ast.unaryop], Tuple[str, ...]] = {
+    ast.USub: ("__neg__",),
+    ast.UAdd: ("__pos__",),
+    # we have to check both __inv__ and __invert__!
+    ast.Invert: ("__invert__", "__inv__"),
+    ast.Not: ("__not__",),
+}
+
+
+def _find_dunder(node_op, dunders) -> Union[Tuple[str, ...], None]:
+    dunder = None
+    for op, candidate_dunder in dunders.items():
+        if isinstance(node_op, op):
+            dunder = candidate_dunder
+    return dunder
+
+
+def eval_node(node: Union[ast.AST, None], context: EvaluationContext):
+    """Evaluate AST node in provided context.
+
+    Applies evaluation restrictions defined in the context. Currently does not support evaluation of functions with keyword arguments.
+
+    Does not evaluate actions that always have side effects:
+
+    - class definitions (``class sth: ...``)
+    - function definitions (``def sth: ...``)
+    - variable assignments (``x = 1``)
+    - augmented assignments (``x += 1``)
+    - deletions (``del x``)
+
+    Does not evaluate operations which do not return values:
+
+    - assertions (``assert x``)
+    - pass (``pass``)
+    - imports (``import x``)
+    - control flow:
+
+        - conditionals (``if x:``) except for ternary IfExp (``a if x else b``)
+        - loops (``for`` and `while``)
+        - exception handling
+
+    The purpose of this function is to guard against unwanted side-effects;
+    it does not give guarantees on protection from malicious code execution.
+    """
+    policy = EVALUATION_POLICIES[context.evaluation]
+    if node is None:
+        return None
+    if isinstance(node, ast.Expression):
+        return eval_node(node.body, context)
+    if isinstance(node, ast.BinOp):
+        left = eval_node(node.left, context)
+        right = eval_node(node.right, context)
+        dunders = _find_dunder(node.op, BINARY_OP_DUNDERS)
+        if dunders:
+            if policy.can_operate(dunders, left, right):
+                return getattr(left, dunders[0])(right)
+            else:
+                raise GuardRejection(
+                    f"Operation (`{dunders}`) for",
+                    type(left),
+                    f"not allowed in {context.evaluation} mode",
+                )
+    if isinstance(node, ast.Compare):
+        left = eval_node(node.left, context)
+        all_true = True
+        negate = False
+        for op, right in zip(node.ops, node.comparators):
+            right = eval_node(right, context)
+            dunder = None
+            dunders = _find_dunder(op, COMP_OP_DUNDERS)
+            if not dunders:
+                if isinstance(op, ast.NotIn):
+                    dunders = COMP_OP_DUNDERS[ast.In]
+                    negate = True
+                if isinstance(op, ast.Is):
+                    dunder = "is_"
+                if isinstance(op, ast.IsNot):
+                    dunder = "is_"
+                    negate = True
+            if not dunder and dunders:
+                dunder = dunders[0]
+            if dunder:
+                a, b = (right, left) if dunder == "__contains__" else (left, right)
+                if dunder == "is_" or dunders and policy.can_operate(dunders, a, b):
+                    result = getattr(operator, dunder)(a, b)
+                    if negate:
+                        result = not result
+                    if not result:
+                        all_true = False
+                    left = right
+                else:
+                    raise GuardRejection(
+                        f"Comparison (`{dunder}`) for",
+                        type(left),
+                        f"not allowed in {context.evaluation} mode",
+                    )
+            else:
+                raise ValueError(
+                    f"Comparison `{dunder}` not supported"
+                )  # pragma: no cover
+        return all_true
+    if isinstance(node, ast.Constant):
+        return node.value
+    if isinstance(node, ast.Index):
+        # deprecated since Python 3.9
+        return eval_node(node.value, context)  # pragma: no cover
+    if isinstance(node, ast.Tuple):
+        return tuple(eval_node(e, context) for e in node.elts)
+    if isinstance(node, ast.List):
+        return [eval_node(e, context) for e in node.elts]
+    if isinstance(node, ast.Set):
+        return {eval_node(e, context) for e in node.elts}
+    if isinstance(node, ast.Dict):
+        return dict(
+            zip(
+                [eval_node(k, context) for k in node.keys],
+                [eval_node(v, context) for v in node.values],
+            )
+        )
+    if isinstance(node, ast.Slice):
+        return slice(
+            eval_node(node.lower, context),
+            eval_node(node.upper, context),
+            eval_node(node.step, context),
+        )
+    if isinstance(node, ast.ExtSlice):
+        # deprecated since Python 3.9
+        return tuple([eval_node(dim, context) for dim in node.dims])  # pragma: no cover
+    if isinstance(node, ast.UnaryOp):
+        value = eval_node(node.operand, context)
+        dunders = _find_dunder(node.op, UNARY_OP_DUNDERS)
+        if dunders:
+            if policy.can_operate(dunders, value):
+                return getattr(value, dunders[0])()
+            else:
+                raise GuardRejection(
+                    f"Operation (`{dunders}`) for",
+                    type(value),
+                    f"not allowed in {context.evaluation} mode",
+                )
+    if isinstance(node, ast.Subscript):
+        value = eval_node(node.value, context)
+        slice_ = eval_node(node.slice, context)
+        if policy.can_get_item(value, slice_):
+            return value[slice_]
+        raise GuardRejection(
+            "Subscript access (`__getitem__`) for",
+            type(value),  # not joined to avoid calling `repr`
+            f" not allowed in {context.evaluation} mode",
+        )
+    if isinstance(node, ast.Name):
+        if policy.allow_locals_access and node.id in context.locals:
+            return context.locals[node.id]
+        if policy.allow_globals_access and node.id in context.globals:
+            return context.globals[node.id]
+        if policy.allow_builtins_access and hasattr(builtins, node.id):
+            # note: do not use __builtins__, it is implementation detail of cPython
+            return getattr(builtins, node.id)
+        if not policy.allow_globals_access and not policy.allow_locals_access:
+            raise GuardRejection(
+                f"Namespace access not allowed in {context.evaluation} mode"
+            )
+        else:
+            raise NameError(f"{node.id} not found in locals, globals, nor builtins")
+    if isinstance(node, ast.Attribute):
+        value = eval_node(node.value, context)
+        if policy.can_get_attr(value, node.attr):
+            return getattr(value, node.attr)
+        raise GuardRejection(
+            "Attribute access (`__getattr__`) for",
+            type(value),  # not joined to avoid calling `repr`
+            f"not allowed in {context.evaluation} mode",
+        )
+    if isinstance(node, ast.IfExp):
+        test = eval_node(node.test, context)
+        if test:
+            return eval_node(node.body, context)
+        else:
+            return eval_node(node.orelse, context)
+    if isinstance(node, ast.Call):
+        func = eval_node(node.func, context)
+        if policy.can_call(func) and not node.keywords:
+            args = [eval_node(arg, context) for arg in node.args]
+            return func(*args)
+        raise GuardRejection(
+            "Call for",
+            func,  # not joined to avoid calling `repr`
+            f"not allowed in {context.evaluation} mode",
+        )
+    raise ValueError("Unhandled node", ast.dump(node))
+
+
+SUPPORTED_EXTERNAL_GETITEM = {
+    ("pandas", "core", "indexing", "_iLocIndexer"),
+    ("pandas", "core", "indexing", "_LocIndexer"),
+    ("pandas", "DataFrame"),
+    ("pandas", "Series"),
+    ("numpy", "ndarray"),
+    ("numpy", "void"),
+}
+
+
+BUILTIN_GETITEM: Set[InstancesHaveGetItem] = {
+    dict,
+    str,
+    bytes,
+    list,
+    tuple,
+    collections.defaultdict,
+    collections.deque,
+    collections.OrderedDict,
+    collections.ChainMap,
+    collections.UserDict,
+    collections.UserList,
+    collections.UserString,
+    _DummyNamedTuple,
+    _IdentitySubscript,
+}
+
+
+def _list_methods(cls, source=None):
+    """For use on immutable objects or with methods returning a copy"""
+    return [getattr(cls, k) for k in (source if source else dir(cls))]
+
+
+dict_non_mutating_methods = ("copy", "keys", "values", "items")
+list_non_mutating_methods = ("copy", "index", "count")
+set_non_mutating_methods = set(dir(set)) & set(dir(frozenset))
+
+
+dict_keys: Type[collections.abc.KeysView] = type({}.keys())
+method_descriptor: Any = type(list.copy)
+
+NUMERICS = {int, float, complex}
+
+ALLOWED_CALLS = {
+    bytes,
+    *_list_methods(bytes),
+    dict,
+    *_list_methods(dict, dict_non_mutating_methods),
+    dict_keys.isdisjoint,
+    list,
+    *_list_methods(list, list_non_mutating_methods),
+    set,
+    *_list_methods(set, set_non_mutating_methods),
+    frozenset,
+    *_list_methods(frozenset),
+    range,
+    str,
+    *_list_methods(str),
+    tuple,
+    *_list_methods(tuple),
+    *NUMERICS,
+    *[method for numeric_cls in NUMERICS for method in _list_methods(numeric_cls)],
+    collections.deque,
+    *_list_methods(collections.deque, list_non_mutating_methods),
+    collections.defaultdict,
+    *_list_methods(collections.defaultdict, dict_non_mutating_methods),
+    collections.OrderedDict,
+    *_list_methods(collections.OrderedDict, dict_non_mutating_methods),
+    collections.UserDict,
+    *_list_methods(collections.UserDict, dict_non_mutating_methods),
+    collections.UserList,
+    *_list_methods(collections.UserList, list_non_mutating_methods),
+    collections.UserString,
+    *_list_methods(collections.UserString, dir(str)),
+    collections.Counter,
+    *_list_methods(collections.Counter, dict_non_mutating_methods),
+    collections.Counter.elements,
+    collections.Counter.most_common,
+}
+
+BUILTIN_GETATTR: Set[MayHaveGetattr] = {
+    *BUILTIN_GETITEM,
+    set,
+    frozenset,
+    object,
+    type,  # `type` handles a lot of generic cases, e.g. numbers as in `int.real`.
+    *NUMERICS,
+    dict_keys,
+    method_descriptor,
+}
+
+
+BUILTIN_OPERATIONS = {*BUILTIN_GETATTR}
+
+EVALUATION_POLICIES = {
+    "minimal": EvaluationPolicy(
+        allow_builtins_access=True,
+        allow_locals_access=False,
+        allow_globals_access=False,
+        allow_item_access=False,
+        allow_attr_access=False,
+        allowed_calls=set(),
+        allow_any_calls=False,
+        allow_all_operations=False,
+    ),
+    "limited": SelectivePolicy(
+        allowed_getitem=BUILTIN_GETITEM,
+        allowed_getitem_external=SUPPORTED_EXTERNAL_GETITEM,
+        allowed_getattr=BUILTIN_GETATTR,
+        allowed_getattr_external={
+            # pandas Series/Frame implements custom `__getattr__`
+            ("pandas", "DataFrame"),
+            ("pandas", "Series"),
+        },
+        allowed_operations=BUILTIN_OPERATIONS,
+        allow_builtins_access=True,
+        allow_locals_access=True,
+        allow_globals_access=True,
+        allowed_calls=ALLOWED_CALLS,
+    ),
+    "unsafe": EvaluationPolicy(
+        allow_builtins_access=True,
+        allow_locals_access=True,
+        allow_globals_access=True,
+        allow_attr_access=True,
+        allow_item_access=True,
+        allow_any_calls=True,
+        allow_all_operations=True,
+    ),
+}
+
+
+__all__ = [
+    "guarded_eval",
+    "eval_node",
+    "GuardRejection",
+    "EvaluationContext",
+    "_unbind_method",
+]

IPython/core/tests/test_guarded_eval.py

@@ -0,0 +1,570 b''
+from contextlib import contextmanager
+from typing import NamedTuple
+from functools import partial
+from IPython.core.guarded_eval import (
+    EvaluationContext,
+    GuardRejection,
+    guarded_eval,
+    _unbind_method,
+)
+from IPython.testing import decorators as dec
+import pytest
+
+
+def create_context(evaluation: str, **kwargs):
+    return EvaluationContext(locals=kwargs, globals={}, evaluation=evaluation)
+
+
+forbidden = partial(create_context, "forbidden")
+minimal = partial(create_context, "minimal")
+limited = partial(create_context, "limited")
+unsafe = partial(create_context, "unsafe")
+dangerous = partial(create_context, "dangerous")
+
+LIMITED_OR_HIGHER = [limited, unsafe, dangerous]
+MINIMAL_OR_HIGHER = [minimal, *LIMITED_OR_HIGHER]
+
+
+@contextmanager
+def module_not_installed(module: str):
+    import sys
+
+    try:
+        to_restore = sys.modules[module]
+        del sys.modules[module]
+    except KeyError:
+        to_restore = None
+    try:
+        yield
+    finally:
+        sys.modules[module] = to_restore
+
+
+def test_external_not_installed():
+    """
+    Because attribute check requires checking if object is not of allowed
+    external type, this tests logic for absence of external module.
+    """
+
+    class Custom:
+        def __init__(self):
+            self.test = 1
+
+        def __getattr__(self, key):
+            return key
+
+    with module_not_installed("pandas"):
+        context = limited(x=Custom())
+        with pytest.raises(GuardRejection):
+            guarded_eval("x.test", context)
+
+
[e-mail]_without("pandas")
+def test_external_changed_api(monkeypatch):
+    """Check that the execution rejects if external API changed paths"""
+    import pandas as pd
+
+    series = pd.Series([1], index=["a"])
+
+    with monkeypatch.context() as m:
+        m.delattr(pd, "Series")
+        context = limited(data=series)
+        with pytest.raises(GuardRejection):
+            guarded_eval("data.iloc[0]", context)
+
+
[e-mail]_without("pandas")
+def test_pandas_series_iloc():
+    import pandas as pd
+
+    series = pd.Series([1], index=["a"])
+    context = limited(data=series)
+    assert guarded_eval("data.iloc[0]", context) == 1
+
+
+def test_rejects_custom_properties():
+    class BadProperty:
+        @property
+        def iloc(self):
+            return [None]
+
+    series = BadProperty()
+    context = limited(data=series)
+
+    with pytest.raises(GuardRejection):
+        guarded_eval("data.iloc[0]", context)
+
+
[e-mail]_without("pandas")
+def test_accepts_non_overriden_properties():
+    import pandas as pd
+
+    class GoodProperty(pd.Series):
+        pass
+
+    series = GoodProperty([1], index=["a"])
+    context = limited(data=series)
+
+    assert guarded_eval("data.iloc[0]", context) == 1
+
+
[e-mail]_without("pandas")
+def test_pandas_series():
+    import pandas as pd
+
+    context = limited(data=pd.Series([1], index=["a"]))
+    assert guarded_eval('data["a"]', context) == 1
+    with pytest.raises(KeyError):
+        guarded_eval('data["c"]', context)
+
+
[e-mail]_without("pandas")
+def test_pandas_bad_series():
+    import pandas as pd
+
+    class BadItemSeries(pd.Series):
+        def __getitem__(self, key):
+            return "CUSTOM_ITEM"
+
+    class BadAttrSeries(pd.Series):
+        def __getattr__(self, key):
+            return "CUSTOM_ATTR"
+
+    bad_series = BadItemSeries([1], index=["a"])
+    context = limited(data=bad_series)
+
+    with pytest.raises(GuardRejection):
+        guarded_eval('data["a"]', context)
+    with pytest.raises(GuardRejection):
+        guarded_eval('data["c"]', context)
+
+    # note: here result is a bit unexpected because
+    # pandas `__getattr__` calls `__getitem__`;
+    # FIXME - special case to handle it?
+    assert guarded_eval("data.a", context) == "CUSTOM_ITEM"
+
+    context = unsafe(data=bad_series)
+    assert guarded_eval('data["a"]', context) == "CUSTOM_ITEM"
+
+    bad_attr_series = BadAttrSeries([1], index=["a"])
+    context = limited(data=bad_attr_series)
+    assert guarded_eval('data["a"]', context) == 1
+    with pytest.raises(GuardRejection):
+        guarded_eval("data.a", context)
+
+
[e-mail]_without("pandas")
+def test_pandas_dataframe_loc():
+    import pandas as pd
+    from pandas.testing import assert_series_equal
+
+    data = pd.DataFrame([{"a": 1}])
+    context = limited(data=data)
+    assert_series_equal(guarded_eval('data.loc[:, "a"]', context), data["a"])
+
+
+def test_named_tuple():
+    class GoodNamedTuple(NamedTuple):
+        a: str
+        pass
+
+    class BadNamedTuple(NamedTuple):
+        a: str
+
+        def __getitem__(self, key):
+            return None
+
+    good = GoodNamedTuple(a="x")
+    bad = BadNamedTuple(a="x")
+
+    context = limited(data=good)
+    assert guarded_eval("data[0]", context) == "x"
+
+    context = limited(data=bad)
+    with pytest.raises(GuardRejection):
+        guarded_eval("data[0]", context)
+
+
+def test_dict():
+    context = limited(data={"a": 1, "b": {"x": 2}, ("x", "y"): 3})
+    assert guarded_eval('data["a"]', context) == 1
+    assert guarded_eval('data["b"]', context) == {"x": 2}
+    assert guarded_eval('data["b"]["x"]', context) == 2
+    assert guarded_eval('data["x", "y"]', context) == 3
+
+    assert guarded_eval("data.keys", context)
+
+
+def test_set():
+    context = limited(data={"a", "b"})
+    assert guarded_eval("data.difference", context)
+
+
+def test_list():
+    context = limited(data=[1, 2, 3])
+    assert guarded_eval("data[1]", context) == 2
+    assert guarded_eval("data.copy", context)
+
+
+def test_dict_literal():
+    context = limited()
+    assert guarded_eval("{}", context) == {}
+    assert guarded_eval('{"a": 1}', context) == {"a": 1}
+
+
+def test_list_literal():
+    context = limited()
+    assert guarded_eval("[]", context) == []
+    assert guarded_eval('[1, "a"]', context) == [1, "a"]
+
+
+def test_set_literal():
+    context = limited()
+    assert guarded_eval("set()", context) == set()
+    assert guarded_eval('{"a"}', context) == {"a"}
+
+
+def test_evaluates_if_expression():
+    context = limited()
+    assert guarded_eval("2 if True else 3", context) == 2
+    assert guarded_eval("4 if False else 5", context) == 5
+
+
+def test_object():
+    obj = object()
+    context = limited(obj=obj)
+    assert guarded_eval("obj.__dir__", context) == obj.__dir__
+
+
[e-mail](
+    "code,expected",
+    [
+        ["int.numerator", int.numerator],
+        ["float.is_integer", float.is_integer],
+        ["complex.real", complex.real],
+    ],
+)
+def test_number_attributes(code, expected):
+    assert guarded_eval(code, limited()) == expected
+
+
+def test_method_descriptor():
+    context = limited()
+    assert guarded_eval("list.copy.__name__", context) == "copy"
+
+
[e-mail](
+    "data,good,bad,expected",
+    [
+        [[1, 2, 3], "data.index(2)", "data.append(4)", 1],
+        [{"a": 1}, "data.keys().isdisjoint({})", "data.update()", True],
+    ],
+)
+def test_evaluates_calls(data, good, bad, expected):
+    context = limited(data=data)
+    assert guarded_eval(good, context) == expected
+
+    with pytest.raises(GuardRejection):
+        guarded_eval(bad, context)
+
+
[e-mail](
+    "code,expected",
+    [
+        ["(1\n+\n1)", 2],
+        ["list(range(10))[-1:]", [9]],
+        ["list(range(20))[3:-2:3]", [3, 6, 9, 12, 15]],
+    ],
+)
[e-mail]("context", LIMITED_OR_HIGHER)
+def test_evaluates_complex_cases(code, expected, context):
+    assert guarded_eval(code, context()) == expected
+
+
[e-mail](
+    "code,expected",
+    [
+        ["1", 1],
+        ["1.0", 1.0],
+        ["0xdeedbeef", 0xDEEDBEEF],
+        ["True", True],
+        ["None", None],
+        ["{}", {}],
+        ["[]", []],
+    ],
+)
[e-mail]("context", MINIMAL_OR_HIGHER)
+def test_evaluates_literals(code, expected, context):
+    assert guarded_eval(code, context()) == expected
+
+
[e-mail](
+    "code,expected",
+    [
+        ["-5", -5],
+        ["+5", +5],
+        ["~5", -6],
+    ],
+)
[e-mail]("context", LIMITED_OR_HIGHER)
+def test_evaluates_unary_operations(code, expected, context):
+    assert guarded_eval(code, context()) == expected
+
+
[e-mail](
+    "code,expected",
+    [
+        ["1 + 1", 2],
+        ["3 - 1", 2],
+        ["2 * 3", 6],
+        ["5 // 2", 2],
+        ["5 / 2", 2.5],
+        ["5**2", 25],
+        ["2 >> 1", 1],
+        ["2 << 1", 4],
+        ["1 | 2", 3],
+        ["1 & 1", 1],
+        ["1 & 2", 0],
+    ],
+)
[e-mail]("context", LIMITED_OR_HIGHER)
+def test_evaluates_binary_operations(code, expected, context):
+    assert guarded_eval(code, context()) == expected
+
+
[e-mail](
+    "code,expected",
+    [
+        ["2 > 1", True],
+        ["2 < 1", False],
+        ["2 <= 1", False],
+        ["2 <= 2", True],
+        ["1 >= 2", False],
+        ["2 >= 2", True],
+        ["2 == 2", True],
+        ["1 == 2", False],
+        ["1 != 2", True],
+        ["1 != 1", False],
+        ["1 < 4 < 3", False],
+        ["(1 < 4) < 3", True],
+        ["4 > 3 > 2 > 1", True],
+        ["4 > 3 > 2 > 9", False],
+        ["1 < 2 < 3 < 4", True],
+        ["9 < 2 < 3 < 4", False],
+        ["1 < 2 > 1 > 0 > -1 < 1", True],
+        ["1 in [1] in [[1]]", True],
+        ["1 in [1] in [[2]]", False],
+        ["1 in [1]", True],
+        ["0 in [1]", False],
+        ["1 not in [1]", False],
+        ["0 not in [1]", True],
+        ["True is True", True],
+        ["False is False", True],
+        ["True is False", False],
+        ["True is not True", False],
+        ["False is not True", True],
+    ],
+)
[e-mail]("context", LIMITED_OR_HIGHER)
+def test_evaluates_comparisons(code, expected, context):
+    assert guarded_eval(code, context()) == expected
+
+
+def test_guards_comparisons():
+    class GoodEq(int):
+        pass
+
+    class BadEq(int):
+        def __eq__(self, other):
+            assert False
+
+    context = limited(bad=BadEq(1), good=GoodEq(1))
+
+    with pytest.raises(GuardRejection):
+        guarded_eval("bad == 1", context)
+
+    with pytest.raises(GuardRejection):
+        guarded_eval("bad != 1", context)
+
+    with pytest.raises(GuardRejection):
+        guarded_eval("1 == bad", context)
+
+    with pytest.raises(GuardRejection):
+        guarded_eval("1 != bad", context)
+
+    assert guarded_eval("good == 1", context) is True
+    assert guarded_eval("good != 1", context) is False
+    assert guarded_eval("1 == good", context) is True
+    assert guarded_eval("1 != good", context) is False
+
+
+def test_guards_unary_operations():
+    class GoodOp(int):
+        pass
+
+    class BadOpInv(int):
+        def __inv__(self, other):
+            assert False
+
+    class BadOpInverse(int):
+        def __inv__(self, other):
+            assert False
+
+    context = limited(good=GoodOp(1), bad1=BadOpInv(1), bad2=BadOpInverse(1))
+
+    with pytest.raises(GuardRejection):
+        guarded_eval("~bad1", context)
+
+    with pytest.raises(GuardRejection):
+        guarded_eval("~bad2", context)
+
+
+def test_guards_binary_operations():
+    class GoodOp(int):
+        pass
+
+    class BadOp(int):
+        def __add__(self, other):
+            assert False
+
+    context = limited(good=GoodOp(1), bad=BadOp(1))
+
+    with pytest.raises(GuardRejection):
+        guarded_eval("1 + bad", context)
+
+    with pytest.raises(GuardRejection):
+        guarded_eval("bad + 1", context)
+
+    assert guarded_eval("good + 1", context) == 2
+    assert guarded_eval("1 + good", context) == 2
+
+
+def test_guards_attributes():
+    class GoodAttr(float):
+        pass
+
+    class BadAttr1(float):
+        def __getattr__(self, key):
+            assert False
+
+    class BadAttr2(float):
+        def __getattribute__(self, key):
+            assert False
+
+    context = limited(good=GoodAttr(0.5), bad1=BadAttr1(0.5), bad2=BadAttr2(0.5))
+
+    with pytest.raises(GuardRejection):
+        guarded_eval("bad1.as_integer_ratio", context)
+
+    with pytest.raises(GuardRejection):
+        guarded_eval("bad2.as_integer_ratio", context)
+
+    assert guarded_eval("good.as_integer_ratio()", context) == (1, 2)
+
+
[e-mail]("context", MINIMAL_OR_HIGHER)
+def test_access_builtins(context):
+    assert guarded_eval("round", context()) == round
+
+
+def test_access_builtins_fails():
+    context = limited()
+    with pytest.raises(NameError):
+        guarded_eval("this_is_not_builtin", context)
+
+
+def test_rejects_forbidden():
+    context = forbidden()
+    with pytest.raises(GuardRejection):
+        guarded_eval("1", context)
+
+
+def test_guards_locals_and_globals():
+    context = EvaluationContext(
+        locals={"local_a": "a"}, globals={"global_b": "b"}, evaluation="minimal"
+    )
+
+    with pytest.raises(GuardRejection):
+        guarded_eval("local_a", context)
+
+    with pytest.raises(GuardRejection):
+        guarded_eval("global_b", context)
+
+
+def test_access_locals_and_globals():
+    context = EvaluationContext(
+        locals={"local_a": "a"}, globals={"global_b": "b"}, evaluation="limited"
+    )
+    assert guarded_eval("local_a", context) == "a"
+    assert guarded_eval("global_b", context) == "b"
+
+
[e-mail](
+    "code",
+    ["def func(): pass", "class C: pass", "x = 1", "x += 1", "del x", "import ast"],
+)
[e-mail]("context", [minimal(), limited(), unsafe()])
+def test_rejects_side_effect_syntax(code, context):
+    with pytest.raises(SyntaxError):
+        guarded_eval(code, context)
+
+
+def test_subscript():
+    context = EvaluationContext(
+        locals={}, globals={}, evaluation="limited", in_subscript=True
+    )
+    empty_slice = slice(None, None, None)
+    assert guarded_eval("", context) == tuple()
+    assert guarded_eval(":", context) == empty_slice
+    assert guarded_eval("1:2:3", context) == slice(1, 2, 3)
+    assert guarded_eval(':, "a"', context) == (empty_slice, "a")
+
+
+def test_unbind_method():
+    class X(list):
+        def index(self, k):
+            return "CUSTOM"
+
+    x = X()
+    assert _unbind_method(x.index) is X.index
+    assert _unbind_method([].index) is list.index
+    assert _unbind_method(list.index) is None
+
+
+def test_assumption_instance_attr_do_not_matter():
+    """This is semi-specified in Python documentation.
+
+    However, since the specification says 'not guaranted
+    to work' rather than 'is forbidden to work', future
+    versions could invalidate this assumptions. This test
+    is meant to catch such a change if it ever comes true.
+    """
+
+    class T:
+        def __getitem__(self, k):
+            return "a"
+
+        def __getattr__(self, k):
+            return "a"
+
+    def f(self):
+        return "b"
+
+    t = T()
+    t.__getitem__ = f
+    t.__getattr__ = f
+    assert t[1] == "a"
+    assert t[1] == "a"
+
+
+def test_assumption_named_tuples_share_getitem():
+    """Check assumption on named tuples sharing __getitem__"""
+    from typing import NamedTuple
+
+    class A(NamedTuple):
+        pass
+
+    class B(NamedTuple):
+        pass
+
+    assert A.__getitem__ == B.__getitem__

IPython/lib/tests/test_pygments.py

@@ -0,0 +1,26 b''
+from typing import List
+
+import pytest
+import pygments.lexers
+import pygments.lexer
+
+from IPython.lib.lexers import IPythonConsoleLexer, IPythonLexer, IPython3Lexer
+
+#: the human-readable names of the IPython lexers with ``entry_points``
+EXPECTED_LEXER_NAMES = [
+    cls.name for cls in [IPythonConsoleLexer, IPythonLexer, IPython3Lexer]
+]
+
+
[e-mail]
+def all_pygments_lexer_names() -> List[str]:
+    """Get all lexer names registered in pygments."""
+    return {l[0] for l in pygments.lexers.get_all_lexers()}
+
+
[e-mail]("expected_lexer", EXPECTED_LEXER_NAMES)
+def test_pygments_entry_points(
+    expected_lexer: str, all_pygments_lexer_names: List[str]
+) -> None:
+    """Check whether the ``entry_points`` for ``pygments.lexers`` are correct."""
+    assert expected_lexer in all_pygments_lexer_names

.github/workflows/mypy.yml

@@ -31,6 +31,8 b' jobs:'
       run: |
         mypy -p IPython.terminal
         mypy -p IPython.core.magics
+        mypy -p IPython.core.guarded_eval
+        mypy -p IPython.core.completer
     - name: Lint with pyflakes
       run: |
         flake8  IPython/core/magics/script.py

IPython/__init__.py

@@ -1,3 +1,4 b''
+# PYTHON_ARGCOMPLETE_OK
 """
 IPython: tools for interactive and parallel computing in Python.
 

IPython/__main__.py

@@ -1,3 +1,4 b''
+# PYTHON_ARGCOMPLETE_OK
 # encoding: utf-8
 """Terminal-based IPython entry point.
 """

IPython/core/completer.py

@@ -50,7 +50,7 b' Backward latex completion'
 
 It is sometime challenging to know how to type a character, if you are using
 IPython, or any compatible frontend you can prepend backslash to the character
-and press ``<tab>`` to expand it to its latex form.
+and press :kbd:`Tab` to expand it to its latex form.
 
 .. code::
 
@@ -59,7 +59,7 b' and press ``<tab>`` to expand it to its latex form.'
 
 
 Both forward and backward completions can be deactivated by setting the
-``Completer.backslash_combining_completions`` option to ``False``.
+:any:`Completer.backslash_combining_completions` option to ``False``.
 
 
 Experimental
@@ -95,7 +95,7 b' having to execute any code:'
    ... myvar[1].bi<tab>
 
 Tab completion will be able to infer that ``myvar[1]`` is a real number without
-executing any code unlike the previously available ``IPCompleter.greedy``
+executing almost any code unlike the deprecated :any:`IPCompleter.greedy`
 option.
 
 Be sure to update :any:`jedi` to the latest stable version or to try the
@@ -178,6 +178,7 b' The suppression behaviour can is user-configurable via'
 
 from __future__ import annotations
 import builtins as builtin_mod
+import enum
 import glob
 import inspect
 import itertools
@@ -186,14 +187,16 b' import os'
 import re
 import string
 import sys
+import tokenize
 import time
 import unicodedata
 import uuid
 import warnings
+from ast import literal_eval
+from collections import defaultdict
 from contextlib import contextmanager
 from dataclasses import dataclass
 from functools import cached_property, partial
-from importlib import import_module
 from types import SimpleNamespace
 from typing import (
     Iterable,
@@ -204,14 +207,15 b' from typing import ('
     Any,
     Sequence,
     Dict,
-    NamedTuple,
-    Pattern,
     Optional,
     TYPE_CHECKING,
     Set,
+    Sized,
+    TypeVar,
     Literal,
 )
 
+from IPython.core.guarded_eval import guarded_eval, EvaluationContext
 from IPython.core.error import TryNext
 from IPython.core.inputtransformer2 import ESC_MAGIC
 from IPython.core.latex_symbols import latex_symbols, reverse_latex_symbol
@@ -231,7 +235,6 b' from traitlets import ('
     Unicode,
     Dict as DictTrait,
     Union as UnionTrait,
-    default,
     observe,
 )
 from traitlets.config.configurable import Configurable
@@ -254,10 +257,11 b' except ImportError:'
 
 if TYPE_CHECKING or GENERATING_DOCUMENTATION:
     from typing import cast
-    from typing_extensions import TypedDict, NotRequired, Protocol, TypeAlias
+    from typing_extensions import TypedDict, NotRequired, Protocol, TypeAlias, TypeGuard
 else:
+    from typing import Generic
 
-    def cast(obj, type_):
+    def cast(type_, obj):
         """Workaround for `TypeError: MatcherAPIv2() takes no arguments`"""
         return obj
 
@@ -266,6 +270,7 b' else:'
     TypedDict = Dict  # by extension of `NotRequired` requires 3.11 too
     Protocol = object  # requires Python >=3.8
     TypeAlias = Any  # requires Python >=3.10
+    TypeGuard = Generic  # requires Python >=3.10
 if GENERATING_DOCUMENTATION:
     from typing import TypedDict
 
@@ -296,6 +301,9 b' MATCHES_LIMIT = 500'
 # Completion type reported when no type can be inferred.
 _UNKNOWN_TYPE = "<unknown>"
 
+# sentinel value to signal lack of a match
+not_found = object()
+
 class ProvisionalCompleterWarning(FutureWarning):
     """
     Exception raise by an experimental feature in this module.
@@ -466,8 +474,9 b' class _FakeJediCompletion:'
         self.complete = name
         self.type = 'crashed'
         self.name_with_symbols = name
-        self.signature = ''
-        self._origin = 'fake'
+        self.signature = ""
+        self._origin = "fake"
+        self.text = "crashed"
 
     def __repr__(self):
         return '<Fake completion object jedi has crashed>'
@@ -503,11 +512,23 b' class Completion:'
 
     __slots__ = ['start', 'end', 'text', 'type', 'signature', '_origin']
 
-    def __init__(self, start: int, end: int, text: str, *, type: str=None, _origin='', signature='') -> None:
-        warnings.warn("``Completion`` is a provisional API (as of IPython 6.0). "
+    def __init__(
+        self,
+        start: int,
+        end: int,
+        text: str,
+        *,
+        type: Optional[str] = None,
+        _origin="",
+        signature="",
+    ) -> None:
+        warnings.warn(
+            "``Completion`` is a provisional API (as of IPython 6.0). "
                       "It may change without warnings. "
                       "Use in corresponding context manager.",
-                      category=ProvisionalCompleterWarning, stacklevel=2)
+            category=ProvisionalCompleterWarning,
+            stacklevel=2,
+        )
 
         self.start = start
         self.end = end
@@ -520,7 +541,7 b' class Completion:'
         return '<Completion start=%s end=%s text=%r type=%r, signature=%r,>' % \
                 (self.start, self.end, self.text, self.type or '?', self.signature or '?')
 
-    def __eq__(self, other)->Bool:
+    def __eq__(self, other) -> bool:
         """
         Equality and hash do not hash the type (as some completer may not be
         able to infer the type), but are use to (partially) de-duplicate
@@ -554,7 +575,7 b' class SimpleCompletion:'
 
     __slots__ = ["text", "type"]
 
-    def __init__(self, text: str, *, type: str = None):
+    def __init__(self, text: str, *, type: Optional[str] = None):
         self.text = text
         self.type = type
 
@@ -588,14 +609,18 b' class SimpleMatcherResult(_MatcherResultBase, TypedDict):'
     # in order to get __orig_bases__ for documentation
 
     #: List of candidate completions
-    completions: Sequence[SimpleCompletion]
+    completions: Sequence[SimpleCompletion] | Iterator[SimpleCompletion]
 
 
 class _JediMatcherResult(_MatcherResultBase):
     """Matching result returned by Jedi (will be processed differently)"""
 
     #: list of candidate completions
-    completions: Iterable[_JediCompletionLike]
+    completions: Iterator[_JediCompletionLike]
+
+
+AnyMatcherCompletion = Union[_JediCompletionLike, SimpleCompletion]
+AnyCompletion = TypeVar("AnyCompletion", AnyMatcherCompletion, Completion)
 
 
 @dataclass
@@ -642,16 +667,21 b' MatcherResult = Union[SimpleMatcherResult, _JediMatcherResult]'
 
 
 class _MatcherAPIv1Base(Protocol):
-    def __call__(self, text: str) -> list[str]:
+    def __call__(self, text: str) -> List[str]:
         """Call signature."""
+        ...
+
+    #: Used to construct the default matcher identifier
+    __qualname__: str
 
 
 class _MatcherAPIv1Total(_MatcherAPIv1Base, Protocol):
     #: API version
     matcher_api_version: Optional[Literal[1]]
 
-    def __call__(self, text: str) -> list[str]:
+    def __call__(self, text: str) -> List[str]:
         """Call signature."""
+        ...
 
 
 #: Protocol describing Matcher API v1.
@@ -666,26 +696,61 b' class MatcherAPIv2(Protocol):'
 
     def __call__(self, context: CompletionContext) -> MatcherResult:
         """Call signature."""
+        ...
+
+    #: Used to construct the default matcher identifier
+    __qualname__: str
 
 
 Matcher: TypeAlias = Union[MatcherAPIv1, MatcherAPIv2]
 
 
+def _is_matcher_v1(matcher: Matcher) -> TypeGuard[MatcherAPIv1]:
+    api_version = _get_matcher_api_version(matcher)
+    return api_version == 1
+
+
+def _is_matcher_v2(matcher: Matcher) -> TypeGuard[MatcherAPIv2]:
+    api_version = _get_matcher_api_version(matcher)
+    return api_version == 2
+
+
+def _is_sizable(value: Any) -> TypeGuard[Sized]:
+    """Determines whether objects is sizable"""
+    return hasattr(value, "__len__")
+
+
+def _is_iterator(value: Any) -> TypeGuard[Iterator]:
+    """Determines whether objects is sizable"""
+    return hasattr(value, "__next__")
+
+
 def has_any_completions(result: MatcherResult) -> bool:
     """Check if any result includes any completions."""
-    if hasattr(result["completions"], "__len__"):
-        return len(result["completions"]) != 0
+    completions = result["completions"]
+    if _is_sizable(completions):
+        return len(completions) != 0
+    if _is_iterator(completions):
     try:
-        old_iterator = result["completions"]
+            old_iterator = completions
         first = next(old_iterator)
-        result["completions"] = itertools.chain([first], old_iterator)
+            result["completions"] = cast(
+                Iterator[SimpleCompletion],
+                itertools.chain([first], old_iterator),
+            )
         return True
     except StopIteration:
         return False
+    raise ValueError(
+        "Completions returned by matcher need to be an Iterator or a Sizable"
+    )
 
 
 def completion_matcher(
-    *, priority: float = None, identifier: str = None, api_version: int = 1
+    *,
+    priority: Optional[float] = None,
+    identifier: Optional[str] = None,
+    api_version: int = 1,
 ):
     """Adds attributes describing the matcher.
 
@@ -708,14 +773,14 b' def completion_matcher('
     """
 
     def wrapper(func: Matcher):
-        func.matcher_priority = priority or 0
-        func.matcher_identifier = identifier or func.__qualname__
-        func.matcher_api_version = api_version
+        func.matcher_priority = priority or 0  # type: ignore
+        func.matcher_identifier = identifier or func.__qualname__  # type: ignore
+        func.matcher_api_version = api_version  # type: ignore
         if TYPE_CHECKING:
             if api_version == 1:
-                func = cast(func, MatcherAPIv1)
+                func = cast(MatcherAPIv1, func)
             elif api_version == 2:
-                func = cast(func, MatcherAPIv2)
+                func = cast(MatcherAPIv2, func)
         return func
 
     return wrapper
@@ -902,12 +967,44 b' class CompletionSplitter(object):'
 
 class Completer(Configurable):
 
-    greedy = Bool(False,
-        help="""Activate greedy completion
-        PENDING DEPRECATION. this is now mostly taken care of with Jedi.
+    greedy = Bool(
+        False,
+        help="""Activate greedy completion.
+
+        .. deprecated:: 8.8
+            Use :any:`Completer.evaluation` and :any:`Completer.auto_close_dict_keys` instead.
 
-        This will enable completion on elements of lists, results of function calls, etc.,
-        but can be unsafe because the code is actually evaluated on TAB.
+        When enabled in IPython 8.8 or newer, changes configuration as follows:
+
+        - ``Completer.evaluation = 'unsafe'``
+        - ``Completer.auto_close_dict_keys = True``
+        """,
+    ).tag(config=True)
+
+    evaluation = Enum(
+        ("forbidden", "minimal", "limited", "unsafe", "dangerous"),
+        default_value="limited",
+        help="""Policy for code evaluation under completion.
+
+        Successive options allow to enable more eager evaluation for better
+        completion suggestions, including for nested dictionaries, nested lists,
+        or even results of function calls.
+        Setting ``unsafe`` or higher can lead to evaluation of arbitrary user
+        code on :kbd:`Tab` with potentially unwanted or dangerous side effects.
+
+        Allowed values are:
+
+        - ``forbidden``: no evaluation of code is permitted,
+        - ``minimal``: evaluation of literals and access to built-in namespace;
+          no item/attribute evaluationm no access to locals/globals,
+          no evaluation of any operations or comparisons.
+        - ``limited``: access to all namespaces, evaluation of hard-coded methods
+          (for example: :any:`dict.keys`, :any:`object.__getattr__`,
+          :any:`object.__getitem__`) on allow-listed objects (for example:
+          :any:`dict`, :any:`list`, :any:`tuple`, ``pandas.Series``),
+        - ``unsafe``: evaluation of all methods and function calls but not of
+          syntax with side-effects like `del x`,
+        - ``dangerous``: completely arbitrary evaluation.
         """,
     ).tag(config=True)
 
@@ -931,6 +1028,18 b' class Completer(Configurable):'
              "Includes completion of latex commands, unicode names, and expanding "
              "unicode characters back to latex commands.").tag(config=True)
 
+    auto_close_dict_keys = Bool(
+        False,
+        help="""
+        Enable auto-closing dictionary keys.
+
+        When enabled string keys will be suffixed with a final quote
+        (matching the opening quote), tuple keys will also receive a
+        separating comma if needed, and keys which are final will
+        receive a closing bracket (``]``).
+        """,
+    ).tag(config=True)
+
     def __init__(self, namespace=None, global_namespace=None, **kwargs):
         """Create a new completer for the command line.
 
@@ -1029,26 +1138,14 b' class Completer(Configurable):'
         with a __getattr__ hook is evaluated.
 
         """
-
-        # Another option, seems to work great. Catches things like ''.<tab>
-        m = re.match(r"(\S+(\.\w+)*)\.(\w*)$", text)
-
-        if m:
-            expr, attr = m.group(1, 3)
-        elif self.greedy:
             m2 = re.match(r"(.+)\.(\w*)$", self.line_buffer)
             if not m2:
                 return []
             expr, attr = m2.group(1,2)
-        else:
-            return []
 
-        try:
-            obj = eval(expr, self.namespace)
-        except:
-            try:
-                obj = eval(expr, self.global_namespace)
-            except:
+        obj = self._evaluate_expr(expr)
+
+        if obj is not_found:
                 return []
 
         if self.limit_to__all__ and hasattr(obj, '__all__'):
@@ -1068,8 +1165,31 b' class Completer(Configurable):'
             pass
         # Build match list to return
         n = len(attr)
-        return [u"%s.%s" % (expr, w) for w in words if w[:n] == attr ]
+        return ["%s.%s" % (expr, w) for w in words if w[:n] == attr]
 
+    def _evaluate_expr(self, expr):
+        obj = not_found
+        done = False
+        while not done and expr:
+            try:
+                obj = guarded_eval(
+                    expr,
+                    EvaluationContext(
+                        globals=self.global_namespace,
+                        locals=self.namespace,
+                        evaluation=self.evaluation,
+                    ),
+                )
+                done = True
+            except Exception as e:
+                if self.debug:
+                    print("Evaluation exception", e)
+                # trim the expression to remove any invalid prefix
+                # e.g. user starts `(d[`, so we get `expr = '(d'`,
+                # where parenthesis is not closed.
+                # TODO: make this faster by reusing parts of the computation?
+                expr = expr[1:]
+        return obj
 
 def get__all__entries(obj):
     """returns the strings in the __all__ attribute"""
@@ -1081,8 +1201,82 b' def get__all__entries(obj):'
     return [w for w in words if isinstance(w, str)]
 
 
-def match_dict_keys(keys: List[Union[str, bytes, Tuple[Union[str, bytes]]]], prefix: str, delims: str,
-                    extra_prefix: Optional[Tuple[str, bytes]]=None) -> Tuple[str, int, List[str]]:
+class _DictKeyState(enum.Flag):
+    """Represent state of the key match in context of other possible matches.
+
+    - given `d1 = {'a': 1}` completion on `d1['<tab>` will yield `{'a': END_OF_ITEM}` as there is no tuple.
+    - given `d2 = {('a', 'b'): 1}`: `d2['a', '<tab>` will yield `{'b': END_OF_TUPLE}` as there is no tuple members to add beyond `'b'`.
+    - given `d3 = {('a', 'b'): 1}`: `d3['<tab>` will yield `{'a': IN_TUPLE}` as `'a'` can be added.
+    - given `d4 = {'a': 1, ('a', 'b'): 2}`: `d4['<tab>` will yield `{'a': END_OF_ITEM & END_OF_TUPLE}`
+    """
+
+    BASELINE = 0
+    END_OF_ITEM = enum.auto()
+    END_OF_TUPLE = enum.auto()
+    IN_TUPLE = enum.auto()
+
+
+def _parse_tokens(c):
+    """Parse tokens even if there is an error."""
+    tokens = []
+    token_generator = tokenize.generate_tokens(iter(c.splitlines()).__next__)
+    while True:
+        try:
+            tokens.append(next(token_generator))
+        except tokenize.TokenError:
+            return tokens
+        except StopIteration:
+            return tokens
+
+
+def _match_number_in_dict_key_prefix(prefix: str) -> Union[str, None]:
+    """Match any valid Python numeric literal in a prefix of dictionary keys.
+
+    References:
+    - https://docs.python.org/3/reference/lexical_analysis.html#numeric-literals
+    - https://docs.python.org/3/library/tokenize.html
+    """
+    if prefix[-1].isspace():
+        # if user typed a space we do not have anything to complete
+        # even if there was a valid number token before
+        return None
+    tokens = _parse_tokens(prefix)
+    rev_tokens = reversed(tokens)
+    skip_over = {tokenize.ENDMARKER, tokenize.NEWLINE}
+    number = None
+    for token in rev_tokens:
+        if token.type in skip_over:
+            continue
+        if number is None:
+            if token.type == tokenize.NUMBER:
+                number = token.string
+                continue
+            else:
+                # we did not match a number
+                return None
+        if token.type == tokenize.OP:
+            if token.string == ",":
+                break
+            if token.string in {"+", "-"}:
+                number = token.string + number
+        else:
+            return None
+    return number
+
+
+_INT_FORMATS = {
+    "0b": bin,
+    "0o": oct,
+    "0x": hex,
+}
+
+
+def match_dict_keys(
+    keys: List[Union[str, bytes, Tuple[Union[str, bytes], ...]]],
+    prefix: str,
+    delims: str,
+    extra_prefix: Optional[Tuple[Union[str, bytes], ...]] = None,
+) -> Tuple[str, int, Dict[str, _DictKeyState]]:
     """Used by dict_key_matches, matching the prefix to a list of keys
 
     Parameters
@@ -1102,47 +1296,89 b' def match_dict_keys(keys: List[Union[str, bytes, Tuple[Union[str, bytes]]]], pre'
     A tuple of three elements: ``quote``, ``token_start``, ``matched``, with
     ``quote`` being the quote that need to be used to close current string.
     ``token_start`` the position where the replacement should start occurring,
-    ``matches`` a list of replacement/completion
-
+    ``matches`` a dictionary of replacement/completion keys on keys and values
+        indicating whether the state.
     """
     prefix_tuple = extra_prefix if extra_prefix else ()
-    Nprefix = len(prefix_tuple)
+
+    prefix_tuple_size = sum(
+        [
+            # for pandas, do not count slices as taking space
+            not isinstance(k, slice)
+            for k in prefix_tuple
+        ]
+    )
+    text_serializable_types = (str, bytes, int, float, slice)
+
     def filter_prefix_tuple(key):
         # Reject too short keys
-        if len(key) <= Nprefix:
+        if len(key) <= prefix_tuple_size:
             return False
-        # Reject keys with non str/bytes in it
+        # Reject keys which cannot be serialised to text
         for k in key:
-            if not isinstance(k, (str, bytes)):
+            if not isinstance(k, text_serializable_types):
                 return False
         # Reject keys that do not match the prefix
         for k, pt in zip(key, prefix_tuple):
-            if k != pt:
+            if k != pt and not isinstance(pt, slice):
                 return False
         # All checks passed!
         return True
 
-    filtered_keys:List[Union[str,bytes]] = []
-    def _add_to_filtered_keys(key):
-        if isinstance(key, (str, bytes)):
-            filtered_keys.append(key)
+    filtered_key_is_final: Dict[
+        Union[str, bytes, int, float], _DictKeyState
+    ] = defaultdict(lambda: _DictKeyState.BASELINE)
 
     for k in keys:
+        # If at least one of the matches is not final, mark as undetermined.
+        # This can happen with `d = {111: 'b', (111, 222): 'a'}` where
+        # `111` appears final on first match but is not final on the second.
+
         if isinstance(k, tuple):
             if filter_prefix_tuple(k):
-                _add_to_filtered_keys(k[Nprefix])
+                key_fragment = k[prefix_tuple_size]
+                filtered_key_is_final[key_fragment] |= (
+                    _DictKeyState.END_OF_TUPLE
+                    if len(k) == prefix_tuple_size + 1
+                    else _DictKeyState.IN_TUPLE
+                )
+        elif prefix_tuple_size > 0:
+            # we are completing a tuple but this key is not a tuple,
+            # so we should ignore it
+            pass
         else:
-            _add_to_filtered_keys(k)
+            if isinstance(k, text_serializable_types):
+                filtered_key_is_final[k] |= _DictKeyState.END_OF_ITEM
+
+    filtered_keys = filtered_key_is_final.keys()
 
     if not prefix:
-        return '', 0, [repr(k) for k in filtered_keys]
-    quote_match = re.search('["\']', prefix)
-    assert quote_match is not None # silence mypy
+        return "", 0, {repr(k): v for k, v in filtered_key_is_final.items()}
+
+    quote_match = re.search("(?:\"|')", prefix)
+    is_user_prefix_numeric = False
+
+    if quote_match:
     quote = quote_match.group()
+        valid_prefix = prefix + quote
     try:
-        prefix_str = eval(prefix + quote, {})
+            prefix_str = literal_eval(valid_prefix)
     except Exception:
-        return '', 0, []
+            return "", 0, {}
+    else:
+        # If it does not look like a string, let's assume
+        # we are dealing with a number or variable.
+        number_match = _match_number_in_dict_key_prefix(prefix)
+
+        # We do not want the key matcher to suggest variable names so we yield:
+        if number_match is None:
+            # The alternative would be to assume that user forgort the quote
+            # and if the substring matches, suggest adding it at the start.
+            return "", 0, {}
+
+        prefix_str = number_match
+        is_user_prefix_numeric = True
+        quote = ""
 
     pattern = '[^' + ''.join('\\' + c for c in delims) + ']*$'
     token_match = re.search(pattern, prefix, re.UNICODE)
@@ -1150,17 +1386,36 b' def match_dict_keys(keys: List[Union[str, bytes, Tuple[Union[str, bytes]]]], pre'
     token_start = token_match.start()
     token_prefix = token_match.group()
 
-    matched:List[str] = []
+    matched: Dict[str, _DictKeyState] = {}
+
+    str_key: Union[str, bytes]
+
     for key in filtered_keys:
+        if isinstance(key, (int, float)):
+            # User typed a number but this key is not a number.
+            if not is_user_prefix_numeric:
+                continue
+            str_key = str(key)
+            if isinstance(key, int):
+                int_base = prefix_str[:2].lower()
+                # if user typed integer using binary/oct/hex notation:
+                if int_base in _INT_FORMATS:
+                    int_format = _INT_FORMATS[int_base]
+                    str_key = int_format(key)
+        else:
+            # User typed a string but this key is a number.
+            if is_user_prefix_numeric:
+                continue
+            str_key = key
         try:
-            if not key.startswith(prefix_str):
+            if not str_key.startswith(prefix_str):
                 continue
-        except (AttributeError, TypeError, UnicodeError):
+        except (AttributeError, TypeError, UnicodeError) as e:
             # Python 3+ TypeError on b'a'.startswith('a') or vice-versa
             continue
 
         # reformat remainder of key to begin with prefix
-        rem = key[len(prefix_str):]
+        rem = str_key[len(prefix_str) :]
         # force repr wrapped in '
         rem_repr = repr(rem + '"') if isinstance(rem, str) else repr(rem + b'"')
         rem_repr = rem_repr[1 + rem_repr.index("'"):-2]
@@ -1171,7 +1426,9 b' def match_dict_keys(keys: List[Union[str, bytes, Tuple[Union[str, bytes]]]], pre'
             rem_repr = rem_repr.replace('"', '\\"')
 
         # then reinsert prefix from start of token
-        matched.append('%s%s' % (token_prefix, rem_repr))
+        match = "%s%s" % (token_prefix, rem_repr)
+
+        matched[match] = filtered_key_is_final[key]
     return quote, token_start, matched
 
 
@@ -1237,11 +1494,14 b' def position_to_cursor(text:str, offset:int)->Tuple[int, int]:'
     return line, col
 
 
-def _safe_isinstance(obj, module, class_name):
+def _safe_isinstance(obj, module, class_name, *attrs):
     """Checks if obj is an instance of module.class_name if loaded
     """
-    return (module in sys.modules and
-            isinstance(obj, getattr(import_module(module), class_name)))
+    if module in sys.modules:
+        m = sys.modules[module]
+        for attr in [class_name, *attrs]:
+            m = getattr(m, attr)
+        return isinstance(obj, m)
 
 
 @context_matcher()
@@ -1394,10 +1654,59 b' def _make_signature(completion)-> str:'
 _CompleteResult = Dict[str, MatcherResult]
 
 
+DICT_MATCHER_REGEX = re.compile(
+    r"""(?x)
+(  # match dict-referring - or any get item object - expression
+    .+
+)
+\[   # open bracket
+\s*  # and optional whitespace
+# Capture any number of serializable objects (e.g. "a", "b", 'c')
+# and slices
+((?:(?:
+    (?: # closed string
+        [uUbB]?  # string prefix (r not handled)
+        (?:
+            '(?:[^']|(?<!\\)\\')*'
+        |
+            "(?:[^"]|(?<!\\)\\")*"
+        )
+    )
+    |
+        # capture integers and slices
+        (?:[-+]?\d+)?(?::(?:[-+]?\d+)?){0,2}
+    |
+        # integer in bin/hex/oct notation
+        0[bBxXoO]_?(?:\w|\d)+
+    )
+    \s*,\s*
+)*)
+((?:
+    (?: # unclosed string
+        [uUbB]?  # string prefix (r not handled)
+        (?:
+            '(?:[^']|(?<!\\)\\')*
+            |
+            "(?:[^"]|(?<!\\)\\")*
+        )
+    )
+    |
+        # unfinished integer
+        (?:[-+]?\d+)
+    |
+        # integer in bin/hex/oct notation
+        0[bBxXoO]_?(?:\w|\d)+
+    )
+)?
+$
+"""
+)
+
+
 def _convert_matcher_v1_result_to_v2(
     matches: Sequence[str],
     type: str,
-    fragment: str = None,
+    fragment: Optional[str] = None,
     suppress_if_matches: bool = False,
 ) -> SimpleMatcherResult:
     """Utility to help with transition"""
@@ -1407,20 +1716,22 b' def _convert_matcher_v1_result_to_v2('
     }
     if fragment is not None:
         result["matched_fragment"] = fragment
-    return result
+    return cast(SimpleMatcherResult, result)
 
 
 class IPCompleter(Completer):
     """Extension of the completer class with IPython-specific features"""
 
-    __dict_key_regexps: Optional[Dict[bool,Pattern]] = None
-
     @observe('greedy')
     def _greedy_changed(self, change):
         """update the splitter and readline delims when greedy is changed"""
-        if change['new']:
+        if change["new"]:
+            self.evaluation = "unsafe"
+            self.auto_close_dict_keys = True
             self.splitter.delims = GREEDY_DELIMS
         else:
+            self.evaluation = "limited"
+            self.auto_close_dict_keys = False
             self.splitter.delims = DELIMS
 
     dict_keys_only = Bool(
@@ -1607,7 +1918,7 b' class IPCompleter(Completer):'
 
         if not self.backslash_combining_completions:
             for matcher in self._backslash_combining_matchers:
-                self.disable_matchers.append(matcher.matcher_identifier)
+                self.disable_matchers.append(_get_matcher_id(matcher))
 
         if not self.merge_completions:
             self.suppress_competing_matchers = True
@@ -1897,7 +2208,7 b' class IPCompleter(Completer):'
 
     def _jedi_matches(
         self, cursor_column: int, cursor_line: int, text: str
-    ) -> Iterable[_JediCompletionLike]:
+    ) -> Iterator[_JediCompletionLike]:
         """
         Return a list of :any:`jedi.api.Completion`s object from a ``text`` and
         cursor position.
@@ -1963,15 +2274,23 b' class IPCompleter(Completer):'
                 print("Error detecting if completing a non-finished string :", e, '|')
 
         if not try_jedi:
-            return []
+            return iter([])
         try:
             return filter(completion_filter, interpreter.complete(column=cursor_column, line=cursor_line + 1))
         except Exception as e:
             if self.debug:
-                return [_FakeJediCompletion('Oops Jedi has crashed, please report a bug with the following:\n"""\n%s\ns"""' % (e))]
+                return iter(
+                    [
+                        _FakeJediCompletion(
+                            'Oops Jedi has crashed, please report a bug with the following:\n"""\n%s\ns"""'
+                            % (e)
+                        )
+                    ]
+                )
             else:
-                return []
+                return iter([])
 
+    @completion_matcher(api_version=1)
     def python_matches(self, text: str) -> Iterable[str]:
         """Match attributes or global python names"""
         if "." in text:
@@ -2149,12 +2468,16 b' class IPCompleter(Completer):'
             return method()
 
         # Special case some common in-memory dict-like types
-        if isinstance(obj, dict) or\
-           _safe_isinstance(obj, 'pandas', 'DataFrame'):
+        if isinstance(obj, dict) or _safe_isinstance(obj, "pandas", "DataFrame"):
             try:
                 return list(obj.keys())
             except Exception:
                 return []
+        elif _safe_isinstance(obj, "pandas", "core", "indexing", "_LocIndexer"):
+            try:
+                return list(obj.obj.keys())
+            except Exception:
+                return []
         elif _safe_isinstance(obj, 'numpy', 'ndarray') or\
              _safe_isinstance(obj, 'numpy', 'void'):
             return obj.dtype.names or []
@@ -2175,74 +2498,49 b' class IPCompleter(Completer):'
             You can use :meth:`dict_key_matcher` instead.
         """
 
-        if self.__dict_key_regexps is not None:
-            regexps = self.__dict_key_regexps
-        else:
-            dict_key_re_fmt = r'''(?x)
-            (  # match dict-referring expression wrt greedy setting
-                %s
-            )
-            \[   # open bracket
-            \s*  # and optional whitespace
-            # Capture any number of str-like objects (e.g. "a", "b", 'c')
-            ((?:[uUbB]?  # string prefix (r not handled)
-                (?:
-                    '(?:[^']|(?<!\\)\\')*'
-                |
-                    "(?:[^"]|(?<!\\)\\")*"
-                )
-                \s*,\s*
-            )*)
-            ([uUbB]?  # string prefix (r not handled)
-                (?:   # unclosed string
-                    '(?:[^']|(?<!\\)\\')*
-                |
-                    "(?:[^"]|(?<!\\)\\")*
-                )
-            )?
-            $
-            '''
-            regexps = self.__dict_key_regexps = {
-                False: re.compile(dict_key_re_fmt % r'''
-                                  # identifiers separated by .
-                                  (?!\d)\w+
-                                  (?:\.(?!\d)\w+)*
-                                  '''),
-                True: re.compile(dict_key_re_fmt % '''
-                                 .+
-                                 ''')
-            }
+        # Short-circuit on closed dictionary (regular expression would
+        # not match anyway, but would take quite a while).
+        if self.text_until_cursor.strip().endswith("]"):
+            return []
 
-        match = regexps[self.greedy].search(self.text_until_cursor)
+        match = DICT_MATCHER_REGEX.search(self.text_until_cursor)
 
         if match is None:
             return []
 
-        expr, prefix0, prefix = match.groups()
-        try:
-            obj = eval(expr, self.namespace)
-        except Exception:
-            try:
-                obj = eval(expr, self.global_namespace)
-            except Exception:
+        expr, prior_tuple_keys, key_prefix = match.groups()
+
+        obj = self._evaluate_expr(expr)
+
+        if obj is not_found:
                 return []
 
         keys = self._get_keys(obj)
         if not keys:
             return keys
 
-        extra_prefix = eval(prefix0) if prefix0 != '' else None
+        tuple_prefix = guarded_eval(
+            prior_tuple_keys,
+            EvaluationContext(
+                globals=self.global_namespace,
+                locals=self.namespace,
+                evaluation=self.evaluation,
+                in_subscript=True,
+            ),
+        )
 
-        closing_quote, token_offset, matches = match_dict_keys(keys, prefix, self.splitter.delims, extra_prefix=extra_prefix)
+        closing_quote, token_offset, matches = match_dict_keys(
+            keys, key_prefix, self.splitter.delims, extra_prefix=tuple_prefix
+        )
         if not matches:
-            return matches
+            return []
 
         # get the cursor position of
         # - the text being completed
         # - the start of the key text
         # - the start of the completion
         text_start = len(self.text_until_cursor) - len(text)
-        if prefix:
+        if key_prefix:
             key_start = match.start(3)
             completion_start = key_start + token_offset
         else:
@@ -2254,26 +2552,61 b' class IPCompleter(Completer):'
         else:
             leading = text[text_start:completion_start]
 
-        # the index of the `[` character
-        bracket_idx = match.end(1)
-
         # append closing quote and bracket as appropriate
         # this is *not* appropriate if the opening quote or bracket is outside
-        # the text given to this method
-        suf = ''
-        continuation = self.line_buffer[len(self.text_until_cursor):]
-        if key_start > text_start and closing_quote:
-            # quotes were opened inside text, maybe close them
+        # the text given to this method, e.g. `d["""a\nt
+        can_close_quote = False
+        can_close_bracket = False
+
+        continuation = self.line_buffer[len(self.text_until_cursor) :].strip()
+
             if continuation.startswith(closing_quote):
+            # do not close if already closed, e.g. `d['a<tab>'`
                 continuation = continuation[len(closing_quote):]
             else:
-                suf += closing_quote
-        if bracket_idx > text_start:
-            # brackets were opened inside text, maybe close them
-            if not continuation.startswith(']'):
-                suf += ']'
+            can_close_quote = True
+
+        continuation = continuation.strip()
+
+        # e.g. `pandas.DataFrame` has different tuple indexer behaviour,
+        # handling it is out of scope, so let's avoid appending suffixes.
+        has_known_tuple_handling = isinstance(obj, dict)
+
+        can_close_bracket = (
+            not continuation.startswith("]") and self.auto_close_dict_keys
+        )
+        can_close_tuple_item = (
+            not continuation.startswith(",")
+            and has_known_tuple_handling
+            and self.auto_close_dict_keys
+        )
+        can_close_quote = can_close_quote and self.auto_close_dict_keys
 
-        return [leading + k + suf for k in matches]
+        # fast path if closing qoute should be appended but not suffix is allowed
+        if not can_close_quote and not can_close_bracket and closing_quote:
+            return [leading + k for k in matches]
+
+        results = []
+
+        end_of_tuple_or_item = _DictKeyState.END_OF_TUPLE | _DictKeyState.END_OF_ITEM
+
+        for k, state_flag in matches.items():
+            result = leading + k
+            if can_close_quote and closing_quote:
+                result += closing_quote
+
+            if state_flag == end_of_tuple_or_item:
+                # We do not know which suffix to add,
+                # e.g. both tuple item and string
+                # match this item.
+                pass
+
+            if state_flag in end_of_tuple_or_item and can_close_bracket:
+                result += "]"
+            if state_flag == _DictKeyState.IN_TUPLE and can_close_tuple_item:
+                result += ", "
+            results.append(result)
+        return results
 
     @context_matcher()
     def unicode_name_matcher(self, context: CompletionContext):
@@ -2516,17 +2849,23 b' class IPCompleter(Completer):'
 
         jedi_matcher_id = _get_matcher_id(self._jedi_matcher)
 
+        def is_non_jedi_result(
+            result: MatcherResult, identifier: str
+        ) -> TypeGuard[SimpleMatcherResult]:
+            return identifier != jedi_matcher_id
+
         results = self._complete(
             full_text=full_text, cursor_line=cursor_line, cursor_pos=cursor_column
         )
+
         non_jedi_results: Dict[str, SimpleMatcherResult] = {
             identifier: result
             for identifier, result in results.items()
-            if identifier != jedi_matcher_id
+            if is_non_jedi_result(result, identifier)
         }
 
         jedi_matches = (
-            cast(results[jedi_matcher_id], _JediMatcherResult)["completions"]
+            cast(_JediMatcherResult, results[jedi_matcher_id])["completions"]
             if jedi_matcher_id in results
             else ()
         )
@@ -2581,8 +2920,8 b' class IPCompleter(Completer):'
                 signature="",
             )
 
-        ordered = []
-        sortable = []
+        ordered: List[Completion] = []
+        sortable: List[Completion] = []
 
         for origin, result in non_jedi_results.items():
             matched_text = result["matched_fragment"]
@@ -2672,8 +3011,8 b' class IPCompleter(Completer):'
         abort_if_offset_changes: bool,
     ):
 
-        sortable = []
-        ordered = []
+        sortable: List[AnyMatcherCompletion] = []
+        ordered: List[AnyMatcherCompletion] = []
         most_recent_fragment = None
         for identifier, result in results.items():
             if identifier in skip_matchers:
@@ -2772,11 +3111,11 b' class IPCompleter(Completer):'
         )
 
         # Start with a clean slate of completions
-        results = {}
+        results: Dict[str, MatcherResult] = {}
 
         jedi_matcher_id = _get_matcher_id(self._jedi_matcher)
 
-        suppressed_matchers = set()
+        suppressed_matchers: Set[str] = set()
 
         matchers = {
             _get_matcher_id(matcher): matcher
@@ -2786,7 +3125,6 b' class IPCompleter(Completer):'
         }
 
         for matcher_id, matcher in matchers.items():
-            api_version = _get_matcher_api_version(matcher)
             matcher_id = _get_matcher_id(matcher)
 
             if matcher_id in self.disable_matchers:
@@ -2798,14 +3136,16 b' class IPCompleter(Completer):'
             if matcher_id in suppressed_matchers:
                 continue
 
+            result: MatcherResult
             try:
-                if api_version == 1:
+                if _is_matcher_v1(matcher):
                     result = _convert_matcher_v1_result_to_v2(
                         matcher(text), type=_UNKNOWN_TYPE
                     )
-                elif api_version == 2:
-                    result = cast(matcher, MatcherAPIv2)(context)
+                elif _is_matcher_v2(matcher):
+                    result = matcher(context)
                 else:
+                    api_version = _get_matcher_api_version(matcher)
                     raise ValueError(f"Unsupported API version {api_version}")
             except:
                 # Show the ugly traceback if the matcher causes an
@@ -2817,7 +3157,9 b' class IPCompleter(Completer):'
             result["matched_fragment"] = result.get("matched_fragment", context.token)
 
             if not suppressed_matchers:
-                suppression_recommended = result.get("suppress", False)
+                suppression_recommended: Union[bool, Set[str]] = result.get(
+                    "suppress", False
+                )
 
                 suppression_config = (
                     self.suppress_competing_matchers.get(matcher_id, None)
@@ -2830,10 +3172,12 b' class IPCompleter(Completer):'
                 ) and has_any_completions(result)
 
                 if should_suppress:
-                    suppression_exceptions = result.get("do_not_suppress", set())
-                    try:
+                    suppression_exceptions: Set[str] = result.get(
+                        "do_not_suppress", set()
+                    )
+                    if isinstance(suppression_recommended, Iterable):
                         to_suppress = set(suppression_recommended)
-                    except TypeError:
+                    else:
                         to_suppress = set(matchers)
                     suppressed_matchers = to_suppress - suppression_exceptions
 
@@ -2860,9 +3204,9 b' class IPCompleter(Completer):'
 
     @staticmethod
     def _deduplicate(
-        matches: Sequence[SimpleCompletion],
-    ) -> Iterable[SimpleCompletion]:
-        filtered_matches = {}
+        matches: Sequence[AnyCompletion],
+    ) -> Iterable[AnyCompletion]:
+        filtered_matches: Dict[str, AnyCompletion] = {}
         for match in matches:
             text = match.text
             if (
@@ -2874,7 +3218,7 b' class IPCompleter(Completer):'
         return filtered_matches.values()
 
     @staticmethod
-    def _sort(matches: Sequence[SimpleCompletion]):
+    def _sort(matches: Sequence[AnyCompletion]):
         return sorted(matches, key=lambda x: completions_sorting_key(x.text))
 
     @context_matcher()

IPython/core/interactiveshell.py

@@ -389,6 +389,9 b' class InteractiveShell(SingletonConfigurable):'
     displayhook_class = Type(DisplayHook)
     display_pub_class = Type(DisplayPublisher)
     compiler_class = Type(CachingCompiler)
+    inspector_class = Type(
+        oinspect.Inspector, help="Class to use to instantiate the shell inspector"
+    ).tag(config=True)
 
     sphinxify_docstring = Bool(False, help=
         """
@@ -755,10 +758,12 b' class InteractiveShell(SingletonConfigurable):'
     @observe('colors')
     def init_inspector(self, changes=None):
         # Object inspector
-        self.inspector = oinspect.Inspector(oinspect.InspectColors,
+        self.inspector = self.inspector_class(
+            oinspect.InspectColors,
                                             PyColorize.ANSICodeColors,
                                             self.colors,
-                                            self.object_info_string_level)
+            self.object_info_string_level,
+        )
 
     def init_io(self):
         # implemented in subclasses, TerminalInteractiveShell does call
@@ -3138,8 +3143,12 b' class InteractiveShell(SingletonConfigurable):'
             else:
                 cell = raw_cell
 
+        # Do NOT store paste/cpaste magic history
+        if "get_ipython().run_line_magic(" in cell and "paste" in cell:
+            store_history = False
+
         # Store raw and processed history
-        if store_history and raw_cell.strip(" %") != "paste":
+        if store_history:
             self.history_manager.store_inputs(self.execution_count, cell, raw_cell)
         if not silent:
             self.logger.log(cell, raw_cell)

IPython/core/magics/config.py

@@ -68,94 +68,22 b' class ConfigMagics(Magics):'
         To view what is configurable on a given class, just pass the class
         name::
 
-            In [2]: %config IPCompleter
-            IPCompleter(Completer) options
-            ----------------------------
-            IPCompleter.backslash_combining_completions=<Bool>
-                Enable unicode completions, e.g. \\alpha<tab> . Includes completion of latex
-                commands, unicode names, and expanding unicode characters back to latex
-                commands.
-                Current: True
-            IPCompleter.debug=<Bool>
-                Enable debug for the Completer. Mostly print extra information for
-                experimental jedi integration.
+            In [2]: %config LoggingMagics
+            LoggingMagics(Magics) options
+            ---------------------------
+            LoggingMagics.quiet=<Bool>
+                Suppress output of log state when logging is enabled
                 Current: False
-            IPCompleter.disable_matchers=<list-item-1>...
-                List of matchers to disable.
-                        The list should contain matcher identifiers (see
-                :any:`completion_matcher`).
-                Current: []
-            IPCompleter.greedy=<Bool>
-                Activate greedy completion
-                        PENDING DEPRECATION. this is now mostly taken care of with Jedi.
-                        This will enable completion on elements of lists, results of function calls, etc.,
-                        but can be unsafe because the code is actually evaluated on TAB.
-                Current: False
-            IPCompleter.jedi_compute_type_timeout=<Int>
-                Experimental: restrict time (in milliseconds) during which Jedi can compute types.
-                        Set to 0 to stop computing types. Non-zero value lower than 100ms may hurt
-                        performance by preventing jedi to build its cache.
-                Current: 400
-            IPCompleter.limit_to__all__=<Bool>
-                DEPRECATED as of version 5.0.
-                Instruct the completer to use __all__ for the completion
-                Specifically, when completing on ``object.<tab>``.
-                When True: only those names in obj.__all__ will be included.
-                When False [default]: the __all__ attribute is ignored
-                Current: False
-            IPCompleter.merge_completions=<Bool>
-                Whether to merge completion results into a single list
-                        If False, only the completion results from the first non-empty
-                        completer will be returned.
-                        As of version 8.6.0, setting the value to ``False`` is an alias for:
-                        ``IPCompleter.suppress_competing_matchers = True.``.
-                Current: True
-            IPCompleter.omit__names=<Enum>
-                Instruct the completer to omit private method names
-                        Specifically, when completing on ``object.<tab>``.
-                        When 2 [default]: all names that start with '_' will be excluded.
-                        When 1: all 'magic' names (``__foo__``) will be excluded.
-                        When 0: nothing will be excluded.
-                Choices: any of [0, 1, 2]
-                Current: 2
-            IPCompleter.profile_completions=<Bool>
-                If True, emit profiling data for completion subsystem using cProfile.
-                Current: False
-            IPCompleter.profiler_output_dir=<Unicode>
-                Template for path at which to output profile data for completions.
-                Current: '.completion_profiles'
-            IPCompleter.suppress_competing_matchers=<Union>
-                Whether to suppress completions from other *Matchers*.
-                When set to ``None`` (default) the matchers will attempt to auto-detect
-                whether suppression of other matchers is desirable. For example, at the
-                beginning of a line followed by `%` we expect a magic completion to be the
-                only applicable option, and after ``my_dict['`` we usually expect a
-                completion with an existing dictionary key.
-                If you want to disable this heuristic and see completions from all matchers,
-                set ``IPCompleter.suppress_competing_matchers = False``. To disable the
-                heuristic for specific matchers provide a dictionary mapping:
-                ``IPCompleter.suppress_competing_matchers = {'IPCompleter.dict_key_matcher':
-                False}``.
-                Set ``IPCompleter.suppress_competing_matchers = True`` to limit completions
-                to the set of matchers with the highest priority; this is equivalent to
-                ``IPCompleter.merge_completions`` and can be beneficial for performance, but
-                will sometimes omit relevant candidates from matchers further down the
-                priority list.
-                Current: None
-            IPCompleter.use_jedi=<Bool>
-                Experimental: Use Jedi to generate autocompletions. Default to True if jedi
-                is installed.
-                Current: True
 
         but the real use is in setting values::
 
-            In [3]: %config IPCompleter.greedy = True
+            In [3]: %config LoggingMagics.quiet = True
 
         and these values are read from the user_ns if they are variables::
 
-            In [4]: feeling_greedy=False
+            In [4]: feeling_quiet=False
 
-            In [5]: %config IPCompleter.greedy = feeling_greedy
+            In [5]: %config LoggingMagics.quiet = feeling_quiet
 
         """
         from traitlets.config.loader import Config

IPython/core/oinspect.py

@@ -16,6 +16,7 b" __all__ = ['Inspector','InspectColors']"
 import ast
 import inspect
 from inspect import signature
+import html
 import linecache
 import warnings
 import os
@@ -530,8 +531,8 b' class Inspector(Colorable):'
 
         """
         defaults = {
-            'text/plain': text,
-            'text/html': '<pre>' + text + '</pre>'
+            "text/plain": text,
+            "text/html": f"<pre>{html.escape(text)}</pre>",
         }
 
         if formatter is None:
@@ -542,66 +543,66 b' class Inspector(Colorable):'
             if not isinstance(formatted, dict):
                 # Handle the deprecated behavior of a formatter returning
                 # a string instead of a mime bundle.
-                return {
-                    'text/plain': formatted,
-                    'text/html': '<pre>' + formatted + '</pre>'
-                }
+                return {"text/plain": formatted, "text/html": f"<pre>{formatted}</pre>"}
 
             else:
                 return dict(defaults, **formatted)
 
 
     def format_mime(self, bundle):
+        """Format a mimebundle being created by _make_info_unformatted into a real mimebundle"""
+        # Format text/plain mimetype
+        if isinstance(bundle["text/plain"], (list, tuple)):
+            # bundle['text/plain'] is a list of (head, formatted body) pairs
+            lines = []
+            _len = max(len(h) for h, _ in bundle["text/plain"])
 
-        text_plain = bundle['text/plain']
-
-        text = ''
-        heads, bodies = list(zip(*text_plain))
-        _len = max(len(h) for h in heads)
+            for head, body in bundle["text/plain"]:
+                body = body.strip("\n")
+                delim = "\n" if "\n" in body else " "
+                lines.append(
+                    f"{self.__head(head+':')}{(_len - len(head))*' '}{delim}{body}"
+                )
 
-        for head, body in zip(heads, bodies):
-            body = body.strip('\n')
-            delim = '\n' if '\n' in body else ' '
-            text += self.__head(head+':') + (_len - len(head))*' ' +delim + body +'\n'
+            bundle["text/plain"] = "\n".join(lines)
 
-        bundle['text/plain'] = text
+        # Format the text/html mimetype
+        if isinstance(bundle["text/html"], (list, tuple)):
+            # bundle['text/html'] is a list of (head, formatted body) pairs
+            bundle["text/html"] = "\n".join(
+                (f"<h1>{head}</h1>\n{body}" for (head, body) in bundle["text/html"])
+            )
         return bundle
 
-    def _get_info(
-        self, obj, oname="", formatter=None, info=None, detail_level=0, omit_sections=()
+    def _append_info_field(
+        self, bundle, title: str, key: str, info, omit_sections, formatter
     ):
-        """Retrieve an info dict and format it.
-
-        Parameters
-        ----------
-        obj : any
-            Object to inspect and return info from
-        oname : str (default: ''):
-            Name of the variable pointing to `obj`.
-        formatter : callable
-        info
-            already computed information
-        detail_level : integer
-            Granularity of detail level, if set to 1, give more information.
-        omit_sections : container[str]
-            Titles or keys to omit from output (can be set, tuple, etc., anything supporting `in`)
-        """
-
-        info = self.info(obj, oname=oname, info=info, detail_level=detail_level)
-
-        _mime = {
-            'text/plain': [],
-            'text/html': '',
-        }
-
-        def append_field(bundle, title:str, key:str, formatter=None):
+        """Append an info value to the unformatted mimebundle being constructed by _make_info_unformatted"""
             if title in omit_sections or key in omit_sections:
                 return
             field = info[key]
             if field is not None:
                 formatted_field = self._mime_format(field, formatter)
-                bundle['text/plain'].append((title, formatted_field['text/plain']))
-                bundle['text/html'] += '<h1>' + title + '</h1>\n' + formatted_field['text/html'] + '\n'
+            bundle["text/plain"].append((title, formatted_field["text/plain"]))
+            bundle["text/html"].append((title, formatted_field["text/html"]))
+
+    def _make_info_unformatted(self, obj, info, formatter, detail_level, omit_sections):
+        """Assemble the mimebundle as unformatted lists of information"""
+        bundle = {
+            "text/plain": [],
+            "text/html": [],
+        }
+
+        # A convenience function to simplify calls below
+        def append_field(bundle, title: str, key: str, formatter=None):
+            self._append_info_field(
+                bundle,
+                title=title,
+                key=key,
+                info=info,
+                omit_sections=omit_sections,
+                formatter=formatter,
+            )
 
         def code_formatter(text):
             return {
@@ -609,57 +610,82 b' class Inspector(Colorable):'
                 'text/html': pylight(text)
             }
 
-        if info['isalias']:
-            append_field(_mime, 'Repr', 'string_form')
+        if info["isalias"]:
+            append_field(bundle, "Repr", "string_form")
 
         elif info['ismagic']:
             if detail_level > 0:
-                append_field(_mime, 'Source', 'source', code_formatter)
+                append_field(bundle, "Source", "source", code_formatter)
             else:
-                append_field(_mime, 'Docstring', 'docstring', formatter)
-            append_field(_mime, 'File', 'file')
+                append_field(bundle, "Docstring", "docstring", formatter)
+            append_field(bundle, "File", "file")
 
         elif info['isclass'] or is_simple_callable(obj):
             # Functions, methods, classes
-            append_field(_mime, 'Signature', 'definition', code_formatter)
-            append_field(_mime, 'Init signature', 'init_definition', code_formatter)
-            append_field(_mime, 'Docstring', 'docstring', formatter)
-            if detail_level > 0 and info['source']:
-                append_field(_mime, 'Source', 'source', code_formatter)
+            append_field(bundle, "Signature", "definition", code_formatter)
+            append_field(bundle, "Init signature", "init_definition", code_formatter)
+            append_field(bundle, "Docstring", "docstring", formatter)
+            if detail_level > 0 and info["source"]:
+                append_field(bundle, "Source", "source", code_formatter)
             else:
-                append_field(_mime, 'Init docstring', 'init_docstring', formatter)
+                append_field(bundle, "Init docstring", "init_docstring", formatter)
 
-            append_field(_mime, 'File', 'file')
-            append_field(_mime, 'Type', 'type_name')
-            append_field(_mime, 'Subclasses', 'subclasses')
+            append_field(bundle, "File", "file")
+            append_field(bundle, "Type", "type_name")
+            append_field(bundle, "Subclasses", "subclasses")
 
         else:
             # General Python objects
-            append_field(_mime, 'Signature', 'definition', code_formatter)
-            append_field(_mime, 'Call signature', 'call_def', code_formatter)
-            append_field(_mime, 'Type', 'type_name')
-            append_field(_mime, 'String form', 'string_form')
+            append_field(bundle, "Signature", "definition", code_formatter)
+            append_field(bundle, "Call signature", "call_def", code_formatter)
+            append_field(bundle, "Type", "type_name")
+            append_field(bundle, "String form", "string_form")
 
             # Namespace
-            if info['namespace'] != 'Interactive':
-                append_field(_mime, 'Namespace', 'namespace')
+            if info["namespace"] != "Interactive":
+                append_field(bundle, "Namespace", "namespace")
 
-            append_field(_mime, 'Length', 'length')
-            append_field(_mime, 'File', 'file')
+            append_field(bundle, "Length", "length")
+            append_field(bundle, "File", "file")
 
             # Source or docstring, depending on detail level and whether
             # source found.
-            if detail_level > 0 and info['source']:
-                append_field(_mime, 'Source', 'source', code_formatter)
+            if detail_level > 0 and info["source"]:
+                append_field(bundle, "Source", "source", code_formatter)
             else:
-                append_field(_mime, 'Docstring', 'docstring', formatter)
+                append_field(bundle, "Docstring", "docstring", formatter)
+
+            append_field(bundle, "Class docstring", "class_docstring", formatter)
+            append_field(bundle, "Init docstring", "init_docstring", formatter)
+            append_field(bundle, "Call docstring", "call_docstring", formatter)
+        return bundle
 
-            append_field(_mime, 'Class docstring', 'class_docstring', formatter)
-            append_field(_mime, 'Init docstring', 'init_docstring', formatter)
-            append_field(_mime, 'Call docstring', 'call_docstring', formatter)
 
+    def _get_info(
+        self, obj, oname="", formatter=None, info=None, detail_level=0, omit_sections=()
+    ):
+        """Retrieve an info dict and format it.
+
+        Parameters
+        ----------
+        obj : any
+            Object to inspect and return info from
+        oname : str (default: ''):
+            Name of the variable pointing to `obj`.
+        formatter : callable
+        info
+            already computed information
+        detail_level : integer
+            Granularity of detail level, if set to 1, give more information.
+        omit_sections : container[str]
+            Titles or keys to omit from output (can be set, tuple, etc., anything supporting `in`)
+        """
 
-        return self.format_mime(_mime)
+        info = self.info(obj, oname=oname, info=info, detail_level=detail_level)
+        bundle = self._make_info_unformatted(
+            obj, info, formatter, detail_level=detail_level, omit_sections=omit_sections
+        )
+        return self.format_mime(bundle)
 
     def pinfo(
         self,

IPython/core/tests/test_completer.py

@@ -24,6 +24,7 b' from IPython.core.completer import ('
     provisionalcompleter,
     match_dict_keys,
     _deduplicate_completions,
+    _match_number_in_dict_key_prefix,
     completion_matcher,
     SimpleCompletion,
     CompletionContext,
@@ -113,6 +114,17 b' def greedy_completion():'
 
 
 @contextmanager
+def evaluation_policy(evaluation: str):
+    ip = get_ipython()
+    evaluation_original = ip.Completer.evaluation
+    try:
+        ip.Completer.evaluation = evaluation
+        yield
+    finally:
+        ip.Completer.evaluation = evaluation_original
+
+
+@contextmanager
 def custom_matchers(matchers):
     ip = get_ipython()
     try:
@@ -170,7 +182,6 b' def check_line_split(splitter, test_specs):'
         out = splitter.split_line(line, cursor_pos)
         assert out == split
 
-
 def test_line_split():
     """Basic line splitter test with default specs."""
     sp = completer.CompletionSplitter()
@@ -841,18 +852,45 b' class TestCompleter(unittest.TestCase):'
         """
         delims = " \t\n`!@#$^&*()=+[{]}\\|;:'\",<>?"
 
-        keys = ["foo", b"far"]
-        assert match_dict_keys(keys, "b'", delims=delims) == ("'", 2, ["far"])
-        assert match_dict_keys(keys, "b'f", delims=delims) == ("'", 2, ["far"])
-        assert match_dict_keys(keys, 'b"', delims=delims) == ('"', 2, ["far"])
-        assert match_dict_keys(keys, 'b"f', delims=delims) == ('"', 2, ["far"])
-
-        assert match_dict_keys(keys, "'", delims=delims) == ("'", 1, ["foo"])
-        assert match_dict_keys(keys, "'f", delims=delims) == ("'", 1, ["foo"])
-        assert match_dict_keys(keys, '"', delims=delims) == ('"', 1, ["foo"])
-        assert match_dict_keys(keys, '"f', delims=delims) == ('"', 1, ["foo"])
+        def match(*args, **kwargs):
+            quote, offset, matches = match_dict_keys(*args, delims=delims, **kwargs)
+            return quote, offset, list(matches)
 
-        match_dict_keys
+        keys = ["foo", b"far"]
+        assert match(keys, "b'") == ("'", 2, ["far"])
+        assert match(keys, "b'f") == ("'", 2, ["far"])
+        assert match(keys, 'b"') == ('"', 2, ["far"])
+        assert match(keys, 'b"f') == ('"', 2, ["far"])
+
+        assert match(keys, "'") == ("'", 1, ["foo"])
+        assert match(keys, "'f") == ("'", 1, ["foo"])
+        assert match(keys, '"') == ('"', 1, ["foo"])
+        assert match(keys, '"f') == ('"', 1, ["foo"])
+
+        # Completion on first item of tuple
+        keys = [("foo", 1111), ("foo", 2222), (3333, "bar"), (3333, "test")]
+        assert match(keys, "'f") == ("'", 1, ["foo"])
+        assert match(keys, "33") == ("", 0, ["3333"])
+
+        # Completion on numbers
+        keys = [
+            0xDEADBEEF,
+            1111,
+            1234,
+            "1999",
+            0b10101,
+            22,
+        ]  # 0xDEADBEEF = 3735928559; 0b10101 = 21
+        assert match(keys, "0xdead") == ("", 0, ["0xdeadbeef"])
+        assert match(keys, "1") == ("", 0, ["1111", "1234"])
+        assert match(keys, "2") == ("", 0, ["21", "22"])
+        assert match(keys, "0b101") == ("", 0, ["0b10101", "0b10110"])
+
+        # Should yield on variables
+        assert match(keys, "a_variable") == ("", 0, [])
+
+        # Should pass over invalid literals
+        assert match(keys, "'' ''") == ("", 0, [])
 
     def test_match_dict_keys_tuple(self):
         """
@@ -863,25 +901,91 b' class TestCompleter(unittest.TestCase):'
         
         keys = [("foo", "bar"), ("foo", "oof"), ("foo", b"bar"), ('other', 'test')]
 
+        def match(*args, extra=None, **kwargs):
+            quote, offset, matches = match_dict_keys(
+                *args, delims=delims, extra_prefix=extra, **kwargs
+            )
+            return quote, offset, list(matches)
+
         # Completion on first key == "foo"
-        assert match_dict_keys(keys, "'", delims=delims, extra_prefix=("foo",)) == ("'", 1, ["bar", "oof"])
-        assert match_dict_keys(keys, "\"", delims=delims, extra_prefix=("foo",)) == ("\"", 1, ["bar", "oof"])
-        assert match_dict_keys(keys, "'o", delims=delims, extra_prefix=("foo",)) == ("'", 1, ["oof"])
-        assert match_dict_keys(keys, "\"o", delims=delims, extra_prefix=("foo",)) == ("\"", 1, ["oof"])
-        assert match_dict_keys(keys, "b'", delims=delims, extra_prefix=("foo",)) == ("'", 2, ["bar"])
-        assert match_dict_keys(keys, "b\"", delims=delims, extra_prefix=("foo",)) == ("\"", 2, ["bar"])
-        assert match_dict_keys(keys, "b'b", delims=delims, extra_prefix=("foo",)) == ("'", 2, ["bar"])
-        assert match_dict_keys(keys, "b\"b", delims=delims, extra_prefix=("foo",)) == ("\"", 2, ["bar"])
+        assert match(keys, "'", extra=("foo",)) == ("'", 1, ["bar", "oof"])
+        assert match(keys, '"', extra=("foo",)) == ('"', 1, ["bar", "oof"])
+        assert match(keys, "'o", extra=("foo",)) == ("'", 1, ["oof"])
+        assert match(keys, '"o', extra=("foo",)) == ('"', 1, ["oof"])
+        assert match(keys, "b'", extra=("foo",)) == ("'", 2, ["bar"])
+        assert match(keys, 'b"', extra=("foo",)) == ('"', 2, ["bar"])
+        assert match(keys, "b'b", extra=("foo",)) == ("'", 2, ["bar"])
+        assert match(keys, 'b"b', extra=("foo",)) == ('"', 2, ["bar"])
 
         # No Completion
-        assert match_dict_keys(keys, "'", delims=delims, extra_prefix=("no_foo",)) == ("'", 1, [])
-        assert match_dict_keys(keys, "'", delims=delims, extra_prefix=("fo",)) == ("'", 1, [])
+        assert match(keys, "'", extra=("no_foo",)) == ("'", 1, [])
+        assert match(keys, "'", extra=("fo",)) == ("'", 1, [])
+
+        keys = [("foo1", "foo2", "foo3", "foo4"), ("foo1", "foo2", "bar", "foo4")]
+        assert match(keys, "'foo", extra=("foo1",)) == ("'", 1, ["foo2"])
+        assert match(keys, "'foo", extra=("foo1", "foo2")) == ("'", 1, ["foo3"])
+        assert match(keys, "'foo", extra=("foo1", "foo2", "foo3")) == ("'", 1, ["foo4"])
+        assert match(keys, "'foo", extra=("foo1", "foo2", "foo3", "foo4")) == (
+            "'",
+            1,
+            [],
+        )
+
+        keys = [("foo", 1111), ("foo", "2222"), (3333, "bar"), (3333, 4444)]
+        assert match(keys, "'", extra=("foo",)) == ("'", 1, ["2222"])
+        assert match(keys, "", extra=("foo",)) == ("", 0, ["1111", "'2222'"])
+        assert match(keys, "'", extra=(3333,)) == ("'", 1, ["bar"])
+        assert match(keys, "", extra=(3333,)) == ("", 0, ["'bar'", "4444"])
+        assert match(keys, "'", extra=("3333",)) == ("'", 1, [])
+        assert match(keys, "33") == ("", 0, ["3333"])
+
+    def test_dict_key_completion_closures(self):
+        ip = get_ipython()
+        complete = ip.Completer.complete
+        ip.Completer.auto_close_dict_keys = True
+
+        ip.user_ns["d"] = {
+            # tuple only
+            ("aa", 11): None,
+            # tuple and non-tuple
+            ("bb", 22): None,
+            "bb": None,
+            # non-tuple only
+            "cc": None,
+            # numeric tuple only
+            (77, "x"): None,
+            # numeric tuple and non-tuple
+            (88, "y"): None,
+            88: None,
+            # numeric non-tuple only
+            99: None,
+        }
 
-        keys = [('foo1', 'foo2', 'foo3', 'foo4'), ('foo1', 'foo2', 'bar', 'foo4')]
-        assert match_dict_keys(keys, "'foo", delims=delims, extra_prefix=('foo1',)) == ("'", 1, ["foo2", "foo2"])
-        assert match_dict_keys(keys, "'foo", delims=delims, extra_prefix=('foo1', 'foo2')) == ("'", 1, ["foo3"])
-        assert match_dict_keys(keys, "'foo", delims=delims, extra_prefix=('foo1', 'foo2', 'foo3')) == ("'", 1, ["foo4"])
-        assert match_dict_keys(keys, "'foo", delims=delims, extra_prefix=('foo1', 'foo2', 'foo3', 'foo4')) == ("'", 1, [])
+        _, matches = complete(line_buffer="d[")
+        # should append `, ` if matches a tuple only
+        self.assertIn("'aa', ", matches)
+        # should not append anything if matches a tuple and an item
+        self.assertIn("'bb'", matches)
+        # should append `]` if matches and item only
+        self.assertIn("'cc']", matches)
+
+        # should append `, ` if matches a tuple only
+        self.assertIn("77, ", matches)
+        # should not append anything if matches a tuple and an item
+        self.assertIn("88", matches)
+        # should append `]` if matches and item only
+        self.assertIn("99]", matches)
+
+        _, matches = complete(line_buffer="d['aa', ")
+        # should restrict matches to those matching tuple prefix
+        self.assertIn("11]", matches)
+        self.assertNotIn("'bb'", matches)
+        self.assertNotIn("'bb', ", matches)
+        self.assertNotIn("'bb']", matches)
+        self.assertNotIn("'cc'", matches)
+        self.assertNotIn("'cc', ", matches)
+        self.assertNotIn("'cc']", matches)
+        ip.Completer.auto_close_dict_keys = False
 
     def test_dict_key_completion_string(self):
         """Test dictionary key completion for string keys"""
@@ -1038,6 +1142,35 b' class TestCompleter(unittest.TestCase):'
         self.assertNotIn("foo", matches)
         self.assertNotIn("bar", matches)
 
+    def test_dict_key_completion_numbers(self):
+        ip = get_ipython()
+        complete = ip.Completer.complete
+
+        ip.user_ns["d"] = {
+            0xDEADBEEF: None,  # 3735928559
+            1111: None,
+            1234: None,
+            "1999": None,
+            0b10101: None,  # 21
+            22: None,
+        }
+        _, matches = complete(line_buffer="d[1")
+        self.assertIn("1111", matches)
+        self.assertIn("1234", matches)
+        self.assertNotIn("1999", matches)
+        self.assertNotIn("'1999'", matches)
+
+        _, matches = complete(line_buffer="d[0xdead")
+        self.assertIn("0xdeadbeef", matches)
+
+        _, matches = complete(line_buffer="d[2")
+        self.assertIn("21", matches)
+        self.assertIn("22", matches)
+
+        _, matches = complete(line_buffer="d[0b101")
+        self.assertIn("0b10101", matches)
+        self.assertIn("0b10110", matches)
+
     def test_dict_key_completion_contexts(self):
         """Test expression contexts in which dict key completion occurs"""
         ip = get_ipython()
@@ -1050,6 +1183,7 b' class TestCompleter(unittest.TestCase):'
 
         ip.user_ns["C"] = C
         ip.user_ns["get"] = lambda: d
+        ip.user_ns["nested"] = {"x": d}
 
         def assert_no_completion(**kwargs):
             _, matches = complete(**kwargs)
@@ -1075,6 +1209,13 b' class TestCompleter(unittest.TestCase):'
         assert_completion(line_buffer="(d[")
         assert_completion(line_buffer="C.data[")
 
+        # nested dict completion
+        assert_completion(line_buffer="nested['x'][")
+
+        with evaluation_policy("minimal"):
+            with pytest.raises(AssertionError):
+                assert_completion(line_buffer="nested['x'][")
+
         # greedy flag
         def assert_completion(**kwargs):
             _, matches = complete(**kwargs)
@@ -1162,12 +1303,22 b' class TestCompleter(unittest.TestCase):'
         _, matches = complete(line_buffer="d['")
         self.assertIn("my_head", matches)
         self.assertIn("my_data", matches)
-        # complete on a nested level
-        with greedy_completion():
+
+        def completes_on_nested():
             ip.user_ns["d"] = numpy.zeros(2, dtype=dt)
             _, matches = complete(line_buffer="d[1]['my_head']['")
             self.assertTrue(any(["my_dt" in m for m in matches]))
             self.assertTrue(any(["my_df" in m for m in matches]))
+        # complete on a nested level
+        with greedy_completion():
+            completes_on_nested()
+
+        with evaluation_policy("limited"):
+            completes_on_nested()
+
+        with evaluation_policy("minimal"):
+            with pytest.raises(AssertionError):
+                completes_on_nested()
 
     @dec.skip_without("pandas")
     def test_dataframe_key_completion(self):
@@ -1180,6 +1331,17 b' class TestCompleter(unittest.TestCase):'
         _, matches = complete(line_buffer="d['")
         self.assertIn("hello", matches)
         self.assertIn("world", matches)
+        _, matches = complete(line_buffer="d.loc[:, '")
+        self.assertIn("hello", matches)
+        self.assertIn("world", matches)
+        _, matches = complete(line_buffer="d.loc[1:, '")
+        self.assertIn("hello", matches)
+        _, matches = complete(line_buffer="d.loc[1:1, '")
+        self.assertIn("hello", matches)
+        _, matches = complete(line_buffer="d.loc[1:1:-1, '")
+        self.assertIn("hello", matches)
+        _, matches = complete(line_buffer="d.loc[::, '")
+        self.assertIn("hello", matches)
 
     def test_dict_key_completion_invalids(self):
         """Smoke test cases dict key completion can't handle"""
@@ -1503,3 +1665,38 b' class TestCompleter(unittest.TestCase):'
             _(["completion_b"])
             a_matcher.matcher_priority = 3
             _(["completion_a"])
+
+
[e-mail](
+    "input, expected",
+    [
+        ["1.234", "1.234"],
+        # should match signed numbers
+        ["+1", "+1"],
+        ["-1", "-1"],
+        ["-1.0", "-1.0"],
+        ["-1.", "-1."],
+        ["+1.", "+1."],
+        [".1", ".1"],
+        # should not match non-numbers
+        ["1..", None],
+        ["..", None],
+        [".1.", None],
+        # should match after comma
+        [",1", "1"],
+        [", 1", "1"],
+        [", .1", ".1"],
+        [", +.1", "+.1"],
+        # should not match after trailing spaces
+        [".1 ", None],
+        # some complex cases
+        ["0b_0011_1111_0100_1110", "0b_0011_1111_0100_1110"],
+        ["0xdeadbeef", "0xdeadbeef"],
+        ["0b_1110_0101", "0b_1110_0101"],
+        # should not match if in an operation
+        ["1 + 1", None],
+        [", 1 + 1", None],
+    ],
+)
+def test_match_numeric_literal_for_dict_key(input, expected):
+    assert _match_number_in_dict_key_prefix(input) == expected

IPython/terminal/magics.py

@@ -147,7 +147,7 b' class TerminalMagics(Magics):'
 
         sentinel = opts.get('s', u'--')
         block = '\n'.join(get_pasted_lines(sentinel, quiet=quiet))
-        self.store_or_execute(block, name, store_history=False)
+        self.store_or_execute(block, name, store_history=True)
 
     @line_magic
     def paste(self, parameter_s=''):

setup.cfg

@@ -100,17 +100,12 b' exclude ='
     setupext
 
 [options.package_data]
+IPython = py.typed
 IPython.core = profile/README*
 IPython.core.tests = *.png, *.jpg, daft_extension/*.py
 IPython.lib.tests = *.wav
 IPython.testing.plugin = *.txt
 
-[options.entry_points]
-pygments.lexers =
-    ipythonconsole = IPython.lib.lexers:IPythonConsoleLexer
-    ipython = IPython.lib.lexers:IPythonLexer
-    ipython3 = IPython.lib.lexers:IPython3Lexer
-
 [velin]
 ignore_patterns =
    IPython/core/tests

setup.py

@@ -139,7 +139,15 b" setup_args['cmdclass'] = {"
     'install_scripts_sym': install_scripts_for_symlink,
     'unsymlink': unsymlink,
 }
-setup_args["entry_points"] = {"console_scripts": find_entry_points()}
+
+setup_args["entry_points"] = {
+    "console_scripts": find_entry_points(),
+    "pygments.lexers": [
+        "ipythonconsole = IPython.lib.lexers:IPythonConsoleLexer",
+        "ipython = IPython.lib.lexers:IPythonLexer",
+        "ipython3 = IPython.lib.lexers:IPython3Lexer",
+    ],
+}
 
 #---------------------------------------------------------------------------
 # Do the actual setup now