Merge branch 'main' into shaperilio/autoreload-verbosity

IPython/utils/docs.py

@@ -0,0 +1,3 b''
+import os
+
+GENERATING_DOCUMENTATION = os.environ.get("IN_SPHINX_RUN", None) == "True"

.github/workflows/python-package.yml

@@ -28,7 +28,7 b' jobs:'
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        pip install darker black==21.12b0
+        pip install darker==1.5.1 black==22.10.0
     - name: Lint with darker
       run: |
         darker -r 60625f241f298b5039cb2debc365db38aa7bb522 --check --diff . || (

.gitignore

@@ -24,7 +24,6 b' __pycache__'
 .cache
 .coverage
 *.swp
-.vscode
 .pytest_cache
 .python-version
 venv*/

IPython/core/compilerop.py

@@ -73,25 +73,6 b' class CachingCompiler(codeop.Compile):'
     def __init__(self):
         codeop.Compile.__init__(self)
 
-        # This is ugly, but it must be done this way to allow multiple
-        # simultaneous ipython instances to coexist.  Since Python itself
-        # directly accesses the data structures in the linecache module, and
-        # the cache therein is global, we must work with that data structure.
-        # We must hold a reference to the original checkcache routine and call
-        # that in our own check_cache() below, but the special IPython cache
-        # must also be shared by all IPython instances.  If we were to hold
-        # separate caches (one in each CachingCompiler instance), any call made
-        # by Python itself to linecache.checkcache() would obliterate the
-        # cached data from the other IPython instances.
-        if not hasattr(linecache, '_ipython_cache'):
-            linecache._ipython_cache = {}
-        if not hasattr(linecache, '_checkcache_ori'):
-            linecache._checkcache_ori = linecache.checkcache
-        # Now, we must monkeypatch the linecache directly so that parts of the
-        # stdlib that call it outside our control go through our codepath
-        # (otherwise we'd lose our tracebacks).
-        linecache.checkcache = check_linecache_ipython
-
         # Caching a dictionary { filename: execution_count } for nicely
         # rendered tracebacks. The filename corresponds to the filename
         # argument used for the builtins.compile function.
@@ -161,14 +142,24 b' class CachingCompiler(codeop.Compile):'
         # Save the execution count
         self._filename_map[name] = number
 
+        # Since Python 2.5, setting mtime to `None` means the lines will
+        # never be removed by `linecache.checkcache`.  This means all the
+        # monkeypatching has *never* been necessary, since this code was
+        # only added in 2010, at which point IPython had already stopped
+        # supporting Python 2.4.
+        #
+        # Note that `linecache.clearcache` and `linecache.updatecache` may
+        # still remove our code from the cache, but those show explicit
+        # intent, and we should not try to interfere.  Normally the former
+        # is never called except when out of memory, and the latter is only
+        # called for lines *not* in the cache.
         entry = (
             len(transformed_code),
-            time.time(),
+            None,
             [line + "\n" for line in transformed_code.splitlines()],
             name,
         )
         linecache.cache[name] = entry
-        linecache._ipython_cache[name] = entry
         return name
 
     @contextmanager
@@ -187,10 +178,22 b' class CachingCompiler(codeop.Compile):'
 
 
 def check_linecache_ipython(*args):
-    """Call linecache.checkcache() safely protecting our cached values.
+    """Deprecated since IPython 8.6.  Call linecache.checkcache() directly.
+
+    It was already not necessary to call this function directly.  If no
+    CachingCompiler had been created, this function would fail badly.  If
+    an instance had been created, this function would've been monkeypatched
+    into place.
+
+    As of IPython 8.6, the monkeypatching has gone away entirely.  But there
+    were still internal callers of this function, so maybe external callers
+    also existed?
     """
-    # First call the original checkcache as intended
-    linecache._checkcache_ori(*args)
-    # Then, update back the cache with our data, so that tracebacks related
-    # to our compiled codes can be produced.
-    linecache.cache.update(linecache._ipython_cache)
+    import warnings
+
+    warnings.warn(
+        "Deprecated Since IPython 8.6, Just call linecache.checkcache() directly.",
+        DeprecationWarning,
+        stacklevel=2,
+    )
+    linecache.checkcache()

IPython/core/completer.py

@@ -100,6 +100,73 b' option.'
 
 Be sure to update :any:`jedi` to the latest stable version or to try the
 current development version to get better completions.
+
+Matchers
+========
+
+All completions routines are implemented using unified *Matchers* API.
+The matchers API is provisional and subject to change without notice.
+
+The built-in matchers include:
+
+- :any:`IPCompleter.dict_key_matcher`:  dictionary key completions,
+- :any:`IPCompleter.magic_matcher`: completions for magics,
+- :any:`IPCompleter.unicode_name_matcher`,
+  :any:`IPCompleter.fwd_unicode_matcher`
+  and :any:`IPCompleter.latex_name_matcher`: see `Forward latex/unicode completion`_,
+- :any:`back_unicode_name_matcher` and :any:`back_latex_name_matcher`: see `Backward latex completion`_,
+- :any:`IPCompleter.file_matcher`: paths to files and directories,
+- :any:`IPCompleter.python_func_kw_matcher` - function keywords,
+- :any:`IPCompleter.python_matches` - globals and attributes (v1 API),
+- ``IPCompleter.jedi_matcher`` - static analysis with Jedi,
+- :any:`IPCompleter.custom_completer_matcher` - pluggable completer with a default
+  implementation in :any:`InteractiveShell` which uses IPython hooks system
+  (`complete_command`) with string dispatch (including regular expressions).
+  Differently to other matchers, ``custom_completer_matcher`` will not suppress
+  Jedi results to match behaviour in earlier IPython versions.
+
+Custom matchers can be added by appending to ``IPCompleter.custom_matchers`` list.
+
+Matcher API
+-----------
+
+Simplifying some details, the ``Matcher`` interface can described as
+
+.. code-block::
+
+    MatcherAPIv1 = Callable[[str], list[str]]
+    MatcherAPIv2 = Callable[[CompletionContext], SimpleMatcherResult]
+
+    Matcher = MatcherAPIv1 | MatcherAPIv2
+
+The ``MatcherAPIv1`` reflects the matcher API as available prior to IPython 8.6.0
+and remains supported as a simplest way for generating completions. This is also
+currently the only API supported by the IPython hooks system `complete_command`.
+
+To distinguish between matcher versions ``matcher_api_version`` attribute is used.
+More precisely, the API allows to omit ``matcher_api_version`` for v1 Matchers,
+and requires a literal ``2`` for v2 Matchers.
+
+Once the API stabilises future versions may relax the requirement for specifying
+``matcher_api_version`` by switching to :any:`functools.singledispatch`, therefore
+please do not rely on the presence of ``matcher_api_version`` for any purposes.
+
+Suppression of competing matchers
+---------------------------------
+
+By default results from all matchers are combined, in the order determined by
+their priority. Matchers can request to suppress results from subsequent
+matchers by setting ``suppress`` to ``True`` in the ``MatcherResult``.
+
+When multiple matchers simultaneously request surpression, the results from of
+the matcher with higher priority will be returned.
+
+Sometimes it is desirable to suppress most but not all other matchers;
+this can be achieved by adding a list of identifiers of matchers which
+should not be suppressed to ``MatcherResult`` under ``do_not_suppress`` key.
+
+The suppression behaviour can is user-configurable via
+:any:`IPCompleter.suppress_competing_matchers`.
 """
 
 
@@ -109,7 +176,7 b' current development version to get better completions.'
 # Some of this code originated from rlcompleter in the Python standard library
 # Copyright (C) 2001 Python Software Foundation, www.python.org
 
-
+from __future__ import annotations
 import builtins as builtin_mod
 import glob
 import inspect
@@ -124,9 +191,26 b' import unicodedata'
 import uuid
 import warnings
 from contextlib import contextmanager
+from dataclasses import dataclass
+from functools import cached_property, partial
 from importlib import import_module
 from types import SimpleNamespace
-from typing import Iterable, Iterator, List, Tuple, Union, Any, Sequence, Dict, NamedTuple, Pattern, Optional
+from typing import (
+    Iterable,
+    Iterator,
+    List,
+    Tuple,
+    Union,
+    Any,
+    Sequence,
+    Dict,
+    NamedTuple,
+    Pattern,
+    Optional,
+    TYPE_CHECKING,
+    Set,
+    Literal,
+)
 
 from IPython.core.error import TryNext
 from IPython.core.inputtransformer2 import ESC_MAGIC
@@ -134,10 +218,22 b' from IPython.core.latex_symbols import latex_symbols, reverse_latex_symbol'
 from IPython.core.oinspect import InspectColors
 from IPython.testing.skipdoctest import skip_doctest
 from IPython.utils import generics
+from IPython.utils.decorators import sphinx_options
 from IPython.utils.dir2 import dir2, get_real_method
+from IPython.utils.docs import GENERATING_DOCUMENTATION
 from IPython.utils.path import ensure_dir_exists
 from IPython.utils.process import arg_split
-from traitlets import Bool, Enum, Int, List as ListTrait, Unicode, default, observe
+from traitlets import (
+    Bool,
+    Enum,
+    Int,
+    List as ListTrait,
+    Unicode,
+    Dict as DictTrait,
+    Union as UnionTrait,
+    default,
+    observe,
+)
 from traitlets.config.configurable import Configurable
 
 import __main__
@@ -145,6 +241,7 b' import __main__'
 # skip module docstests
 __skip_doctest__ = True
 
+
 try:
     import jedi
     jedi.settings.case_insensitive_completion = False
@@ -153,6 +250,25 b' try:'
     JEDI_INSTALLED = True
 except ImportError:
     JEDI_INSTALLED = False
+
+
+if TYPE_CHECKING or GENERATING_DOCUMENTATION:
+    from typing import cast
+    from typing_extensions import TypedDict, NotRequired, Protocol, TypeAlias
+else:
+
+    def cast(obj, type_):
+        """Workaround for `TypeError: MatcherAPIv2() takes no arguments`"""
+        return obj
+
+    # do not require on runtime
+    NotRequired = Tuple  # requires Python >=3.11
+    TypedDict = Dict  # by extension of `NotRequired` requires 3.11 too
+    Protocol = object  # requires Python >=3.8
+    TypeAlias = Any  # requires Python >=3.10
+if GENERATING_DOCUMENTATION:
+    from typing import TypedDict
+
 #-----------------------------------------------------------------------------
 # Globals
 #-----------------------------------------------------------------------------
@@ -166,7 +282,7 b' except ImportError:'
 _UNICODE_RANGES = [(32, 0x3134b), (0xe0001, 0xe01f0)]
 
 # Public API
-__all__ = ['Completer','IPCompleter']
+__all__ = ["Completer", "IPCompleter"]
 
 if sys.platform == 'win32':
     PROTECTABLES = ' '
@@ -177,6 +293,8 b' else:'
 # may have trouble processing.
 MATCHES_LIMIT = 500
 
+# Completion type reported when no type can be inferred.
+_UNKNOWN_TYPE = "<unknown>"
 
 class ProvisionalCompleterWarning(FutureWarning):
     """
@@ -355,9 +473,12 b' class _FakeJediCompletion:'
         return '<Fake completion object jedi has crashed>'
 
 
+_JediCompletionLike = Union[jedi.api.Completion, _FakeJediCompletion]
+
+
 class Completion:
     """
-    Completion object used and return by IPython completers.
+    Completion object used and returned by IPython completers.
 
     .. warning::
 
@@ -417,6 +538,188 b' class Completion:'
         return hash((self.start, self.end, self.text))
 
 
+class SimpleCompletion:
+    """Completion item to be included in the dictionary returned by new-style Matcher (API v2).
+
+    .. warning::
+
+        Provisional
+
+        This class is used to describe the currently supported attributes of
+        simple completion items, and any additional implementation details
+        should not be relied on. Additional attributes may be included in
+        future versions, and meaning of text disambiguated from the current
+        dual meaning of "text to insert" and "text to used as a label".
+    """
+
+    __slots__ = ["text", "type"]
+
+    def __init__(self, text: str, *, type: str = None):
+        self.text = text
+        self.type = type
+
+    def __repr__(self):
+        return f"<SimpleCompletion text={self.text!r} type={self.type!r}>"
+
+
+class _MatcherResultBase(TypedDict):
+    """Definition of dictionary to be returned by new-style Matcher (API v2)."""
+
+    #: Suffix of the provided ``CompletionContext.token``, if not given defaults to full token.
+    matched_fragment: NotRequired[str]
+
+    #: Whether to suppress results from all other matchers (True), some
+    #: matchers (set of identifiers) or none (False); default is False.
+    suppress: NotRequired[Union[bool, Set[str]]]
+
+    #: Identifiers of matchers which should NOT be suppressed when this matcher
+    #: requests to suppress all other matchers; defaults to an empty set.
+    do_not_suppress: NotRequired[Set[str]]
+
+    #: Are completions already ordered and should be left as-is? default is False.
+    ordered: NotRequired[bool]
+
+
+@sphinx_options(show_inherited_members=True, exclude_inherited_from=["dict"])
+class SimpleMatcherResult(_MatcherResultBase, TypedDict):
+    """Result of new-style completion matcher."""
+
+    # note: TypedDict is added again to the inheritance chain
+    # in order to get __orig_bases__ for documentation
+
+    #: List of candidate completions
+    completions: Sequence[SimpleCompletion]
+
+
+class _JediMatcherResult(_MatcherResultBase):
+    """Matching result returned by Jedi (will be processed differently)"""
+
+    #: list of candidate completions
+    completions: Iterable[_JediCompletionLike]
+
+
+@dataclass
+class CompletionContext:
+    """Completion context provided as an argument to matchers in the Matcher API v2."""
+
+    # rationale: many legacy matchers relied on completer state (`self.text_until_cursor`)
+    # which was not explicitly visible as an argument of the matcher, making any refactor
+    # prone to errors; by explicitly passing `cursor_position` we can decouple the matchers
+    # from the completer, and make substituting them in sub-classes easier.
+
+    #: Relevant fragment of code directly preceding the cursor.
+    #: The extraction of token is implemented via splitter heuristic
+    #: (following readline behaviour for legacy reasons), which is user configurable
+    #: (by switching the greedy mode).
+    token: str
+
+    #: The full available content of the editor or buffer
+    full_text: str
+
+    #: Cursor position in the line (the same for ``full_text`` and ``text``).
+    cursor_position: int
+
+    #: Cursor line in ``full_text``.
+    cursor_line: int
+
+    #: The maximum number of completions that will be used downstream.
+    #: Matchers can use this information to abort early.
+    #: The built-in Jedi matcher is currently excepted from this limit.
+    # If not given, return all possible completions.
+    limit: Optional[int]
+
+    @cached_property
+    def text_until_cursor(self) -> str:
+        return self.line_with_cursor[: self.cursor_position]
+
+    @cached_property
+    def line_with_cursor(self) -> str:
+        return self.full_text.split("\n")[self.cursor_line]
+
+
+#: Matcher results for API v2.
+MatcherResult = Union[SimpleMatcherResult, _JediMatcherResult]
+
+
+class _MatcherAPIv1Base(Protocol):
+    def __call__(self, text: str) -> list[str]:
+        """Call signature."""
+
+
+class _MatcherAPIv1Total(_MatcherAPIv1Base, Protocol):
+    #: API version
+    matcher_api_version: Optional[Literal[1]]
+
+    def __call__(self, text: str) -> list[str]:
+        """Call signature."""
+
+
+#: Protocol describing Matcher API v1.
+MatcherAPIv1: TypeAlias = Union[_MatcherAPIv1Base, _MatcherAPIv1Total]
+
+
+class MatcherAPIv2(Protocol):
+    """Protocol describing Matcher API v2."""
+
+    #: API version
+    matcher_api_version: Literal[2] = 2
+
+    def __call__(self, context: CompletionContext) -> MatcherResult:
+        """Call signature."""
+
+
+Matcher: TypeAlias = Union[MatcherAPIv1, MatcherAPIv2]
+
+
+def completion_matcher(
+    *, priority: float = None, identifier: str = None, api_version: int = 1
+):
+    """Adds attributes describing the matcher.
+
+    Parameters
+    ----------
+    priority : Optional[float]
+        The priority of the matcher, determines the order of execution of matchers.
+        Higher priority means that the matcher will be executed first. Defaults to 0.
+    identifier : Optional[str]
+        identifier of the matcher allowing users to modify the behaviour via traitlets,
+        and also used to for debugging (will be passed as ``origin`` with the completions).
+        Defaults to matcher function ``__qualname__``.
+    api_version: Optional[int]
+        version of the Matcher API used by this matcher.
+        Currently supported values are 1 and 2.
+        Defaults to 1.
+    """
+
+    def wrapper(func: Matcher):
+        func.matcher_priority = priority or 0
+        func.matcher_identifier = identifier or func.__qualname__
+        func.matcher_api_version = api_version
+        if TYPE_CHECKING:
+            if api_version == 1:
+                func = cast(func, MatcherAPIv1)
+            elif api_version == 2:
+                func = cast(func, MatcherAPIv2)
+        return func
+
+    return wrapper
+
+
+def _get_matcher_priority(matcher: Matcher):
+    return getattr(matcher, "matcher_priority", 0)
+
+
+def _get_matcher_id(matcher: Matcher):
+    return getattr(matcher, "matcher_identifier", matcher.__qualname__)
+
+
+def _get_matcher_api_version(matcher):
+    return getattr(matcher, "matcher_api_version", 1)
+
+
+context_matcher = partial(completion_matcher, api_version=2)
+
+
 _IC = Iterable[Completion]
 
 
@@ -924,6 +1227,19 b' def _safe_isinstance(obj, module, class_name):'
     return (module in sys.modules and
             isinstance(obj, getattr(import_module(module), class_name)))
 
+
+@context_matcher()
+def back_unicode_name_matcher(context: CompletionContext):
+    """Match Unicode characters back to Unicode name
+
+    Same as :any:`back_unicode_name_matches`, but adopted to new Matcher API.
+    """
+    fragment, matches = back_unicode_name_matches(context.text_until_cursor)
+    return _convert_matcher_v1_result_to_v2(
+        matches, type="unicode", fragment=fragment, suppress_if_matches=True
+    )
+
+
 def back_unicode_name_matches(text:str) -> Tuple[str, Sequence[str]]:
     """Match Unicode characters back to Unicode name
 
@@ -934,6 +1250,9 b' def back_unicode_name_matches(text:str) -> Tuple[str, Sequence[str]]:'
 
     This will not either back-complete standard sequences like \\n, \\b ...
 
+    .. deprecated:: 8.6
+        You can use :meth:`back_unicode_name_matcher` instead.
+
     Returns
     =======
 
@@ -943,7 +1262,6 b' def back_unicode_name_matches(text:str) -> Tuple[str, Sequence[str]]:'
         empty string,
     - a sequence (of 1), name for the match Unicode character, preceded by
         backslash, or empty if no match.
-    
     """
     if len(text)<2:
         return '', ()
@@ -963,11 +1281,26 b' def back_unicode_name_matches(text:str) -> Tuple[str, Sequence[str]]:'
         pass
     return '', ()
 
+
+@context_matcher()
+def back_latex_name_matcher(context: CompletionContext):
+    """Match latex characters back to unicode name
+
+    Same as :any:`back_latex_name_matches`, but adopted to new Matcher API.
+    """
+    fragment, matches = back_latex_name_matches(context.text_until_cursor)
+    return _convert_matcher_v1_result_to_v2(
+        matches, type="latex", fragment=fragment, suppress_if_matches=True
+    )
+
+
 def back_latex_name_matches(text:str) -> Tuple[str, Sequence[str]] :
     """Match latex characters back to unicode name
 
     This does ``\\ℵ`` -> ``\\aleph``
 
+    .. deprecated:: 8.6
+        You can use :meth:`back_latex_name_matcher` instead.
     """
     if len(text)<2:
         return '', ()
@@ -1042,11 +1375,23 b' def _make_signature(completion)-> str:'
                                           for p in signature.defined_names()) if f])
 
 
-class _CompleteResult(NamedTuple):
-    matched_text : str
-    matches: Sequence[str]
-    matches_origin: Sequence[str]
-    jedi_matches: Any
+_CompleteResult = Dict[str, MatcherResult]
+
+
+def _convert_matcher_v1_result_to_v2(
+    matches: Sequence[str],
+    type: str,
+    fragment: str = None,
+    suppress_if_matches: bool = False,
+) -> SimpleMatcherResult:
+    """Utility to help with transition"""
+    result = {
+        "completions": [SimpleCompletion(text=match, type=type) for match in matches],
+        "suppress": (True if matches else False) if suppress_if_matches else False,
+    }
+    if fragment is not None:
+        result["matched_fragment"] = fragment
+    return result
 
 
 class IPCompleter(Completer):
@@ -1062,17 +1407,59 b' class IPCompleter(Completer):'
         else:
             self.splitter.delims = DELIMS
 
-    dict_keys_only = Bool(False,
-        help="""Whether to show dict key matches only""")
+    dict_keys_only = Bool(
+        False,
+        help="""
+        Whether to show dict key matches only.
+        
+        (disables all matchers except for `IPCompleter.dict_key_matcher`).
+        """,
+    )
+
+    suppress_competing_matchers = UnionTrait(
+        [Bool(allow_none=True), DictTrait(Bool(None, allow_none=True))],
+        default_value=None,
+        help="""
+        Whether to suppress completions from other *Matchers*.
+
+        When set to ``None`` (default) the matchers will attempt to auto-detect
+        whether suppression of other matchers is desirable. For example, at
+        the beginning of a line followed by `%` we expect a magic completion
+        to be the only applicable option, and after ``my_dict['`` we usually
+        expect a completion with an existing dictionary key.
+
+        If you want to disable this heuristic and see completions from all matchers,
+        set ``IPCompleter.suppress_competing_matchers = False``.
+        To disable the heuristic for specific matchers provide a dictionary mapping:
+        ``IPCompleter.suppress_competing_matchers = {'IPCompleter.dict_key_matcher': False}``.
+
+        Set ``IPCompleter.suppress_competing_matchers = True`` to limit
+        completions to the set of matchers with the highest priority;
+        this is equivalent to ``IPCompleter.merge_completions`` and
+        can be beneficial for performance, but will sometimes omit relevant
+        candidates from matchers further down the priority list.
+        """,
+    ).tag(config=True)
 
-    merge_completions = Bool(True,
+    merge_completions = Bool(
+        True,
         help="""Whether to merge completion results into a single list
 
         If False, only the completion results from the first non-empty
         completer will be returned.
-        """
+        
+        As of version 8.6.0, setting the value to ``False`` is an alias for:
+        ``IPCompleter.suppress_competing_matchers = True.``.
+        """,
+    ).tag(config=True)
+
+    disable_matchers = ListTrait(
+        Unicode(), help="""List of matchers to disable."""
     ).tag(config=True)
-    omit__names = Enum((0,1,2), default_value=2,
+
+    omit__names = Enum(
+        (0, 1, 2),
+        default_value=2,
         help="""Instruct the completer to omit private method names
 
         Specifically, when completing on ``object.<tab>``.
@@ -1148,7 +1535,7 b' class IPCompleter(Completer):'
             namespace=namespace,
             global_namespace=global_namespace,
             config=config,
-            **kwargs
+            **kwargs,
         )
 
         # List where completion matches will be stored
@@ -1177,8 +1564,8 b' class IPCompleter(Completer):'
         #= re.compile(r'[\s|\[]*(\w+)(?:\s*=?\s*.*)')
 
         self.magic_arg_matchers = [
-            self.magic_config_matches,
-            self.magic_color_matches,
+            self.magic_config_matcher,
+            self.magic_color_matcher,
         ]
 
         # This is set externally by InteractiveShell
@@ -1190,27 +1577,50 b' class IPCompleter(Completer):'
         # attribute through the `@unicode_names` property.
         self._unicode_names = None
 
+        self._backslash_combining_matchers = [
+            self.latex_name_matcher,
+            self.unicode_name_matcher,
+            back_latex_name_matcher,
+            back_unicode_name_matcher,
+            self.fwd_unicode_matcher,
+        ]
+
+        if not self.backslash_combining_completions:
+            for matcher in self._backslash_combining_matchers:
+                self.disable_matchers.append(matcher.matcher_identifier)
+
+        if not self.merge_completions:
+            self.suppress_competing_matchers = True
+
     @property
-    def matchers(self) -> List[Any]:
+    def matchers(self) -> List[Matcher]:
         """All active matcher routines for completion"""
         if self.dict_keys_only:
-            return [self.dict_key_matches]
+            return [self.dict_key_matcher]
 
         if self.use_jedi:
             return [
                 *self.custom_matchers,
-                self.dict_key_matches,
-                self.file_matches,
-                self.magic_matches,
+                *self._backslash_combining_matchers,
+                *self.magic_arg_matchers,
+                self.custom_completer_matcher,
+                self.magic_matcher,
+                self._jedi_matcher,
+                self.dict_key_matcher,
+                self.file_matcher,
             ]
         else:
             return [
                 *self.custom_matchers,
-                self.dict_key_matches,
+                *self._backslash_combining_matchers,
+                *self.magic_arg_matchers,
+                self.custom_completer_matcher,
+                self.dict_key_matcher,
+                # TODO: convert python_matches to v2 API
+                self.magic_matcher,
                 self.python_matches,
-                self.file_matches,
-                self.magic_matches,
-                self.python_func_kw_matches,
+                self.file_matcher,
+                self.python_func_kw_matcher,
             ]
 
     def all_completions(self, text:str) -> List[str]:
@@ -1231,6 +1641,14 b' class IPCompleter(Completer):'
         return [f.replace("\\","/")
                 for f in self.glob("%s*" % text)]
 
+    @context_matcher()
+    def file_matcher(self, context: CompletionContext) -> SimpleMatcherResult:
+        """Same as :any:`file_matches`, but adopted to new Matcher API."""
+        matches = self.file_matches(context.token)
+        # TODO: add a heuristic for suppressing (e.g. if it has OS-specific delimiter,
+        #  starts with `/home/`, `C:\`, etc)
+        return _convert_matcher_v1_result_to_v2(matches, type="path")
+
     def file_matches(self, text:str)->List[str]:
         """Match filenames, expanding ~USER type strings.
 
@@ -1243,7 +1661,11 b' class IPCompleter(Completer):'
         only the parts after what's already been typed (instead of the
         full completions, as is normally done).  I don't think with the
         current (as of Python 2.3) Python readline it's possible to do
-        better."""
+        better.
+
+        .. deprecated:: 8.6
+            You can use :meth:`file_matcher` instead.
+        """
 
         # chars that require escaping with backslash - i.e. chars
         # that readline treats incorrectly as delimiters, but we
@@ -1313,8 +1735,22 b' class IPCompleter(Completer):'
         # Mark directories in input list by appending '/' to their names.
         return [x+'/' if os.path.isdir(x) else x for x in matches]
 
+    @context_matcher()
+    def magic_matcher(self, context: CompletionContext) -> SimpleMatcherResult:
+        """Match magics."""
+        text = context.token
+        matches = self.magic_matches(text)
+        result = _convert_matcher_v1_result_to_v2(matches, type="magic")
+        is_magic_prefix = len(text) > 0 and text[0] == "%"
+        result["suppress"] = is_magic_prefix and bool(result["completions"])
+        return result
+
     def magic_matches(self, text:str):
-        """Match magics"""
+        """Match magics.
+
+        .. deprecated:: 8.6
+            You can use :meth:`magic_matcher` instead.
+        """
         # Get all shell magics now rather than statically, so magics loaded at
         # runtime show up too.
         lsm = self.shell.magics_manager.lsmagic()
@@ -1355,8 +1791,19 b' class IPCompleter(Completer):'
 
         return comp
 
+    @context_matcher()
+    def magic_config_matcher(self, context: CompletionContext) -> SimpleMatcherResult:
+        """Match class names and attributes for %config magic."""
+        # NOTE: uses `line_buffer` equivalent for compatibility
+        matches = self.magic_config_matches(context.line_with_cursor)
+        return _convert_matcher_v1_result_to_v2(matches, type="param")
+
     def magic_config_matches(self, text:str) -> List[str]:
-        """ Match class names and attributes for %config magic """
+        """Match class names and attributes for %config magic.
+
+        .. deprecated:: 8.6
+            You can use :meth:`magic_config_matcher` instead.
+        """
         texts = text.strip().split()
 
         if len(texts) > 0 and (texts[0] == 'config' or texts[0] == '%config'):
@@ -1390,8 +1837,19 b' class IPCompleter(Completer):'
                          if attr.startswith(texts[1]) ]
         return []
 
+    @context_matcher()
+    def magic_color_matcher(self, context: CompletionContext) -> SimpleMatcherResult:
+        """Match color schemes for %colors magic."""
+        # NOTE: uses `line_buffer` equivalent for compatibility
+        matches = self.magic_color_matches(context.line_with_cursor)
+        return _convert_matcher_v1_result_to_v2(matches, type="param")
+
     def magic_color_matches(self, text:str) -> List[str] :
-        """ Match color schemes for %colors magic"""
+        """Match color schemes for %colors magic.
+
+        .. deprecated:: 8.6
+            You can use :meth:`magic_color_matcher` instead.
+        """
         texts = text.split()
         if text.endswith(' '):
             # .split() strips off the trailing whitespace. Add '' back
@@ -1404,9 +1862,24 b' class IPCompleter(Completer):'
                      if color.startswith(prefix) ]
         return []
 
-    def _jedi_matches(self, cursor_column:int, cursor_line:int, text:str) -> Iterable[Any]:
+    @context_matcher(identifier="IPCompleter.jedi_matcher")
+    def _jedi_matcher(self, context: CompletionContext) -> _JediMatcherResult:
+        matches = self._jedi_matches(
+            cursor_column=context.cursor_position,
+            cursor_line=context.cursor_line,
+            text=context.full_text,
+        )
+        return {
+            "completions": matches,
+            # static analysis should not suppress other matchers
+            "suppress": False,
+        }
+
+    def _jedi_matches(
+        self, cursor_column: int, cursor_line: int, text: str
+    ) -> Iterable[_JediCompletionLike]:
         """
-        Return a list of :any:`jedi.api.Completions` object from a ``text`` and
+        Return a list of :any:`jedi.api.Completion`s object from a ``text`` and
         cursor position.
 
         Parameters
@@ -1422,6 +1895,9 b' class IPCompleter(Completer):'
         -----
         If ``IPCompleter.debug`` is ``True`` may return a :any:`_FakeJediCompletion`
         object containing a string with the Jedi debug information attached.
+
+        .. deprecated:: 8.6
+            You can use :meth:`_jedi_matcher` instead.
         """
         namespaces = [self.namespace]
         if self.global_namespace is not None:
@@ -1558,8 +2034,18 b' class IPCompleter(Completer):'
 
         return list(set(ret))
 
+    @context_matcher()
+    def python_func_kw_matcher(self, context: CompletionContext) -> SimpleMatcherResult:
+        """Match named parameters (kwargs) of the last open function."""
+        matches = self.python_func_kw_matches(context.token)
+        return _convert_matcher_v1_result_to_v2(matches, type="param")
+
     def python_func_kw_matches(self, text):
-        """Match named parameters (kwargs) of the last open function"""
+        """Match named parameters (kwargs) of the last open function.
+
+        .. deprecated:: 8.6
+            You can use :meth:`python_func_kw_matcher` instead.
+        """
 
         if "." in text: # a parameter cannot be dotted
             return []
@@ -1654,9 +2140,20 b' class IPCompleter(Completer):'
             return obj.dtype.names or []
         return []
 
+    @context_matcher()
+    def dict_key_matcher(self, context: CompletionContext) -> SimpleMatcherResult:
+        """Match string keys in a dictionary, after e.g. ``foo[``."""
+        matches = self.dict_key_matches(context.token)
+        return _convert_matcher_v1_result_to_v2(
+            matches, type="dict key", suppress_if_matches=True
+        )
+
     def dict_key_matches(self, text:str) -> List[str]:
-        "Match string keys in a dictionary, after e.g. 'foo[' "
+        """Match string keys in a dictionary, after e.g. ``foo[``.
 
+        .. deprecated:: 8.6
+            You can use :meth:`dict_key_matcher` instead.
+        """
 
         if self.__dict_key_regexps is not None:
             regexps = self.__dict_key_regexps
@@ -1758,6 +2255,14 b' class IPCompleter(Completer):'
 
         return [leading + k + suf for k in matches]
 
+    @context_matcher()
+    def unicode_name_matcher(self, context: CompletionContext):
+        """Same as :any:`unicode_name_matches`, but adopted to new Matcher API."""
+        fragment, matches = self.unicode_name_matches(context.text_until_cursor)
+        return _convert_matcher_v1_result_to_v2(
+            matches, type="unicode", fragment=fragment, suppress_if_matches=True
+        )
+
     @staticmethod
     def unicode_name_matches(text:str) -> Tuple[str, List[str]] :
         """Match Latex-like syntax for unicode characters base
@@ -1780,11 +2285,24 b' class IPCompleter(Completer):'
                 pass
         return '', []
 
+    @context_matcher()
+    def latex_name_matcher(self, context: CompletionContext):
+        """Match Latex syntax for unicode characters.
+
+        This does both ``\\alp`` -> ``\\alpha`` and ``\\alpha`` -> ``α``
+        """
+        fragment, matches = self.latex_matches(context.text_until_cursor)
+        return _convert_matcher_v1_result_to_v2(
+            matches, type="latex", fragment=fragment, suppress_if_matches=True
+        )
 
     def latex_matches(self, text:str) -> Tuple[str, Sequence[str]]:
         """Match Latex syntax for unicode characters.
 
         This does both ``\\alp`` -> ``\\alpha`` and ``\\alpha`` -> ``α``
+
+        .. deprecated:: 8.6
+            You can use :meth:`latex_name_matcher` instead.
         """
         slashpos = text.rfind('\\')
         if slashpos > -1:
@@ -1801,7 +2319,25 b' class IPCompleter(Completer):'
                     return s, matches
         return '', ()
 
+    @context_matcher()
+    def custom_completer_matcher(self, context):
+        """Dispatch custom completer.
+
+        If a match is found, suppresses all other matchers except for Jedi.
+        """
+        matches = self.dispatch_custom_completer(context.token) or []
+        result = _convert_matcher_v1_result_to_v2(
+            matches, type=_UNKNOWN_TYPE, suppress_if_matches=True
+        )
+        result["ordered"] = True
+        result["do_not_suppress"] = {_get_matcher_id(self._jedi_matcher)}
+        return result
+
     def dispatch_custom_completer(self, text):
+        """
+        .. deprecated:: 8.6
+            You can use :meth:`custom_completer_matcher` instead.
+        """
         if not self.custom_completers:
             return
 
@@ -1955,12 +2491,25 b' class IPCompleter(Completer):'
         """
         deadline = time.monotonic() + _timeout
 
-
         before = full_text[:offset]
         cursor_line, cursor_column = position_to_cursor(full_text, offset)
 
-        matched_text, matches, matches_origin, jedi_matches = self._complete(
-            full_text=full_text, cursor_line=cursor_line, cursor_pos=cursor_column)
+        jedi_matcher_id = _get_matcher_id(self._jedi_matcher)
+
+        results = self._complete(
+            full_text=full_text, cursor_line=cursor_line, cursor_pos=cursor_column
+        )
+        non_jedi_results: Dict[str, SimpleMatcherResult] = {
+            identifier: result
+            for identifier, result in results.items()
+            if identifier != jedi_matcher_id
+        }
+
+        jedi_matches = (
+            cast(results[jedi_matcher_id], _JediMatcherResult)["completions"]
+            if jedi_matcher_id in results
+            else ()
+        )
 
         iter_jm = iter(jedi_matches)
         if _timeout:
@@ -1988,28 +2537,57 b' class IPCompleter(Completer):'
 
         for jm in iter_jm:
             delta = len(jm.name_with_symbols) - len(jm.complete)
-            yield Completion(start=offset - delta,
+            yield Completion(
+                start=offset - delta,
                              end=offset,
                              text=jm.name_with_symbols,
-                             type='<unknown>',  # don't compute type for speed
-                             _origin='jedi',
-                             signature='')
-
-
-        start_offset = before.rfind(matched_text)
+                type=_UNKNOWN_TYPE,  # don't compute type for speed
+                _origin="jedi",
+                signature="",
+            )
 
         # TODO:
         # Suppress this, right now just for debug.
-        if jedi_matches and matches and self.debug:
-            yield Completion(start=start_offset, end=offset, text='--jedi/ipython--',
-                             _origin='debug', type='none', signature='')
+        if jedi_matches and non_jedi_results and self.debug:
+            some_start_offset = before.rfind(
+                next(iter(non_jedi_results.values()))["matched_fragment"]
+            )
+            yield Completion(
+                start=some_start_offset,
+                end=offset,
+                text="--jedi/ipython--",
+                _origin="debug",
+                type="none",
+                signature="",
+            )
+
+        ordered = []
+        sortable = []
+
+        for origin, result in non_jedi_results.items():
+            matched_text = result["matched_fragment"]
+            start_offset = before.rfind(matched_text)
+            is_ordered = result.get("ordered", False)
+            container = ordered if is_ordered else sortable
 
         # I'm unsure if this is always true, so let's assert and see if it
         # crash
         assert before.endswith(matched_text)
-        for m, t in zip(matches, matches_origin):
-            yield Completion(start=start_offset, end=offset, text=m, _origin=t, signature='', type='<unknown>')
 
+            for simple_completion in result["completions"]:
+                completion = Completion(
+                    start=start_offset,
+                    end=offset,
+                    text=simple_completion.text,
+                    _origin=origin,
+                    signature="",
+                    type=simple_completion.type or _UNKNOWN_TYPE,
+                )
+                container.append(completion)
+
+        yield from list(self._deduplicate(ordered + self._sort(sortable)))[
+            :MATCHES_LIMIT
+        ]
 
     def complete(self, text=None, line_buffer=None, cursor_pos=None) -> Tuple[str, Sequence[str]]:
         """Find completions for the given text and line context.
@@ -2050,7 +2628,56 b' class IPCompleter(Completer):'
                       PendingDeprecationWarning)
         # potential todo, FOLD the 3rd throw away argument of _complete
         # into the first 2 one.
-        return self._complete(line_buffer=line_buffer, cursor_pos=cursor_pos, text=text, cursor_line=0)[:2]
+        # TODO: Q: does the above refer to jedi completions (i.e. 0-indexed?)
+        # TODO: should we deprecate now, or does it stay?
+
+        results = self._complete(
+            line_buffer=line_buffer, cursor_pos=cursor_pos, text=text, cursor_line=0
+        )
+
+        jedi_matcher_id = _get_matcher_id(self._jedi_matcher)
+
+        return self._arrange_and_extract(
+            results,
+            # TODO: can we confirm that excluding Jedi here was a deliberate choice in previous version?
+            skip_matchers={jedi_matcher_id},
+            # this API does not support different start/end positions (fragments of token).
+            abort_if_offset_changes=True,
+        )
+
+    def _arrange_and_extract(
+        self,
+        results: Dict[str, MatcherResult],
+        skip_matchers: Set[str],
+        abort_if_offset_changes: bool,
+    ):
+
+        sortable = []
+        ordered = []
+        most_recent_fragment = None
+        for identifier, result in results.items():
+            if identifier in skip_matchers:
+                continue
+            if not result["completions"]:
+                continue
+            if not most_recent_fragment:
+                most_recent_fragment = result["matched_fragment"]
+            if (
+                abort_if_offset_changes
+                and result["matched_fragment"] != most_recent_fragment
+            ):
+                break
+            if result.get("ordered", False):
+                ordered.extend(result["completions"])
+            else:
+                sortable.extend(result["completions"])
+
+        if not most_recent_fragment:
+            most_recent_fragment = ""  # to satisfy typechecker (and just in case)
+
+        return most_recent_fragment, [
+            m.text for m in self._deduplicate(ordered + self._sort(sortable))
+        ]
 
     def _complete(self, *, cursor_line, cursor_pos, line_buffer=None, text=None,
                   full_text=None) -> _CompleteResult:
@@ -2085,14 +2712,10 b' class IPCompleter(Completer):'
 
         Returns
         -------
-        A tuple of N elements which are (likely):
-            matched_text: ? the text that the complete matched
-            matches: list of completions ?
-            matches_origin: ? list same length as matches, and where each completion came from
-            jedi_matches: list of Jedi matches, have it's own structure.
+        An ordered dictionary where keys are identifiers of completion
+        matchers and values are ``MatcherResult``s.
         """
 
-
         # if the cursor position isn't given, the only sane assumption we can
         # make is that it's at the end of the line (the common case)
         if cursor_pos is None:
@@ -2105,97 +2728,155 b' class IPCompleter(Completer):'
         if (not line_buffer) and full_text:
             line_buffer = full_text.split('\n')[cursor_line]
         if not text: # issue #11508: check line_buffer before calling split_line
-            text = self.splitter.split_line(line_buffer, cursor_pos)  if line_buffer else ''
-
-        if self.backslash_combining_completions:
-            # allow deactivation of these on windows.
-            base_text = text if not line_buffer else line_buffer[:cursor_pos]
-
-            for meth in (self.latex_matches,
-                         self.unicode_name_matches,
-                         back_latex_name_matches,
-                         back_unicode_name_matches,
-                         self.fwd_unicode_match):
-                name_text, name_matches = meth(base_text)
-                if name_text:
-                    return _CompleteResult(name_text, name_matches[:MATCHES_LIMIT], \
-                           [meth.__qualname__]*min(len(name_matches), MATCHES_LIMIT), ())
-
+            text = (
+                self.splitter.split_line(line_buffer, cursor_pos) if line_buffer else ""
+            )
 
         # If no line buffer is given, assume the input text is all there was
         if line_buffer is None:
             line_buffer = text
 
+        # deprecated - do not use `line_buffer` in new code.
         self.line_buffer = line_buffer
         self.text_until_cursor = self.line_buffer[:cursor_pos]
 
-        # Do magic arg matches
-        for matcher in self.magic_arg_matchers:
-            matches = list(matcher(line_buffer))[:MATCHES_LIMIT]
-            if matches:
-                origins = [matcher.__qualname__] * len(matches)
-                return _CompleteResult(text, matches, origins, ())
+        if not full_text:
+            full_text = line_buffer
+
+        context = CompletionContext(
+            full_text=full_text,
+            cursor_position=cursor_pos,
+            cursor_line=cursor_line,
+            token=text,
+            limit=MATCHES_LIMIT,
+        )
 
         # Start with a clean slate of completions
-        matches = []
+        results = {}
 
-        # FIXME: we should extend our api to return a dict with completions for
-        # different types of objects.  The rlcomplete() method could then
-        # simply collapse the dict into a list for readline, but we'd have
-        # richer completion semantics in other environments.
-        is_magic_prefix = len(text) > 0 and text[0] == "%"
-        completions: Iterable[Any] = []
-        if self.use_jedi and not is_magic_prefix:
-            if not full_text:
-                full_text = line_buffer
-            completions = self._jedi_matches(
-                cursor_pos, cursor_line, full_text)
+        jedi_matcher_id = _get_matcher_id(self._jedi_matcher)
+
+        suppressed_matchers = set()
+
+        matchers = {
+            _get_matcher_id(matcher): matcher
+            for matcher in sorted(
+                self.matchers, key=_get_matcher_priority, reverse=True
+            )
+        }
+
+        for matcher_id, matcher in matchers.items():
+            api_version = _get_matcher_api_version(matcher)
+            matcher_id = _get_matcher_id(matcher)
+
+            if matcher_id in self.disable_matchers:
+                continue
+
+            if matcher_id in results:
+                warnings.warn(f"Duplicate matcher ID: {matcher_id}.")
+
+            if matcher_id in suppressed_matchers:
+                continue
 
-        if self.merge_completions:
-            matches = []
-            for matcher in self.matchers:
                 try:
-                    matches.extend([(m, matcher.__qualname__)
-                                    for m in matcher(text)])
+                if api_version == 1:
+                    result = _convert_matcher_v1_result_to_v2(
+                        matcher(text), type=_UNKNOWN_TYPE
+                    )
+                elif api_version == 2:
+                    result = cast(matcher, MatcherAPIv2)(context)
+                else:
+                    raise ValueError(f"Unsupported API version {api_version}")
                 except:
                     # Show the ugly traceback if the matcher causes an
                     # exception, but do NOT crash the kernel!
                     sys.excepthook(*sys.exc_info())
-        else:
-            for matcher in self.matchers:
-                matches = [(m, matcher.__qualname__)
-                            for m in matcher(text)]
-                if matches:
-                    break
+                continue
                     
-        seen = set()
-        filtered_matches = set()
-        for m in matches:
-            t, c = m
-            if t not in seen:
-                filtered_matches.add(m)
-                seen.add(t)
+            # set default value for matched fragment if suffix was not selected.
+            result["matched_fragment"] = result.get("matched_fragment", context.token)
 
-        _filtered_matches = sorted(filtered_matches, key=lambda x: completions_sorting_key(x[0]))
+            if not suppressed_matchers:
+                suppression_recommended = result.get("suppress", False)
 
-        custom_res = [(m, 'custom') for m in self.dispatch_custom_completer(text) or []]
+                suppression_config = (
+                    self.suppress_competing_matchers.get(matcher_id, None)
+                    if isinstance(self.suppress_competing_matchers, dict)
+                    else self.suppress_competing_matchers
+                )
+                should_suppress = (
+                    (suppression_config is True)
+                    or (suppression_recommended and (suppression_config is not False))
+                ) and len(result["completions"])
         
-        _filtered_matches = custom_res or _filtered_matches
+                if should_suppress:
+                    suppression_exceptions = result.get("do_not_suppress", set())
+                    try:
+                        to_suppress = set(suppression_recommended)
+                    except TypeError:
+                        to_suppress = set(matchers)
+                    suppressed_matchers = to_suppress - suppression_exceptions
+
+                    new_results = {}
+                    for previous_matcher_id, previous_result in results.items():
+                        if previous_matcher_id not in suppressed_matchers:
+                            new_results[previous_matcher_id] = previous_result
+                    results = new_results
+
+            results[matcher_id] = result
+
+        _, matches = self._arrange_and_extract(
+            results,
+            # TODO Jedi completions non included in legacy stateful API; was this deliberate or omission?
+            #  if it was omission, we can remove the filtering step, otherwise remove this comment.
+            skip_matchers={jedi_matcher_id},
+            abort_if_offset_changes=False,
+        )
         
-        _filtered_matches = _filtered_matches[:MATCHES_LIMIT]
-        _matches = [m[0] for m in _filtered_matches]
-        origins = [m[1] for m in _filtered_matches]
+        # populate legacy stateful API
+        self.matches = matches
 
-        self.matches = _matches
+        return results
 
-        return _CompleteResult(text, _matches, origins, completions)
+    @staticmethod
+    def _deduplicate(
+        matches: Sequence[SimpleCompletion],
+    ) -> Iterable[SimpleCompletion]:
+        filtered_matches = {}
+        for match in matches:
+            text = match.text
+            if (
+                text not in filtered_matches
+                or filtered_matches[text].type == _UNKNOWN_TYPE
+            ):
+                filtered_matches[text] = match
+
+        return filtered_matches.values()
+
+    @staticmethod
+    def _sort(matches: Sequence[SimpleCompletion]):
+        return sorted(matches, key=lambda x: completions_sorting_key(x.text))
+
+    @context_matcher()
+    def fwd_unicode_matcher(self, context: CompletionContext):
+        """Same as :any:`fwd_unicode_match`, but adopted to new Matcher API."""
+        # TODO: use `context.limit` to terminate early once we matched the maximum
+        #  number that will be used downstream; can be added as an optional to
+        #  `fwd_unicode_match(text: str, limit: int = None)` or we could re-implement here.
+        fragment, matches = self.fwd_unicode_match(context.text_until_cursor)
+        return _convert_matcher_v1_result_to_v2(
+            matches, type="unicode", fragment=fragment, suppress_if_matches=True
+        )
         
     def fwd_unicode_match(self, text:str) -> Tuple[str, Sequence[str]]:
         """
         Forward match a string starting with a backslash with a list of
         potential Unicode completions.
 
-        Will compute list list of Unicode character names on first call and cache it.
+        Will compute list of Unicode character names on first call and cache it.
+
+        .. deprecated:: 8.6
+            You can use :meth:`fwd_unicode_matcher` instead.
 
         Returns
         -------

IPython/core/display.py

@@ -389,7 +389,19 b' class DisplayObject(object):'
 
 
 class TextDisplayObject(DisplayObject):
-    """Validate that display data is text"""
+    """Create a text display object given raw data.
+
+    Parameters
+    ----------
+    data : str or unicode
+        The raw data or a URL or file to load the data from.
+    url : unicode
+        A URL to download the data from.
+    filename : unicode
+        Path to a local file to load the data from.
+    metadata : dict
+        Dict of metadata associated to be the object when displayed
+    """
     def _check_data(self):
         if self.data is not None and not isinstance(self.data, str):
             raise TypeError("%s expects text, not %r" % (self.__class__.__name__, self.data))
@@ -613,8 +625,9 b' class JSON(DisplayObject):'
     def _repr_json_(self):
         return self._data_and_metadata()
 
+
 _css_t = """var link = document.createElement("link");
-	link.ref = "stylesheet";
+	link.rel = "stylesheet";
 	link.type = "text/css";
 	link.href = "%s";
 	document.head.appendChild(link);

IPython/core/extensions.py

@@ -88,13 +88,7 b' class ExtensionManager(Configurable):'
 
         with self.shell.builtin_trap:
             if module_str not in sys.modules:
-                with prepended_to_syspath(self.ipython_extension_dir):
                     mod = import_module(module_str)
-                    if mod.__file__.startswith(self.ipython_extension_dir):
-                        print(("Loading extensions from {dir} is deprecated. "
-                               "We recommend managing extensions like any "
-                               "other Python packages, in site-packages.").format(
-                              dir=compress_user(self.ipython_extension_dir)))
             mod = sys.modules[module_str]
             if self._call_load_ipython_extension(mod):
                 self.loaded.add(module_str)
@@ -155,13 +149,3 b' class ExtensionManager(Configurable):'
         if hasattr(mod, 'unload_ipython_extension'):
             mod.unload_ipython_extension(self.shell)
             return True
-
-    @undoc
-    def install_extension(self, url, filename=None):
-        """
-        Deprecated.
-        """
-        # Ensure the extension directory exists
-        raise DeprecationWarning(
-            '`install_extension` and the `install_ext` magic have been deprecated since IPython 4.0'
-            'Use pip or other package managers to manage ipython extensions.')

IPython/core/inputtransformer2.py

@@ -429,13 +429,17 b' class EscapedCommand(TokenTransformBase):'
 
         return lines_before + [new_line] + lines_after
 
-_help_end_re = re.compile(r"""(%{0,2}
+
+_help_end_re = re.compile(
+    r"""(%{0,2}
                               (?!\d)[\w*]+            # Variable name
-                              (\.(?!\d)[\w*]+)*       # .etc.etc
+    (\.(?!\d)[\w*]+|\[-?[0-9]+\])*       # .etc.etc or [0], we only support literal integers.
                               )
                               (\?\??)$                # ? or ??
                               """,
-                              re.VERBOSE)
+    re.VERBOSE,
+)
+
 
 class HelpEnd(TokenTransformBase):
     """Transformer for help syntax: obj? and obj??"""
@@ -464,7 +468,8 b' class HelpEnd(TokenTransformBase):'
     def transform(self, lines):
         """Transform a help command found by the ``find()`` classmethod.
         """
-        piece = ''.join(lines[self.start_line:self.q_line+1])
+
+        piece = "".join(lines[self.start_line : self.q_line + 1])
         indent, content = piece[:self.start_col], piece[self.start_col:]
         lines_before = lines[:self.start_line]
         lines_after = lines[self.q_line + 1:]
@@ -543,8 +548,13 b' def has_sunken_brackets(tokens: List[tokenize.TokenInfo]):'
 
 def show_linewise_tokens(s: str):
     """For investigation and debugging"""
-    if not s.endswith('\n'):
-        s += '\n'
+    warnings.warn(
+        "show_linewise_tokens is deprecated since IPython 8.6",
+        DeprecationWarning,
+        stacklevel=2,
+    )
+    if not s.endswith("\n"):
+        s += "\n"
     lines = s.splitlines(keepends=True)
     for line in make_tokens_by_line(lines):
         print("Line -------")

IPython/core/interactiveshell.py

@@ -61,7 +61,7 b' from IPython.core import magic, oinspect, page, prefilter, ultratb'
 from IPython.core.alias import Alias, AliasManager
 from IPython.core.autocall import ExitAutocall
 from IPython.core.builtin_trap import BuiltinTrap
-from IPython.core.compilerop import CachingCompiler, check_linecache_ipython
+from IPython.core.compilerop import CachingCompiler
 from IPython.core.debugger import InterruptiblePdb
 from IPython.core.display_trap import DisplayTrap
 from IPython.core.displayhook import DisplayHook
@@ -147,6 +147,19 b" dedent_re = re.compile(r'^\\s+raise|^\\s+return|^\\s+pass')"
 # Utilities
 #-----------------------------------------------------------------------------
 
+
+def is_integer_string(s: str):
+    """
+    Variant of "str.isnumeric()" that allow negative values and other ints.
+    """
+    try:
+        int(s)
+        return True
+    except ValueError:
+        return False
+    raise ValueError("Unexpected error")
+
+
 @undoc
 def softspace(file, newvalue):
     """Copied from code.py, to remove the dependency"""
@@ -213,7 +226,9 b' class ExecutionInfo(object):'
         raw_cell = (
             (self.raw_cell[:50] + "..") if len(self.raw_cell) > 50 else self.raw_cell
         )
-        return '<%s object at %x, raw_cell="%s" store_history=%s silent=%s shell_futures=%s cell_id=%s>' % (
+        return (
+            '<%s object at %x, raw_cell="%s" store_history=%s silent=%s shell_futures=%s cell_id=%s>'
+            % (
             name,
             id(self),
             raw_cell,
@@ -222,6 +237,7 b' class ExecutionInfo(object):'
             self.shell_futures,
             self.cell_id,
         )
+        )
 
 
 class ExecutionResult(object):
@@ -254,6 +270,16 b' class ExecutionResult(object):'
         return '<%s object at %x, execution_count=%s error_before_exec=%s error_in_exec=%s info=%s result=%s>' %\
                 (name, id(self), self.execution_count, self.error_before_exec, self.error_in_exec, repr(self.info), repr(self.result))
 
[e-mail](io_open)
+def _modified_open(file, *args, **kwargs):
+    if file in {0, 1, 2}:
+        raise ValueError(
+            f"IPython won't let you open fd={file} by default "
+            "as it is likely to crash IPython. If you know what you are doing, "
+            "you can use builtins' open."
+        )
+
+    return io_open(file, *args, **kwargs)
 
 class InteractiveShell(SingletonConfigurable):
     """An enhanced, interactive shell for Python."""
@@ -1307,6 +1333,7 b' class InteractiveShell(SingletonConfigurable):'
 
         ns['exit'] = self.exiter
         ns['quit'] = self.exiter
+        ns["open"] = _modified_open
 
         # Sync what we've added so far to user_ns_hidden so these aren't seen
         # by %who
@@ -1537,10 +1564,33 b' class InteractiveShell(SingletonConfigurable):'
         Has special code to detect magic functions.
         """
         oname = oname.strip()
-        if not oname.startswith(ESC_MAGIC) and \
-                not oname.startswith(ESC_MAGIC2) and \
-                not all(a.isidentifier() for a in oname.split(".")):
-            return {'found': False}
+        raw_parts = oname.split(".")
+        parts = []
+        parts_ok = True
+        for p in raw_parts:
+            if p.endswith("]"):
+                var, *indices = p.split("[")
+                if not var.isidentifier():
+                    parts_ok = False
+                    break
+                parts.append(var)
+                for ind in indices:
+                    if ind[-1] != "]" and not is_integer_string(ind[:-1]):
+                        parts_ok = False
+                        break
+                    parts.append(ind[:-1])
+                continue
+
+            if not p.isidentifier():
+                parts_ok = False
+            parts.append(p)
+
+        if (
+            not oname.startswith(ESC_MAGIC)
+            and not oname.startswith(ESC_MAGIC2)
+            and not parts_ok
+        ):
+            return {"found": False}
 
         if namespaces is None:
             # Namespaces to search in:
@@ -1562,7 +1612,7 b' class InteractiveShell(SingletonConfigurable):'
         # Look for the given name by splitting it in parts.  If the head is
         # found, then we look for all the remaining parts as members, and only
         # declare success if we can find them all.
-        oname_parts = oname.split('.')
+        oname_parts = parts
         oname_head, oname_rest = oname_parts[0],oname_parts[1:]
         for nsname,ns in namespaces:
             try:
@@ -1579,6 +1629,9 b' class InteractiveShell(SingletonConfigurable):'
                         if idx == len(oname_rest) - 1:
                             obj = self._getattr_property(obj, part)
                         else:
+                            if is_integer_string(part):
+                                obj = obj[int(part)]
+                            else:
                             obj = getattr(obj, part)
                     except:
                         # Blanket except b/c some badly implemented objects
@@ -1643,6 +1696,9 b' class InteractiveShell(SingletonConfigurable):'
                 #
                 # The universal alternative is to traverse the mro manually
                 # searching for attrname in class dicts.
+                if is_integer_string(attrname):
+                    return obj[int(attrname)]
+                else:
                 attr = getattr(type(obj), attrname)
             except AttributeError:
                 pass
@@ -1765,7 +1821,6 b' class InteractiveShell(SingletonConfigurable):'
         self.InteractiveTB = ultratb.AutoFormattedTB(mode = 'Plain',
                                                      color_scheme='NoColor',
                                                      tb_offset = 1,
-                                   check_cache=check_linecache_ipython,
                                    debugger_cls=self.debugger_cls, parent=self)
 
         # The instance will store a pointer to the system-wide exception hook,

IPython/core/magics/basic.py

@@ -297,7 +297,10 b' Currently the magic system has the following functions:""",'
         oname = args and args or '_'
         info = self.shell._ofind(oname)
         if info['found']:
-            txt = (raw and str or pformat)( info['obj'] )
+            if raw:
+                txt = str(info["obj"])
+            else:
+                txt = pformat(info["obj"])
             page.page(txt)
         else:
             print('Object `%s` not found' % oname)

IPython/core/magics/config.py

@@ -80,6 +80,9 b' class ConfigMagics(Magics):'
                 Enable debug for the Completer. Mostly print extra information for
                 experimental jedi integration.
                 Current: False
+            IPCompleter.disable_matchers=<list-item-1>...
+                List of matchers to disable.
+                Current: []
             IPCompleter.greedy=<Bool>
                 Activate greedy completion
                         PENDING DEPRECATION. this is now mostly taken care of with Jedi.
@@ -102,6 +105,8 b' class ConfigMagics(Magics):'
                 Whether to merge completion results into a single list
                         If False, only the completion results from the first non-empty
                         completer will be returned.
+                        As of version 8.6.0, setting the value to ``False`` is an alias for:
+                        ``IPCompleter.suppress_competing_matchers = True.``.
                 Current: True
             IPCompleter.omit__names=<Enum>
                 Instruct the completer to omit private method names
@@ -117,6 +122,24 b' class ConfigMagics(Magics):'
             IPCompleter.profiler_output_dir=<Unicode>
                 Template for path at which to output profile data for completions.
                 Current: '.completion_profiles'
+            IPCompleter.suppress_competing_matchers=<Union>
+                Whether to suppress completions from other *Matchers*.
+                When set to ``None`` (default) the matchers will attempt to auto-detect
+                whether suppression of other matchers is desirable. For example, at the
+                beginning of a line followed by `%` we expect a magic completion to be the
+                only applicable option, and after ``my_dict['`` we usually expect a
+                completion with an existing dictionary key.
+                If you want to disable this heuristic and see completions from all matchers,
+                set ``IPCompleter.suppress_competing_matchers = False``. To disable the
+                heuristic for specific matchers provide a dictionary mapping:
+                ``IPCompleter.suppress_competing_matchers = {'IPCompleter.dict_key_matcher':
+                False}``.
+                Set ``IPCompleter.suppress_competing_matchers = True`` to limit completions
+                to the set of matchers with the highest priority; this is equivalent to
+                ``IPCompleter.merge_completions`` and can be beneficial for performance, but
+                will sometimes omit relevant candidates from matchers further down the
+                priority list.
+                Current: None
             IPCompleter.use_jedi=<Bool>
                 Experimental: Use Jedi to generate autocompletions. Default to True if jedi
                 is installed.

IPython/core/magics/namespace.py

@@ -492,7 +492,7 b' class NamespaceMagics(Magics):'
         --aggressive
             Try to aggressively remove modules from sys.modules ; this
             may allow you to reimport Python modules that have been updated and
-            pick up changes, but can have unattended consequences.
+            pick up changes, but can have unintended consequences.
 
         in
             reset input history

IPython/core/pylabtools.py

@@ -26,6 +26,7 b' backends = {'
     "qt": "Qt5Agg",
     "osx": "MacOSX",
     "nbagg": "nbAgg",
+    "webagg": "WebAgg",
     "notebook": "nbAgg",
     "agg": "agg",
     "svg": "svg",

IPython/core/release.py

@@ -16,7 +16,7 b''
 # release.  'dev' as a _version_extra string means this is a development
 # version
 _version_major = 8
-_version_minor = 6
+_version_minor = 7
 _version_patch = 0
 _version_extra = ".dev"
 # _version_extra = "rc1"
@@ -36,7 +36,7 b' version_info = (_version_major, _version_minor, _version_patch, _version_extra)'
 kernel_protocol_version_info = (5, 0)
 kernel_protocol_version = "%i.%i" % kernel_protocol_version_info
 
-license = 'BSD'
+license = "BSD-3-Clause"
 
 authors = {'Fernando' : ('Fernando Perez','[e-mail]'),
            'Janko'    : ('Janko Hauser','[e-mail]'),

IPython/core/tests/nonascii.py

@@ -1,4 +1,4 b''
 # coding: iso-8859-5
 # (Unlikely to be the default encoding for most testers.)
 # ������������������� <- Cyrillic characters
-u = '����'
+u = "����"

IPython/core/tests/test_completer.py

@@ -24,6 +24,9 b' from IPython.core.completer import ('
     provisionalcompleter,
     match_dict_keys,
     _deduplicate_completions,
+    completion_matcher,
+    SimpleCompletion,
+    CompletionContext,
 )
 
 # -----------------------------------------------------------------------------
@@ -109,6 +112,16 b' def greedy_completion():'
         ip.Completer.greedy = greedy_original
 
 
+@contextmanager
+def custom_matchers(matchers):
+    ip = get_ipython()
+    try:
+        ip.Completer.custom_matchers.extend(matchers)
+        yield
+    finally:
+        ip.Completer.custom_matchers.clear()
+
+
 def test_protect_filename():
     if sys.platform == "win32":
         pairs = [
@@ -298,7 +311,7 b' class TestCompleter(unittest.TestCase):'
         ip = get_ipython()
 
         name, matches = ip.complete("\\Ⅴ")
-        self.assertEqual(matches, ("\\ROMAN NUMERAL FIVE",))
+        self.assertEqual(matches, ["\\ROMAN NUMERAL FIVE"])
 
     def test_forward_unicode_completion(self):
         ip = get_ipython()
@@ -379,6 +392,12 b' class TestCompleter(unittest.TestCase):'
 
     def test_quoted_file_completions(self):
         ip = get_ipython()
+
+        def _(text):
+            return ip.Completer._complete(
+                cursor_line=0, cursor_pos=len(text), full_text=text
+            )["IPCompleter.file_matcher"]["completions"]
+
         with TemporaryWorkingDirectory():
             name = "foo'bar"
             open(name, "w", encoding="utf-8").close()
@@ -387,25 +406,16 b' class TestCompleter(unittest.TestCase):'
             escaped = name if sys.platform == "win32" else "foo\\'bar"
 
             # Single quote matches embedded single quote
-            text = "open('foo"
-            c = ip.Completer._complete(
-                cursor_line=0, cursor_pos=len(text), full_text=text
-            )[1]
-            self.assertEqual(c, [escaped])
+            c = _("open('foo")[0]
+            self.assertEqual(c.text, escaped)
 
             # Double quote requires no escape
-            text = 'open("foo'
-            c = ip.Completer._complete(
-                cursor_line=0, cursor_pos=len(text), full_text=text
-            )[1]
-            self.assertEqual(c, [name])
+            c = _('open("foo')[0]
+            self.assertEqual(c.text, name)
 
             # No quote requires an escape
-            text = "%ls foo"
-            c = ip.Completer._complete(
-                cursor_line=0, cursor_pos=len(text), full_text=text
-            )[1]
-            self.assertEqual(c, [escaped])
+            c = _("%ls foo")[0]
+            self.assertEqual(c.text, escaped)
 
     def test_all_completions_dups(self):
         """
@@ -475,6 +485,17 b' class TestCompleter(unittest.TestCase):'
             "encoding" in c.signature
         ), "Signature of function was not found by completer"
 
+    def test_completions_have_type(self):
+        """
+        Lets make sure matchers provide completion type.
+        """
+        ip = get_ipython()
+        with provisionalcompleter():
+            ip.Completer.use_jedi = False
+            completions = ip.Completer.completions("%tim", 3)
+            c = next(completions)  # should be `%time` or similar
+        assert c.type == "magic", "Type of magic was not assigned by completer"
+
     @pytest.mark.xfail(reason="Known failure on jedi<=0.18.0")
     def test_deduplicate_completions(self):
         """
@@ -1273,3 +1294,153 b' class TestCompleter(unittest.TestCase):'
             completions = completer.completions(text, len(text))
             for c in completions:
                 self.assertEqual(c.text[0], "%")
+
+    def test_fwd_unicode_restricts(self):
+        ip = get_ipython()
+        completer = ip.Completer
+        text = "\\ROMAN NUMERAL FIVE"
+
+        with provisionalcompleter():
+            completer.use_jedi = True
+            completions = [
+                completion.text for completion in completer.completions(text, len(text))
+            ]
+            self.assertEqual(completions, ["\u2164"])
+
+    def test_dict_key_restrict_to_dicts(self):
+        """Test that dict key suppresses non-dict completion items"""
+        ip = get_ipython()
+        c = ip.Completer
+        d = {"abc": None}
+        ip.user_ns["d"] = d
+
+        text = 'd["a'
+
+        def _():
+            with provisionalcompleter():
+                c.use_jedi = True
+                return [
+                    completion.text for completion in c.completions(text, len(text))
+                ]
+
+        completions = _()
+        self.assertEqual(completions, ["abc"])
+
+        # check that it can be disabled in granular manner:
+        cfg = Config()
+        cfg.IPCompleter.suppress_competing_matchers = {
+            "IPCompleter.dict_key_matcher": False
+        }
+        c.update_config(cfg)
+
+        completions = _()
+        self.assertIn("abc", completions)
+        self.assertGreater(len(completions), 1)
+
+    def test_matcher_suppression(self):
+        @completion_matcher(identifier="a_matcher")
+        def a_matcher(text):
+            return ["completion_a"]
+
+        @completion_matcher(identifier="b_matcher", api_version=2)
+        def b_matcher(context: CompletionContext):
+            text = context.token
+            result = {"completions": [SimpleCompletion("completion_b")]}
+
+            if text == "suppress c":
+                result["suppress"] = {"c_matcher"}
+
+            if text.startswith("suppress all"):
+                result["suppress"] = True
+                if text == "suppress all but c":
+                    result["do_not_suppress"] = {"c_matcher"}
+                if text == "suppress all but a":
+                    result["do_not_suppress"] = {"a_matcher"}
+
+            return result
+
+        @completion_matcher(identifier="c_matcher")
+        def c_matcher(text):
+            return ["completion_c"]
+
+        with custom_matchers([a_matcher, b_matcher, c_matcher]):
+            ip = get_ipython()
+            c = ip.Completer
+
+            def _(text, expected):
+                c.use_jedi = False
+                s, matches = c.complete(text)
+                self.assertEqual(expected, matches)
+
+            _("do not suppress", ["completion_a", "completion_b", "completion_c"])
+            _("suppress all", ["completion_b"])
+            _("suppress all but a", ["completion_a", "completion_b"])
+            _("suppress all but c", ["completion_b", "completion_c"])
+
+            def configure(suppression_config):
+                cfg = Config()
+                cfg.IPCompleter.suppress_competing_matchers = suppression_config
+                c.update_config(cfg)
+
+            # test that configuration takes priority over the run-time decisions
+
+            configure(False)
+            _("suppress all", ["completion_a", "completion_b", "completion_c"])
+
+            configure({"b_matcher": False})
+            _("suppress all", ["completion_a", "completion_b", "completion_c"])
+
+            configure({"a_matcher": False})
+            _("suppress all", ["completion_b"])
+
+            configure({"b_matcher": True})
+            _("do not suppress", ["completion_b"])
+
+    def test_matcher_disabling(self):
+        @completion_matcher(identifier="a_matcher")
+        def a_matcher(text):
+            return ["completion_a"]
+
+        @completion_matcher(identifier="b_matcher")
+        def b_matcher(text):
+            return ["completion_b"]
+
+        def _(expected):
+            s, matches = c.complete("completion_")
+            self.assertEqual(expected, matches)
+
+        with custom_matchers([a_matcher, b_matcher]):
+            ip = get_ipython()
+            c = ip.Completer
+
+            _(["completion_a", "completion_b"])
+
+            cfg = Config()
+            cfg.IPCompleter.disable_matchers = ["b_matcher"]
+            c.update_config(cfg)
+
+            _(["completion_a"])
+
+            cfg.IPCompleter.disable_matchers = []
+            c.update_config(cfg)
+
+    def test_matcher_priority(self):
+        @completion_matcher(identifier="a_matcher", priority=0, api_version=2)
+        def a_matcher(text):
+            return {"completions": [SimpleCompletion("completion_a")], "suppress": True}
+
+        @completion_matcher(identifier="b_matcher", priority=2, api_version=2)
+        def b_matcher(text):
+            return {"completions": [SimpleCompletion("completion_b")], "suppress": True}
+
+        def _(expected):
+            s, matches = c.complete("completion_")
+            self.assertEqual(expected, matches)
+
+        with custom_matchers([a_matcher, b_matcher]):
+            ip = get_ipython()
+            c = ip.Completer
+
+            _(["completion_b"])
+            a_matcher.matcher_priority = 3
+            _(["completion_a"])

IPython/core/tests/test_interactiveshell.py

@@ -103,6 +103,18 b' class InteractiveShellTestCase(unittest.TestCase):'
         res = ip.run_cell("raise = 3")
         self.assertIsInstance(res.error_before_exec, SyntaxError)
 
+    def test_open_standard_input_stream(self):
+        res = ip.run_cell("open(0)")
+        self.assertIsInstance(res.error_in_exec, ValueError)
+
+    def test_open_standard_output_stream(self):
+        res = ip.run_cell("open(1)")
+        self.assertIsInstance(res.error_in_exec, ValueError)
+
+    def test_open_standard_error_stream(self):
+        res = ip.run_cell("open(2)")
+        self.assertIsInstance(res.error_in_exec, ValueError)
+
     def test_In_variable(self):
         "Verify that In variable grows with user input (GH-284)"
         oldlen = len(ip.user_ns['In'])

IPython/core/tests/test_iplib.py

@@ -1,15 +1,11 b''
 """Tests for the key interactiveshell module, where the main ipython class is defined.
 """
-#-----------------------------------------------------------------------------
-# Module imports
-#-----------------------------------------------------------------------------
 
+import stack_data
+import sys
 
-# our own packages
+SV_VERSION = tuple([int(x) for x in stack_data.__version__.split(".")[0:2]])
 
-#-----------------------------------------------------------------------------
-# Test functions
-#-----------------------------------------------------------------------------
 
 def test_reset():
     """reset must clear most namespaces."""
@@ -170,7 +166,10 b' def doctest_tb_sysexit():'
     """
 
 
-def doctest_tb_sysexit_verbose():
+if sys.version_info >= (3, 9):
+    if SV_VERSION < (0, 6):
+
+        def doctest_tb_sysexit_verbose_stack_data_05():
     """
     In [18]: %run simpleerr.py exit
     An exception has occurred, use %tb to see the full traceback.
@@ -210,6 +209,50 b' def doctest_tb_sysexit_verbose():'
     SystemExit: (2, 'Mode = exit')
     """
 
+    else:
+        # currently the only difference is
+        # + mode = 'exit'
+
+        def doctest_tb_sysexit_verbose_stack_data_06():
+            """
+            In [18]: %run simpleerr.py exit
+            An exception has occurred, use %tb to see the full traceback.
+            SystemExit: (1, 'Mode = exit')
+
+            In [19]: %run simpleerr.py exit 2
+            An exception has occurred, use %tb to see the full traceback.
+            SystemExit: (2, 'Mode = exit')
+
+            In [23]: %xmode verbose
+            Exception reporting mode: Verbose
+
+            In [24]: %tb
+            ---------------------------------------------------------------------------
+            SystemExit                                Traceback (most recent call last)
+            <BLANKLINE>
+            ...
+                 30     except IndexError:
+                 31         mode = 'div'
+            ---> 33     bar(mode)
+                    mode = 'exit'
+            <BLANKLINE>
+            ... in bar(mode='exit')
+                 ...     except:
+                 ...         stat = 1
+            ---> ...     sysexit(stat, mode)
+                    mode = 'exit'
+                    stat = 2
+                 ...     else:
+                 ...         raise ValueError('Unknown mode')
+            <BLANKLINE>
+            ... in sysexit(stat=2, mode='exit')
+                 10 def sysexit(stat, mode):
+            ---> 11     raise SystemExit(stat, f"Mode = {mode}")
+                    stat = 2
+                    mode = 'exit'
+            <BLANKLINE>
+            SystemExit: (2, 'Mode = exit')
+            """
 
 def test_run_cell():
     import textwrap

IPython/core/tests/test_magic.py

@@ -84,7 +84,7 b' def test_extract_symbols_raises_exception_with_non_python_code():'
 def test_magic_not_found():
     # magic not found raises UsageError
     with pytest.raises(UsageError):
-        _ip.magic('doesntexist')
+        _ip.run_line_magic("doesntexist", "")
 
     # ensure result isn't success when a magic isn't found
     result = _ip.run_cell('%doesntexist')
@@ -116,13 +116,14 b' def test_config():'
     magic.
     """
     ## should not raise.
-    _ip.magic('config')
+    _ip.run_line_magic("config", "")
+
 
 def test_config_available_configs():
     """ test that config magic prints available configs in unique and
     sorted order. """
     with capture_output() as captured:
-        _ip.magic('config')
+        _ip.run_line_magic("config", "")
 
     stdout = captured.stdout
     config_classes = stdout.strip().split('\n')[1:]
@@ -131,7 +132,7 b' def test_config_available_configs():'
 def test_config_print_class():
     """ test that config with a classname prints the class's options. """
     with capture_output() as captured:
-        _ip.magic('config TerminalInteractiveShell')
+        _ip.run_line_magic("config", "TerminalInteractiveShell")
 
     stdout = captured.stdout
     assert re.match(
@@ -144,7 +145,7 b' def test_rehashx():'
     _ip.alias_manager.clear_aliases()
     del _ip.db['syscmdlist']
 
-    _ip.magic('rehashx')
+    _ip.run_line_magic("rehashx", "")
     # Practically ALL ipython development systems will have more than 10 aliases
 
     assert len(_ip.alias_manager.aliases) > 10
@@ -277,11 +278,11 b' def test_macro():'
     cmds = ["a=1", "def b():\n  return a**2", "print(a,b())"]
     for i, cmd in enumerate(cmds, start=1):
         ip.history_manager.store_inputs(i, cmd)
-    ip.magic("macro test 1-3")
+    ip.run_line_magic("macro", "test 1-3")
     assert ip.user_ns["test"].value == "\n".join(cmds) + "\n"
 
     # List macros
-    assert "test" in ip.magic("macro")
+    assert "test" in ip.run_line_magic("macro", "")
 
 
 def test_macro_run():
@@ -302,7 +303,7 b' def test_magic_magic():'
     """Test %magic"""
     ip = get_ipython()
     with capture_output() as captured:
-        ip.magic("magic")
+        ip.run_line_magic("magic", "")
 
     stdout = captured.stdout
     assert "%magic" in stdout
@@ -316,7 +317,7 b' def test_numpy_reset_array_undec():'
     _ip.ex("import numpy as np")
     _ip.ex("a = np.empty(2)")
     assert "a" in _ip.user_ns
-    _ip.magic("reset -f array")
+    _ip.run_line_magic("reset", "-f array")
     assert "a" not in _ip.user_ns
 
 
@@ -326,7 +327,7 b' def test_reset_out():'
     # test '%reset -f out', make an Out prompt
     _ip.run_cell("parrot", store_history=True)
     assert "dead" in [_ip.user_ns[x] for x in ("_", "__", "___")]
-    _ip.magic("reset -f out")
+    _ip.run_line_magic("reset", "-f out")
     assert "dead" not in [_ip.user_ns[x] for x in ("_", "__", "___")]
     assert len(_ip.user_ns["Out"]) == 0
 
@@ -336,7 +337,7 b' def test_reset_in():'
     # test '%reset -f in'
     _ip.run_cell("parrot", store_history=True)
     assert "parrot" in [_ip.user_ns[x] for x in ("_i", "_ii", "_iii")]
-    _ip.magic("%reset -f in")
+    _ip.run_line_magic("reset", "-f in")
     assert "parrot" not in [_ip.user_ns[x] for x in ("_i", "_ii", "_iii")]
     assert len(set(_ip.user_ns["In"])) == 1
 
@@ -344,10 +345,10 b' def test_reset_in():'
 def test_reset_dhist():
     "Test '%reset dhist' magic"
     _ip.run_cell("tmp = [d for d in _dh]")  # copy before clearing
-    _ip.magic("cd " + os.path.dirname(pytest.__file__))
-    _ip.magic("cd -")
+    _ip.run_line_magic("cd", os.path.dirname(pytest.__file__))
+    _ip.run_line_magic("cd", "-")
     assert len(_ip.user_ns["_dh"]) > 0
-    _ip.magic("reset -f dhist")
+    _ip.run_line_magic("reset", "-f dhist")
     assert len(_ip.user_ns["_dh"]) == 0
     _ip.run_cell("_dh = [d for d in tmp]")  # restore
 
@@ -472,8 +473,8 b' def test_time_local_ns():'
 
 def test_doctest_mode():
     "Toggle doctest_mode twice, it should be a no-op and run without error"
-    _ip.magic('doctest_mode')
-    _ip.magic('doctest_mode')
+    _ip.run_line_magic("doctest_mode", "")
+    _ip.run_line_magic("doctest_mode", "")
 
 
 def test_parse_options():
@@ -498,7 +499,9 b' def test_parse_options_preserve_non_option_string():'
 def test_run_magic_preserve_code_block():
     """Test to assert preservation of non-option part of magic-block, while running magic."""
     _ip.user_ns["spaces"] = []
-    _ip.magic("timeit -n1 -r1 spaces.append([s.count(' ') for s in ['document']])")
+    _ip.run_line_magic(
+        "timeit", "-n1 -r1 spaces.append([s.count(' ') for s in ['document']])"
+    )
     assert _ip.user_ns["spaces"] == [[0]]
 
 
@@ -509,13 +512,13 b' def test_dirops():'
     startdir = os.getcwd()
     ipdir = os.path.realpath(_ip.ipython_dir)
     try:
-        _ip.magic('cd "%s"' % ipdir)
+        _ip.run_line_magic("cd", '"%s"' % ipdir)
         assert curpath() == ipdir
-        _ip.magic('cd -')
+        _ip.run_line_magic("cd", "-")
         assert curpath() == startdir
-        _ip.magic('pushd "%s"' % ipdir)
+        _ip.run_line_magic("pushd", '"%s"' % ipdir)
         assert curpath() == ipdir
-        _ip.magic('popd')
+        _ip.run_line_magic("popd", "")
         assert curpath() == startdir
     finally:
         os.chdir(startdir)
@@ -542,7 +545,7 b' def test_xmode():'
     # Calling xmode three times should be a no-op
     xmode = _ip.InteractiveTB.mode
     for i in range(4):
-        _ip.magic("xmode")
+        _ip.run_line_magic("xmode", "")
     assert _ip.InteractiveTB.mode == xmode
 
 def test_reset_hard():
@@ -557,7 +560,7 b' def test_reset_hard():'
     _ip.run_cell("a")
 
     assert monitor == []
-    _ip.magic("reset -f")
+    _ip.run_line_magic("reset", "-f")
     assert monitor == [1]
 
 class TestXdel(tt.TempFileMixin):
@@ -570,14 +573,14 b' class TestXdel(tt.TempFileMixin):'
                "a = A()\n")
         self.mktmp(src)
         # %run creates some hidden references...
-        _ip.magic("run %s" % self.fname)
+        _ip.run_line_magic("run", "%s" % self.fname)
         # ... as does the displayhook.
         _ip.run_cell("a")
 
         monitor = _ip.user_ns["A"].monitor
         assert monitor == []
 
-        _ip.magic("xdel a")
+        _ip.run_line_magic("xdel", "a")
 
         # Check that a's __del__ method has been called.
         gc.collect(0)
@@ -614,7 +617,7 b' def test_whos():'
         def __repr__(self):
             raise Exception()
     _ip.user_ns['a'] = A()
-    _ip.magic("whos")
+    _ip.run_line_magic("whos", "")
 
 def doctest_precision():
     """doctest for %precision
@@ -655,12 +658,12 b' def test_psearch():'
 def test_timeit_shlex():
     """test shlex issues with timeit (#1109)"""
     _ip.ex("def f(*a,**kw): pass")
-    _ip.magic('timeit -n1 "this is a bug".count(" ")')
-    _ip.magic('timeit -r1 -n1 f(" ", 1)')
-    _ip.magic('timeit -r1 -n1 f(" ", 1, " ", 2, " ")')
-    _ip.magic('timeit -r1 -n1 ("a " + "b")')
-    _ip.magic('timeit -r1 -n1 f("a " + "b")')
-    _ip.magic('timeit -r1 -n1 f("a " + "b ")')
+    _ip.run_line_magic("timeit", '-n1 "this is a bug".count(" ")')
+    _ip.run_line_magic("timeit", '-r1 -n1 f(" ", 1)')
+    _ip.run_line_magic("timeit", '-r1 -n1 f(" ", 1, " ", 2, " ")')
+    _ip.run_line_magic("timeit", '-r1 -n1 ("a " + "b")')
+    _ip.run_line_magic("timeit", '-r1 -n1 f("a " + "b")')
+    _ip.run_line_magic("timeit", '-r1 -n1 f("a " + "b ")')
 
 
 def test_timeit_special_syntax():
@@ -738,9 +741,9 b' def test_extension():'
     try:
         _ip.user_ns.pop('arq', None)
         invalidate_caches()   # Clear import caches
-        _ip.magic("load_ext daft_extension")
+        _ip.run_line_magic("load_ext", "daft_extension")
         assert _ip.user_ns["arq"] == 185
-        _ip.magic("unload_ext daft_extension")
+        _ip.run_line_magic("unload_ext", "daft_extension")
         assert 'arq' not in _ip.user_ns
     finally:
         sys.path.remove(daft_path)
@@ -755,17 +758,17 b' def test_notebook_export_json():'
         _ip.history_manager.store_inputs(i, cmd)
     with TemporaryDirectory() as td:
         outfile = os.path.join(td, "nb.ipynb")
-        _ip.magic("notebook %s" % outfile)
+        _ip.run_line_magic("notebook", "%s" % outfile)
 
 
 class TestEnv(TestCase):
 
     def test_env(self):
-        env = _ip.magic("env")
+        env = _ip.run_line_magic("env", "")
         self.assertTrue(isinstance(env, dict))
 
     def test_env_secret(self):
-        env = _ip.magic("env")
+        env = _ip.run_line_magic("env", "")
         hidden = "<hidden>"
         with mock.patch.dict(
             os.environ,
@@ -776,35 +779,35 b' class TestEnv(TestCase):'
                 "VAR": "abc"
             }
         ):
-            env = _ip.magic("env")
+            env = _ip.run_line_magic("env", "")
         assert env["API_KEY"] == hidden
         assert env["SECRET_THING"] == hidden
         assert env["JUPYTER_TOKEN"] == hidden
         assert env["VAR"] == "abc"
 
     def test_env_get_set_simple(self):
-        env = _ip.magic("env var val1")
+        env = _ip.run_line_magic("env", "var val1")
         self.assertEqual(env, None)
-        self.assertEqual(os.environ['var'], 'val1')
-        self.assertEqual(_ip.magic("env var"), 'val1')
-        env = _ip.magic("env var=val2")
+        self.assertEqual(os.environ["var"], "val1")
+        self.assertEqual(_ip.run_line_magic("env", "var"), "val1")
+        env = _ip.run_line_magic("env", "var=val2")
         self.assertEqual(env, None)
         self.assertEqual(os.environ['var'], 'val2')
 
     def test_env_get_set_complex(self):
-        env = _ip.magic("env var 'val1 '' 'val2")
+        env = _ip.run_line_magic("env", "var 'val1 '' 'val2")
         self.assertEqual(env, None)
         self.assertEqual(os.environ['var'], "'val1 '' 'val2")
-        self.assertEqual(_ip.magic("env var"), "'val1 '' 'val2")
-        env = _ip.magic('env var=val2 val3="val4')
+        self.assertEqual(_ip.run_line_magic("env", "var"), "'val1 '' 'val2")
+        env = _ip.run_line_magic("env", 'var=val2 val3="val4')
         self.assertEqual(env, None)
         self.assertEqual(os.environ['var'], 'val2 val3="val4')
 
     def test_env_set_bad_input(self):
-        self.assertRaises(UsageError, lambda: _ip.magic("set_env var"))
+        self.assertRaises(UsageError, lambda: _ip.run_line_magic("set_env", "var"))
 
     def test_env_set_whitespace(self):
-        self.assertRaises(UsageError, lambda: _ip.magic("env var A=B"))
+        self.assertRaises(UsageError, lambda: _ip.run_line_magic("env", "var A=B"))
 
 
 class CellMagicTestCase(TestCase):
@@ -1308,7 +1311,7 b' def test_ls_magic():'
     ip = get_ipython()
     json_formatter = ip.display_formatter.formatters['application/json']
     json_formatter.enabled = True
-    lsmagic = ip.magic('lsmagic')
+    lsmagic = ip.run_line_magic("lsmagic", "")
     with warnings.catch_warnings(record=True) as w:
         j = json_formatter(lsmagic)
     assert sorted(j) == ["cell", "line"]
@@ -1358,16 +1361,16 b' def test_logging_magic_not_quiet():'
 
 
 def test_time_no_var_expand():
-    _ip.user_ns['a'] = 5
-    _ip.user_ns['b'] = []
-    _ip.magic('time b.append("{a}")')
-    assert _ip.user_ns['b'] == ['{a}']
+    _ip.user_ns["a"] = 5
+    _ip.user_ns["b"] = []
+    _ip.run_line_magic("time", 'b.append("{a}")')
+    assert _ip.user_ns["b"] == ["{a}"]
 
 
 # this is slow, put at the end for local testing.
 def test_timeit_arguments():
     "Test valid timeit arguments, should not cause SyntaxError (GH #1269)"
-    _ip.magic("timeit -n1 -r1 a=('#')")
+    _ip.run_line_magic("timeit", "-n1 -r1 a=('#')")
 
 
 MINIMAL_LAZY_MAGIC = """
@@ -1442,7 +1445,7 b' def test_run_module_from_import_hook():'
         sys.meta_path.insert(0, MyTempImporter())
 
         with capture_output() as captured:
-            _ip.magic("run -m my_tmp")
+            _ip.run_line_magic("run", "-m my_tmp")
             _ip.run_cell("import my_tmp")
 
         output = "Loaded my_tmp\nI just ran a script\nLoaded my_tmp\n"

IPython/core/tests/test_oinspect.py

@@ -5,6 +5,7 b''
 # Distributed under the terms of the Modified BSD License.
 
 
+from contextlib import contextmanager
 from inspect import signature, Signature, Parameter
 import inspect
 import os
@@ -43,7 +44,7 b' class SourceModuleMainTest:'
 # defined, if any code is inserted above, the following line will need to be
 # updated.  Do NOT insert any whitespace between the next line and the function
 # definition below.
-THIS_LINE_NUMBER = 46  # Put here the actual number of this line
+THIS_LINE_NUMBER = 47  # Put here the actual number of this line
 
 
 def test_find_source_lines():
@@ -345,6 +346,70 b' def test_pdef():'
     inspector.pdef(foo, 'foo')
 
 
+@contextmanager
+def cleanup_user_ns(**kwargs):
+    """
+    On exit delete all the keys that were not in user_ns before entering.
+
+    It does not restore old values !
+
+    Parameters
+    ----------
+
+    **kwargs
+        used to update ip.user_ns
+
+    """
+    try:
+        known = set(ip.user_ns.keys())
+        ip.user_ns.update(kwargs)
+        yield
+    finally:
+        added = set(ip.user_ns.keys()) - known
+        for k in added:
+            del ip.user_ns[k]
+
+
+def test_pinfo_getindex():
+    def dummy():
+        """
+        MARKER
+        """
+
+    container = [dummy]
+    with cleanup_user_ns(container=container):
+        with AssertPrints("MARKER"):
+            ip._inspect("pinfo", "container[0]", detail_level=0)
+    assert "container" not in ip.user_ns.keys()
+
+
+def test_qmark_getindex():
+    def dummy():
+        """
+        MARKER 2
+        """
+
+    container = [dummy]
+    with cleanup_user_ns(container=container):
+        with AssertPrints("MARKER 2"):
+            ip.run_cell("container[0]?")
+    assert "container" not in ip.user_ns.keys()
+
+
+def test_qmark_getindex_negatif():
+    def dummy():
+        """
+        MARKER 3
+        """
+
+    container = [dummy]
+    with cleanup_user_ns(container=container):
+        with AssertPrints("MARKER 3"):
+            ip.run_cell("container[-1]?")
+    assert "container" not in ip.user_ns.keys()
+
+
+
 def test_pinfo_nonascii():
     # See gh-1177
     from . import nonascii2

IPython/core/tests/test_run.py

@@ -180,13 +180,13 b' class TestMagicRunPass(tt.TempFileMixin):'
         _ip = get_ipython()
         # This fails on Windows if self.tmpfile.name has spaces or "~" in it.
         # See below and ticket https://bugs.launchpad.net/bugs/366353
-        _ip.magic('run %s' % self.fname)
+        _ip.run_line_magic("run", self.fname)
 
     def run_tmpfile_p(self):
         _ip = get_ipython()
         # This fails on Windows if self.tmpfile.name has spaces or "~" in it.
         # See below and ticket https://bugs.launchpad.net/bugs/366353
-        _ip.magic('run -p %s' % self.fname)
+        _ip.run_line_magic("run", "-p %s" % self.fname)
 
     def test_builtins_id(self):
         """Check that %run doesn't damage __builtins__ """
@@ -216,20 +216,20 b' class TestMagicRunPass(tt.TempFileMixin):'
     def test_run_debug_twice(self):
         # https://github.com/ipython/ipython/issues/10028
         _ip = get_ipython()
-        with tt.fake_input(['c']):
-            _ip.magic('run -d %s' % self.fname)
-        with tt.fake_input(['c']):
-            _ip.magic('run -d %s' % self.fname)
+        with tt.fake_input(["c"]):
+            _ip.run_line_magic("run", "-d %s" % self.fname)
+        with tt.fake_input(["c"]):
+            _ip.run_line_magic("run", "-d %s" % self.fname)
 
     def test_run_debug_twice_with_breakpoint(self):
         """Make a valid python temp file."""
         _ip = get_ipython()
-        with tt.fake_input(['b 2', 'c', 'c']):
-            _ip.magic('run -d %s' % self.fname)
+        with tt.fake_input(["b 2", "c", "c"]):
+            _ip.run_line_magic("run", "-d %s" % self.fname)
 
-        with tt.fake_input(['c']):
-            with tt.AssertNotPrints('KeyError'):
-                _ip.magic('run -d %s' % self.fname)
+        with tt.fake_input(["c"]):
+            with tt.AssertNotPrints("KeyError"):
+                _ip.run_line_magic("run", "-d %s" % self.fname)
 
 
 class TestMagicRunSimple(tt.TempFileMixin):
@@ -239,7 +239,7 b' class TestMagicRunSimple(tt.TempFileMixin):'
         src = ("class foo: pass\n"
                "def f(): return foo()")
         self.mktmp(src)
-        _ip.magic("run %s" % self.fname)
+        _ip.run_line_magic("run", str(self.fname))
         _ip.run_cell("t = isinstance(f(), foo)")
         assert _ip.user_ns["t"] is True
 
@@ -277,7 +277,7 b' class TestMagicRunSimple(tt.TempFileMixin):'
                 "       break\n" % ("run " + empty.fname)
             )
             self.mktmp(src)
-            _ip.magic("run %s" % self.fname)
+            _ip.run_line_magic("run", str(self.fname))
             _ip.run_cell("ip == get_ipython()")
             assert _ip.user_ns["i"] == 4
 
@@ -288,8 +288,8 b' class TestMagicRunSimple(tt.TempFileMixin):'
         with tt.TempFileMixin() as empty:
             empty.mktmp("")
 
-            _ip.magic("run %s" % self.fname)
-            _ip.magic("run %s" % empty.fname)
+            _ip.run_line_magic("run", self.fname)
+            _ip.run_line_magic("run", empty.fname)
             assert _ip.user_ns["afunc"]() == 1
 
     def test_tclass(self):
@@ -323,22 +323,22 b' tclass.py: deleting object: C-third'
         self.mktmp(src)
         _ip.run_cell("zz = 23")
         try:
-            _ip.magic("run -i %s" % self.fname)
+            _ip.run_line_magic("run", "-i %s" % self.fname)
             assert _ip.user_ns["yy"] == 23
         finally:
-            _ip.magic('reset -f')
+            _ip.run_line_magic("reset", "-f")
 
         _ip.run_cell("zz = 23")
         try:
-            _ip.magic("run -i %s" % self.fname)
+            _ip.run_line_magic("run", "-i %s" % self.fname)
             assert _ip.user_ns["yy"] == 23
         finally:
-            _ip.magic('reset -f')
+            _ip.run_line_magic("reset", "-f")
 
     def test_unicode(self):
         """Check that files in odd encodings are accepted."""
         mydir = os.path.dirname(__file__)
-        na = os.path.join(mydir, 'nonascii.py')
+        na = os.path.join(mydir, "nonascii.py")
         _ip.magic('run "%s"' % na)
         assert _ip.user_ns["u"] == "Ўт№Ф"
 
@@ -347,9 +347,9 b' tclass.py: deleting object: C-third'
         src = "t = __file__\n"
         self.mktmp(src)
         _missing = object()
-        file1 = _ip.user_ns.get('__file__', _missing)
-        _ip.magic('run %s' % self.fname)
-        file2 = _ip.user_ns.get('__file__', _missing)
+        file1 = _ip.user_ns.get("__file__", _missing)
+        _ip.run_line_magic("run", self.fname)
+        file2 = _ip.user_ns.get("__file__", _missing)
 
         # Check that __file__ was equal to the filename in the script's
         # namespace.
@@ -363,9 +363,9 b' tclass.py: deleting object: C-third'
         src = "t = __file__\n"
         self.mktmp(src, ext='.ipy')
         _missing = object()
-        file1 = _ip.user_ns.get('__file__', _missing)
-        _ip.magic('run %s' % self.fname)
-        file2 = _ip.user_ns.get('__file__', _missing)
+        file1 = _ip.user_ns.get("__file__", _missing)
+        _ip.run_line_magic("run", self.fname)
+        file2 = _ip.user_ns.get("__file__", _missing)
 
         # Check that __file__ was equal to the filename in the script's
         # namespace.
@@ -378,18 +378,18 b' tclass.py: deleting object: C-third'
         """ Test that %run -t -N<N> does not raise a TypeError for N > 1."""
         src = "pass"
         self.mktmp(src)
-        _ip.magic('run -t -N 1 %s' % self.fname)
-        _ip.magic('run -t -N 10 %s' % self.fname)
+        _ip.run_line_magic("run", "-t -N 1 %s" % self.fname)
+        _ip.run_line_magic("run", "-t -N 10 %s" % self.fname)
 
     def test_ignore_sys_exit(self):
         """Test the -e option to ignore sys.exit()"""
         src = "import sys; sys.exit(1)"
         self.mktmp(src)
-        with tt.AssertPrints('SystemExit'):
-            _ip.magic('run %s' % self.fname)
+        with tt.AssertPrints("SystemExit"):
+            _ip.run_line_magic("run", self.fname)
 
-        with tt.AssertNotPrints('SystemExit'):
-            _ip.magic('run -e %s' % self.fname)
+        with tt.AssertNotPrints("SystemExit"):
+            _ip.run_line_magic("run", "-e %s" % self.fname)
 
     def test_run_nb(self):
         """Test %run notebook.ipynb"""
@@ -404,7 +404,7 b' tclass.py: deleting object: C-third'
         src = writes(nb, version=4)
         self.mktmp(src, ext='.ipynb')
 
-        _ip.magic("run %s" % self.fname)
+        _ip.run_line_magic("run", self.fname)
 
         assert _ip.user_ns["answer"] == 42
 
@@ -478,12 +478,16 b' class TestMagicRunWithPackage(unittest.TestCase):'
         sys.path[:] = [p for p in sys.path if p != self.tempdir.name]
         self.tempdir.cleanup()
 
-    def check_run_submodule(self, submodule, opts=''):
-        _ip.user_ns.pop('x', None)
-        _ip.magic('run {2} -m {0}.{1}'.format(self.package, submodule, opts))
-        self.assertEqual(_ip.user_ns['x'], self.value,
-                         'Variable `x` is not loaded from module `{0}`.'
-                         .format(submodule))
+    def check_run_submodule(self, submodule, opts=""):
+        _ip.user_ns.pop("x", None)
+        _ip.run_line_magic(
+            "run", "{2} -m {0}.{1}".format(self.package, submodule, opts)
+        )
+        self.assertEqual(
+            _ip.user_ns["x"],
+            self.value,
+            "Variable `x` is not loaded from module `{0}`.".format(submodule),
+        )
 
     def test_run_submodule_with_absolute_import(self):
         self.check_run_submodule('absolute')
@@ -533,17 +537,17 b' def test_run__name__():'
             f.write("q = __name__")
 
         _ip.user_ns.pop("q", None)
-        _ip.magic("run {}".format(path))
+        _ip.run_line_magic("run", "{}".format(path))
         assert _ip.user_ns.pop("q") == "__main__"
 
-        _ip.magic("run -n {}".format(path))
+        _ip.run_line_magic("run", "-n {}".format(path))
         assert _ip.user_ns.pop("q") == "foo"
 
         try:
-            _ip.magic("run -i -n {}".format(path))
+            _ip.run_line_magic("run", "-i -n {}".format(path))
             assert _ip.user_ns.pop("q") == "foo"
         finally:
-            _ip.magic('reset -f')
+            _ip.run_line_magic("reset", "-f")
 
 
 def test_run_tb():
@@ -563,7 +567,7 b' def test_run_tb():'
                 )
             )
         with capture_output() as io:
-            _ip.magic('run {}'.format(path))
+            _ip.run_line_magic("run", "{}".format(path))
         out = io.stdout
         assert "execfile" not in out
         assert "RuntimeError" in out

IPython/core/ultratb.py

@@ -610,6 +610,8 b' class VerboseTB(TBTools):'
     traceback, to be used with alternate interpreters (because their own code
     would appear in the traceback)."""
 
+    _tb_highlight = "bg:ansiyellow"
+
     def __init__(
         self,
         color_scheme: str = "Linux",
@@ -642,10 +644,8 b' class VerboseTB(TBTools):'
         self.long_header = long_header
         self.include_vars = include_vars
         # By default we use linecache.checkcache, but the user can provide a
-        # different check_cache implementation.  This is used by the IPython
-        # kernel to provide tracebacks for interactive code that is cached,
-        # by a compiler instance that flushes the linecache but preserves its
-        # own code cache.
+        # different check_cache implementation.  This was formerly used by the
+        # IPython kernel for interactive code, but is no longer necessary.
         if check_cache is None:
             check_cache = linecache.checkcache
         self.check_cache = check_cache
@@ -836,7 +836,7 b' class VerboseTB(TBTools):'
         before = context - after
         if self.has_colors:
             style = get_style_by_name("default")
-            style = stack_data.style_with_executing_node(style, "bg:ansiyellow")
+            style = stack_data.style_with_executing_node(style, self._tb_highlight)
             formatter = Terminal256Formatter(style=style)
         else:
             formatter = None

IPython/extensions/autoreload.py

@@ -107,6 +107,10 b' Some of the known remaining caveats are:'
   before it is reloaded are not upgraded.
 
 - C extension modules cannot be reloaded, and so cannot be autoreloaded.
+
+- While comparing Enum and Flag, the 'is' Identity Operator is used (even in the case '==' has been used (Similar to the 'None' keyword)).
+
+- Reloading a module, or importing the same module by a different name, creates new Enums. These may look the same, but are not.
 """
 
 from IPython.core.magic import Magics, magics_class, line_magic

IPython/lib/tests/test_imports.py

@@ -1,11 +1,14 b''
 # encoding: utf-8
 from IPython.testing import decorators as dec
 
+
 def test_import_backgroundjobs():
     from IPython.lib import backgroundjobs
 
+
 def test_import_deepreload():
     from IPython.lib import deepreload
 
+
 def test_import_demo():
     from IPython.lib import demo

IPython/sphinxext/ipython_directive.py

@@ -981,8 +981,9 b' class IPythonDirective(Directive):'
         self.shell.warning_is_error = warning_is_error
 
         # setup bookmark for saving figures directory
-        self.shell.process_input_line('bookmark ipy_savedir %s'%savefig_dir,
-                                      store_history=False)
+        self.shell.process_input_line(
+            'bookmark ipy_savedir "%s"' % savefig_dir, store_history=False
+        )
         self.shell.clear_cout()
 
         return rgxin, rgxout, promptin, promptout

IPython/terminal/interactiveshell.py

@@ -405,14 +405,14 b' class TerminalInteractiveShell(InteractiveShell):'
     @observe('term_title')
     def init_term_title(self, change=None):
         # Enable or disable the terminal title.
-        if self.term_title:
+        if self.term_title and _is_tty:
             toggle_set_term_title(True)
             set_term_title(self.term_title_format.format(cwd=abbrev_cwd()))
         else:
             toggle_set_term_title(False)
 
     def restore_term_title(self):
-        if self.term_title:
+        if self.term_title and _is_tty:
             restore_term_title()
 
     def init_display_formatter(self):
@@ -711,9 +711,8 b' class TerminalInteractiveShell(InteractiveShell):'
 
     active_eventloop = None
     def enable_gui(self, gui=None):
-        if gui and (gui != 'inline') :
-            self.active_eventloop, self._inputhook =\
-                get_inputhook_name_and_func(gui)
+        if gui and (gui not in {"inline", "webagg"}):
+            self.active_eventloop, self._inputhook = get_inputhook_name_and_func(gui)
         else:
             self.active_eventloop = self._inputhook = None
 

IPython/terminal/ipapp.py

@@ -318,6 +318,7 b' class TerminalIPythonApp(BaseIPythonApplication, InteractiveShellApp):'
             self.shell.mainloop()
         else:
             self.log.debug("IPython not interactive...")
+            self.shell.restore_term_title()
             if not self.shell.last_execution_succeeded:
                 sys.exit(1)
 

IPython/terminal/magics.py

@@ -41,7 +41,7 b' class TerminalMagics(Magics):'
     def __init__(self, shell):
         super(TerminalMagics, self).__init__(shell)
 
-    def store_or_execute(self, block, name):
+    def store_or_execute(self, block, name, store_history=False):
         """ Execute a block, or store it in a variable, per the user's request.
         """
         if name:
@@ -53,7 +53,7 b' class TerminalMagics(Magics):'
             self.shell.user_ns['pasted_block'] = b
             self.shell.using_paste_magics = True
             try:
-                self.shell.run_cell(b, store_history=True)
+                self.shell.run_cell(b, store_history)
             finally:
                 self.shell.using_paste_magics = False
 
@@ -147,7 +147,7 b' class TerminalMagics(Magics):'
 
         sentinel = opts.get('s', u'--')
         block = '\n'.join(get_pasted_lines(sentinel, quiet=quiet))
-        self.store_or_execute(block, name)
+        self.store_or_execute(block, name, store_history=False)
 
     @line_magic
     def paste(self, parameter_s=''):
@@ -203,7 +203,7 b' class TerminalMagics(Magics):'
                 sys.stdout.write("\n")
             sys.stdout.write("## -- End pasted text --\n")
 
-        self.store_or_execute(block, name)
+        self.store_or_execute(block, name, store_history=True)
 
     # Class-level: add a '%cls' magic only on Windows
     if sys.platform == 'win32':

IPython/terminal/pt_inputhooks/asyncio.py

@@ -31,8 +31,7 b' from prompt_toolkit import __version__ as ptk_version'
 
 from IPython.core.async_helpers import get_asyncio_loop
 
-PTK3 = ptk_version.startswith('3.')
-
+PTK3 = ptk_version.startswith("3.")
 
 
 def inputhook(context):

IPython/terminal/pt_inputhooks/gtk.py

@@ -41,6 +41,7 b' import gtk, gobject'
 # Enable threading in GTK. (Otherwise, GTK will keep the GIL.)
 gtk.gdk.threads_init()
 
+
 def inputhook(context):
     """
     When the eventloop of prompt-toolkit is idle, call this inputhook.
@@ -50,6 +51,7 b' def inputhook(context):'
 
     :param context: An `InputHookContext` instance.
     """
+
     def _main_quit(*a, **kw):
         gtk.main_quit()
         return False

IPython/terminal/pt_inputhooks/gtk3.py

@@ -3,10 +3,12 b''
 
 from gi.repository import Gtk, GLib
 
+
 def _main_quit(*args, **kwargs):
     Gtk.main_quit()
     return False
 
+
 def inputhook(context):
     GLib.io_add_watch(context.fileno(), GLib.PRIORITY_DEFAULT, GLib.IO_IN, _main_quit)
     Gtk.main()

IPython/testing/plugin/simplevars.py

@@ -1,2 +1,2 b''
 x = 1
-print('x is:',x)
+print("x is:", x)

IPython/utils/_sysinfo.py

@@ -1,2 +1,2 b''
 # GENERATED BY setup.py
-commit = u""
+commit = ""

IPython/utils/decorators.py

@@ -2,7 +2,7 b''
 """Decorators that don't go anywhere else.
 
 This module contains misc. decorators that don't really go with another module
-in :mod:`IPython.utils`. Beore putting something here please see if it should
+in :mod:`IPython.utils`. Before putting something here please see if it should
 go into another topical module in :mod:`IPython.utils`.
 """
 
@@ -16,6 +16,10 b' go into another topical module in :mod:`IPython.utils`.'
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
+from typing import Sequence
+
+from IPython.utils.docs import GENERATING_DOCUMENTATION
+
 
 #-----------------------------------------------------------------------------
 # Code
@@ -48,6 +52,7 b' def flag_calls(func):'
     wrapper.__doc__ = func.__doc__
     return wrapper
 
+
 def undoc(func):
     """Mark a function or class as undocumented.
 
@@ -56,3 +61,23 b' def undoc(func):'
     """
     return func
 
+
+def sphinx_options(
+    show_inheritance: bool = True,
+    show_inherited_members: bool = False,
+    exclude_inherited_from: Sequence[str] = tuple(),
+):
+    """Set sphinx options"""
+
+    def wrapper(func):
+        if not GENERATING_DOCUMENTATION:
+            return func
+
+        func._sphinx_options = dict(
+            show_inheritance=show_inheritance,
+            show_inherited_members=show_inherited_members,
+            exclude_inherited_from=exclude_inherited_from,
+        )
+        return func
+
+    return wrapper

IPython/utils/terminal.py

@@ -62,15 +62,27 b' def _restore_term_title():'
     pass
 
 
+_xterm_term_title_saved = False
+
+
 def _set_term_title_xterm(title):
     """ Change virtual terminal title in xterm-workalikes """
+    global _xterm_term_title_saved
+    # Only save the title the first time we set, otherwise restore will only
+    # go back one title (probably undoing a %cd title change).
+    if not _xterm_term_title_saved:
     # save the current title to the xterm "stack"
-    sys.stdout.write('\033[22;0t') 
+        sys.stdout.write("\033[22;0t")
+        _xterm_term_title_saved = True
     sys.stdout.write('\033]0;%s\007' % title)
 
 
 def _restore_term_title_xterm():
+    # Make sure the restore has at least one accompanying set.
+    global _xterm_term_title_saved
+    assert _xterm_term_title_saved
     sys.stdout.write('\033[23;0t') 
+    _xterm_term_title_saved = False
 
 
 if os.name == 'posix':

IPython/utils/tests/test_dir2.py

@@ -19,7 +19,6 b' def test_base():'
 
 
 def test_SubClass():
-
     class SubClass(Base):
         y = 2
 
@@ -53,7 +52,7 b' def test_misbehaving_object_without_trait_names():'
 
     class SillierWithDir(MisbehavingGetattr):
         def __dir__(self):
-            return ['some_method']
+            return ["some_method"]
 
     for bad_klass in (MisbehavingGetattr, SillierWithDir):
         obj = bad_klass()

README.rst

@@ -13,8 +13,8 b''
 .. image:: https://raster.shields.io/badge/Follows-NEP29-brightgreen.png
     :target: https://numpy.org/neps/nep-0029-deprecation_policy.html
 
-.. image:: https://tidelift.com/subscription/pkg/pypi-ipython
-    :target: https://tidelift.com/badges/package/pypi/ipython?style=flat
+.. image:: https://tidelift.com/badges/package/pypi/ipython?style=flat
+    :target: https://tidelift.com/subscription/pkg/pypi-ipython
 
 
 ===========================================

docs/source/conf.py

@@ -41,6 +41,14 b' else:'
     html_theme = "sphinx_rtd_theme"
     html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
 
+# Allow Python scripts to change behaviour during sphinx run
+os.environ["IN_SPHINX_RUN"] = "True"
+
+autodoc_type_aliases = {
+    "Matcher": " IPython.core.completer.Matcher",
+    "MatcherAPIv1": " IPython.core.completer.MatcherAPIv1",
+}
+
 # If your extensions are in another directory, add it here. If the directory
 # is relative to the documentation root, use os.path.abspath to make it
 # absolute, like shown here.

docs/source/config/details.rst

@@ -69,7 +69,7 b' shell:'
 
     /home/bob >>> # it works
 
-See ``IPython/example/utils/cwd_prompt.py`` for an example of how to write an
+See ``IPython/example/utils/cwd_prompt.py`` for an example of how to write
 extensions to customise prompts.
 
 Inside IPython or in a startup script, you can use a custom prompts class

docs/source/config/extensions/index.rst

@@ -6,8 +6,7 b' IPython extensions'
 
 A level above configuration are IPython extensions, Python modules which modify
 the behaviour of the shell. They are referred to by an importable module name,
-and can be placed anywhere you'd normally import from, or in
-``.ipython/extensions/``.
+and can be placed anywhere you'd normally import from.
 
 Getting extensions
 ==================
@@ -71,10 +70,7 b' Useful :class:`InteractiveShell` methods include :meth:`~IPython.core.interactiv'
    :ref:`defining_magics`
 
 You can put your extension modules anywhere you want, as long as they can be
-imported by Python's standard import mechanism. However, to make it easy to
-write extensions, you can also put your extensions in :file:`extensions/`
-within the :ref:`IPython directory <ipythondir>`. This directory is
-added to :data:`sys.path` automatically.
+imported by Python's standard import mechanism.
 
 When your extension is ready for general use, please add it to the `extensions
 index <https://github.com/ipython/ipython/wiki/Extensions-Index>`_. We also

docs/source/whatsnew/version8.rst

@@ -2,6 +2,70 b''
  8.x Series
 ============
 
+.. _version 8.6.0:
+
+IPython 8.6.0
+-------------
+
+Back to a more regular release schedule (at least I try), as Friday is
+already over by more than 24h hours. This is a slightly bigger release with a
+few new features that contain no less then 25 PRs.
+
+We'll notably found a couple of non negligible changes:
+
+The ``install_ext`` and related functions have been removed after being
+deprecated for years. You can use pip to install extensions. ``pip`` did not
+exists when ``install_ext`` was introduced. You can still load local extensions
+without installing them. Just set your ``sys.path`` for example. :ghpull:`13744`
+
+IPython now have extra entry points that that the major *and minor* version of
+python. For some of you this mean that you can do a quick ``ipython3.10`` to
+launch IPython from the Python 3.10 interpreter, while still using Python 3.11
+as your main Python. :ghpull:`13743`
+
+The completer matcher API have been improved. See :ghpull:`13745`. This should
+improve the type inference and improve dict keys completions in many use case.
+Tanks ``@krassowski`` for all the works, and the D.E. Shaw group for sponsoring
+it.
+
+The color of error nodes in tracebacks can now be customized. See
+:ghpull:`13756`. This is a private attribute until someone find the time to
+properly add a configuration option. Note that with Python 3.11 that also show
+the relevant nodes in traceback, it would be good to leverage this informations
+(plus the "did you mean" info added on attribute errors). But that's likely work
+I won't have time to do before long, so contributions welcome.
+
+As we follow NEP 29, we removed support for numpy 1.19 :ghpull:`13760`.
+
+
+The ``open()`` function present in the user namespace by default will now refuse
+to open the file descriptors 0,1,2 (stdin, out, err), to avoid crashing IPython.
+This mostly occurs in teaching context when incorrect values get passed around. 
+
+
+The ``?``, ``??``, and corresponding ``pinfo``, ``pinfo2`` magics can now find
+objects insides arrays. That is to say, the following now works::
+
+
+   >>> def my_func(*arg, **kwargs):pass
+   >>> container = [my_func]
+   >>> container[0]?
+
+
+If ``container`` define a custom ``getitem``, this __will__ trigger the custom
+method. So don't put side effects in your ``getitems``. Thanks the D.E. Shaw
+group for the request and sponsoring the work.
+
+
+As usual you can find the full list of PRs on GitHub under `the 8.6 milestone
+<https://github.com/ipython/ipython/pulls?q=milestone%3A8.6>`__.
+
+Thanks to all hacktoberfest contributors, please contribute to
+`closember.org <https://closember.org/>`__.
+
+Thanks to the `D. E. Shaw group <https://deshaw.com/>`__ for sponsoring
+work on IPython and related libraries.
+
 .. _version 8.5.0:
 
 IPython 8.5.0

docs/sphinxext/apigen.py

@@ -24,14 +24,9 b' import inspect'
 import os
 import re
 from importlib import import_module
+from types import SimpleNamespace as Obj
 
 
-class Obj(object):
-    '''Namespace to hold arbitrary information.'''
-    def __init__(self, **kwargs):
-        for k, v in kwargs.items():
-            setattr(self, k, v)
-                                        
 class FuncClsScanner(ast.NodeVisitor):
     """Scan a module for top-level functions and classes.
     
@@ -62,11 +57,15 b' class FuncClsScanner(ast.NodeVisitor):'
             self.functions.append(node.name)
     
     def visit_ClassDef(self, node):
-        if not (node.name.startswith('_') or self.has_undoc_decorator(node)) \
-                and node.name not in self.classes_seen:
-            cls = Obj(name=node.name)
-            cls.has_init = any(isinstance(n, ast.FunctionDef) and \
-                                n.name=='__init__' for n in node.body)
+        if (
+            not (node.name.startswith("_") or self.has_undoc_decorator(node))
+            and node.name not in self.classes_seen
+        ):
+            cls = Obj(name=node.name, sphinx_options={})
+            cls.has_init = any(
+                isinstance(n, ast.FunctionDef) and n.name == "__init__"
+                for n in node.body
+            )
             self.classes.append(cls)
             self.classes_seen.add(node.name)
     
@@ -221,7 +220,11 b' class ApiDocWriter(object):'
         funcs, classes = [], []
         for name, obj in ns.items():
             if inspect.isclass(obj):
-                cls = Obj(name=name, has_init='__init__' in obj.__dict__)
+                cls = Obj(
+                    name=name,
+                    has_init="__init__" in obj.__dict__,
+                    sphinx_options=getattr(obj, "_sphinx_options", {}),
+                )
                 classes.append(cls)
             elif inspect.isfunction(obj):
                 funcs.append(name)
@@ -279,10 +282,18 b' class ApiDocWriter(object):'
                   self.rst_section_levels[2] * len(subhead) + '\n'
 
         for c in classes:
-            ad += '\n.. autoclass:: ' + c.name + '\n'
+            opts = c.sphinx_options
+            ad += "\n.. autoclass:: " + c.name + "\n"
             # must NOT exclude from index to keep cross-refs working
-            ad += '  :members:\n' \
-                  '  :show-inheritance:\n'
+            ad += "  :members:\n"
+            if opts.get("show_inheritance", True):
+                ad += "  :show-inheritance:\n"
+            if opts.get("show_inherited_members", False):
+                exclusions_list = opts.get("exclude_inherited_from", [])
+                exclusions = (
+                    (" " + " ".join(exclusions_list)) if exclusions_list else ""
+                )
+                ad += f"  :inherited-members:{exclusions}\n"
             if c.has_init:
                   ad += '\n  .. automethod:: __init__\n'
         

setup.cfg

@@ -54,6 +54,7 b' doc ='
     matplotlib
     stack_data
     pytest<7
+    typing_extensions
     %(test)s
 kernel =
     ipykernel
@@ -78,7 +79,7 b' test_extra ='
     curio
     matplotlib!=3.2.0
     nbformat
-    numpy>=1.19
+    numpy>=1.20
     pandas
     trio
 all =

setupbase.py

@@ -211,14 +211,20 b' def find_entry_points():'
     use, our own build_scripts_entrypt class below parses these and builds
     command line scripts.
 
-    Each of our entry points gets both a plain name, e.g. ipython, and one
-    suffixed with the Python major version number, e.g. ipython3.
+    Each of our entry points gets a plain name, e.g. ipython, a name
+    suffixed with the Python major version number, e.g. ipython3, and
+    a name suffixed with the Python major.minor version number, eg. ipython3.8.
     """
     ep = [
             'ipython%s = IPython:start_ipython',
         ]
-    suffix = str(sys.version_info[0])
-    return [e % '' for e in ep] + [e % suffix for e in ep]
+    major_suffix = str(sys.version_info[0])
+    minor_suffix = ".".join([str(sys.version_info[0]), str(sys.version_info[1])])
+    return (
+        [e % "" for e in ep]
+        + [e % major_suffix for e in ep]
+        + [e % minor_suffix for e in ep]
+    )
 
 class install_lib_symlink(Command):
     user_options = [
@@ -340,7 +346,7 b' def git_prebuild(pkg_dir, build_cmd=build_py):'
                 out_file.writelines(
                     [
                         "# GENERATED BY setup.py\n",
-                        'commit = u"%s"\n' % repo_commit,
+                        'commit = "%s"\n' % repo_commit,
                     ]
                 )