Merge branch 'ipython:main' into main

IPython/terminal/shortcuts/__init__.py

@@ -0,0 +1,670 b''
+"""
+Module to define and register Terminal IPython shortcuts with
+:mod:`prompt_toolkit`
+"""
+
+# Copyright (c) IPython Development Team.
+# Distributed under the terms of the Modified BSD License.
+
+import os
+import re
+import signal
+import sys
+import warnings
+from typing import Callable, Dict, Union
+
+from prompt_toolkit.application.current import get_app
+from prompt_toolkit.enums import DEFAULT_BUFFER, SEARCH_BUFFER
+from prompt_toolkit.filters import Condition, emacs_insert_mode, has_completions
+from prompt_toolkit.filters import has_focus as has_focus_impl
+from prompt_toolkit.filters import (
+    has_selection,
+    has_suggestion,
+    vi_insert_mode,
+    vi_mode,
+)
+from prompt_toolkit.key_binding import KeyBindings
+from prompt_toolkit.key_binding.bindings import named_commands as nc
+from prompt_toolkit.key_binding.bindings.completion import (
+    display_completions_like_readline,
+)
+from prompt_toolkit.key_binding.vi_state import InputMode, ViState
+from prompt_toolkit.layout.layout import FocusableElement
+
+from IPython.terminal.shortcuts import auto_match as match
+from IPython.terminal.shortcuts import auto_suggest
+from IPython.utils.decorators import undoc
+
+__all__ = ["create_ipython_shortcuts"]
+
+
+@undoc
+@Condition
+def cursor_in_leading_ws():
+    before = get_app().current_buffer.document.current_line_before_cursor
+    return (not before) or before.isspace()
+
+
+def has_focus(value: FocusableElement):
+    """Wrapper around has_focus adding a nice `__name__` to tester function"""
+    tester = has_focus_impl(value).func
+    tester.__name__ = f"is_focused({value})"
+    return Condition(tester)
+
+
+@undoc
+@Condition
+def has_line_below() -> bool:
+    document = get_app().current_buffer.document
+    return document.cursor_position_row < len(document.lines) - 1
+
+
+@undoc
+@Condition
+def has_line_above() -> bool:
+    document = get_app().current_buffer.document
+    return document.cursor_position_row != 0
+
+
+def create_ipython_shortcuts(shell, for_all_platforms: bool = False) -> KeyBindings:
+    """Set up the prompt_toolkit keyboard shortcuts for IPython.
+
+    Parameters
+    ----------
+    shell: InteractiveShell
+        The current IPython shell Instance
+    for_all_platforms: bool (default false)
+        This parameter is mostly used in generating the documentation
+        to create the shortcut binding for all the platforms, and export
+        them.
+
+    Returns
+    -------
+    KeyBindings
+        the keybinding instance for prompt toolkit.
+
+    """
+    # Warning: if possible, do NOT define handler functions in the locals
+    # scope of this function, instead define functions in the global
+    # scope, or a separate module, and include a user-friendly docstring
+    # describing the action.
+
+    kb = KeyBindings()
+    insert_mode = vi_insert_mode | emacs_insert_mode
+
+    if getattr(shell, "handle_return", None):
+        return_handler = shell.handle_return(shell)
+    else:
+        return_handler = newline_or_execute_outer(shell)
+
+    kb.add("enter", filter=(has_focus(DEFAULT_BUFFER) & ~has_selection & insert_mode))(
+        return_handler
+    )
+
+    @Condition
+    def ebivim():
+        return shell.emacs_bindings_in_vi_insert_mode
+
+    @kb.add(
+        "escape",
+        "enter",
+        filter=(has_focus(DEFAULT_BUFFER) & ~has_selection & insert_mode & ebivim),
+    )
+    def reformat_and_execute(event):
+        """Reformat code and execute it"""
+        reformat_text_before_cursor(
+            event.current_buffer, event.current_buffer.document, shell
+        )
+        event.current_buffer.validate_and_handle()
+
+    kb.add("c-\\")(quit)
+
+    kb.add("c-p", filter=(vi_insert_mode & has_focus(DEFAULT_BUFFER)))(
+        previous_history_or_previous_completion
+    )
+
+    kb.add("c-n", filter=(vi_insert_mode & has_focus(DEFAULT_BUFFER)))(
+        next_history_or_next_completion
+    )
+
+    kb.add("c-g", filter=(has_focus(DEFAULT_BUFFER) & has_completions))(
+        dismiss_completion
+    )
+
+    kb.add("c-c", filter=has_focus(DEFAULT_BUFFER))(reset_buffer)
+
+    kb.add("c-c", filter=has_focus(SEARCH_BUFFER))(reset_search_buffer)
+
+    supports_suspend = Condition(lambda: hasattr(signal, "SIGTSTP"))
+    kb.add("c-z", filter=supports_suspend)(suspend_to_bg)
+
+    # Ctrl+I == Tab
+    kb.add(
+        "tab",
+        filter=(
+            has_focus(DEFAULT_BUFFER)
+            & ~has_selection
+            & insert_mode
+            & cursor_in_leading_ws
+        ),
+    )(indent_buffer)
+    kb.add("c-o", filter=(has_focus(DEFAULT_BUFFER) & emacs_insert_mode))(
+        newline_autoindent_outer(shell.input_transformer_manager)
+    )
+
+    kb.add("f2", filter=has_focus(DEFAULT_BUFFER))(open_input_in_editor)
+
+    @Condition
+    def auto_match():
+        return shell.auto_match
+
+    def all_quotes_paired(quote, buf):
+        paired = True
+        i = 0
+        while i < len(buf):
+            c = buf[i]
+            if c == quote:
+                paired = not paired
+            elif c == "\\":
+                i += 1
+            i += 1
+        return paired
+
+    focused_insert = (vi_insert_mode | emacs_insert_mode) & has_focus(DEFAULT_BUFFER)
+    _preceding_text_cache: Dict[Union[str, Callable], Condition] = {}
+    _following_text_cache: Dict[Union[str, Callable], Condition] = {}
+
+    def preceding_text(pattern: Union[str, Callable]):
+        if pattern in _preceding_text_cache:
+            return _preceding_text_cache[pattern]
+
+        if callable(pattern):
+
+            def _preceding_text():
+                app = get_app()
+                before_cursor = app.current_buffer.document.current_line_before_cursor
+                # mypy can't infer if(callable): https://github.com/python/mypy/issues/3603
+                return bool(pattern(before_cursor))  # type: ignore[operator]
+
+        else:
+            m = re.compile(pattern)
+
+            def _preceding_text():
+                app = get_app()
+                before_cursor = app.current_buffer.document.current_line_before_cursor
+                return bool(m.match(before_cursor))
+
+            _preceding_text.__name__ = f"preceding_text({pattern!r})"
+
+        condition = Condition(_preceding_text)
+        _preceding_text_cache[pattern] = condition
+        return condition
+
+    def following_text(pattern):
+        try:
+            return _following_text_cache[pattern]
+        except KeyError:
+            pass
+        m = re.compile(pattern)
+
+        def _following_text():
+            app = get_app()
+            return bool(m.match(app.current_buffer.document.current_line_after_cursor))
+
+        _following_text.__name__ = f"following_text({pattern!r})"
+
+        condition = Condition(_following_text)
+        _following_text_cache[pattern] = condition
+        return condition
+
+    @Condition
+    def not_inside_unclosed_string():
+        app = get_app()
+        s = app.current_buffer.document.text_before_cursor
+        # remove escaped quotes
+        s = s.replace('\\"', "").replace("\\'", "")
+        # remove triple-quoted string literals
+        s = re.sub(r"(?:\"\"\"[\s\S]*\"\"\"|'''[\s\S]*''')", "", s)
+        # remove single-quoted string literals
+        s = re.sub(r"""(?:"[^"]*["\n]|'[^']*['\n])""", "", s)
+        return not ('"' in s or "'" in s)
+
+    # auto match
+    for key, cmd in match.auto_match_parens.items():
+        kb.add(key, filter=focused_insert & auto_match & following_text(r"[,)}\]]|$"))(
+            cmd
+        )
+
+    # raw string
+    for key, cmd in match.auto_match_parens_raw_string.items():
+        kb.add(
+            key,
+            filter=focused_insert & auto_match & preceding_text(r".*(r|R)[\"'](-*)$"),
+        )(cmd)
+
+    kb.add(
+        '"',
+        filter=focused_insert
+        & auto_match
+        & not_inside_unclosed_string
+        & preceding_text(lambda line: all_quotes_paired('"', line))
+        & following_text(r"[,)}\]]|$"),
+    )(match.double_quote)
+
+    kb.add(
+        "'",
+        filter=focused_insert
+        & auto_match
+        & not_inside_unclosed_string
+        & preceding_text(lambda line: all_quotes_paired("'", line))
+        & following_text(r"[,)}\]]|$"),
+    )(match.single_quote)
+
+    kb.add(
+        '"',
+        filter=focused_insert
+        & auto_match
+        & not_inside_unclosed_string
+        & preceding_text(r'^.*""$'),
+    )(match.docstring_double_quotes)
+
+    kb.add(
+        "'",
+        filter=focused_insert
+        & auto_match
+        & not_inside_unclosed_string
+        & preceding_text(r"^.*''$"),
+    )(match.docstring_single_quotes)
+
+    # just move cursor
+    kb.add(")", filter=focused_insert & auto_match & following_text(r"^\)"))(
+        match.skip_over
+    )
+    kb.add("]", filter=focused_insert & auto_match & following_text(r"^\]"))(
+        match.skip_over
+    )
+    kb.add("}", filter=focused_insert & auto_match & following_text(r"^\}"))(
+        match.skip_over
+    )
+    kb.add('"', filter=focused_insert & auto_match & following_text('^"'))(
+        match.skip_over
+    )
+    kb.add("'", filter=focused_insert & auto_match & following_text("^'"))(
+        match.skip_over
+    )
+
+    kb.add(
+        "backspace",
+        filter=focused_insert
+        & preceding_text(r".*\($")
+        & auto_match
+        & following_text(r"^\)"),
+    )(match.delete_pair)
+    kb.add(
+        "backspace",
+        filter=focused_insert
+        & preceding_text(r".*\[$")
+        & auto_match
+        & following_text(r"^\]"),
+    )(match.delete_pair)
+    kb.add(
+        "backspace",
+        filter=focused_insert
+        & preceding_text(r".*\{$")
+        & auto_match
+        & following_text(r"^\}"),
+    )(match.delete_pair)
+    kb.add(
+        "backspace",
+        filter=focused_insert
+        & preceding_text('.*"$')
+        & auto_match
+        & following_text('^"'),
+    )(match.delete_pair)
+    kb.add(
+        "backspace",
+        filter=focused_insert
+        & preceding_text(r".*'$")
+        & auto_match
+        & following_text(r"^'"),
+    )(match.delete_pair)
+
+    if shell.display_completions == "readlinelike":
+        kb.add(
+            "c-i",
+            filter=(
+                has_focus(DEFAULT_BUFFER)
+                & ~has_selection
+                & insert_mode
+                & ~cursor_in_leading_ws
+            ),
+        )(display_completions_like_readline)
+
+    if sys.platform == "win32" or for_all_platforms:
+        kb.add("c-v", filter=(has_focus(DEFAULT_BUFFER) & ~vi_mode))(win_paste)
+
+    focused_insert_vi = has_focus(DEFAULT_BUFFER) & vi_insert_mode
+
+    # autosuggestions
+    @Condition
+    def navigable_suggestions():
+        return isinstance(
+            shell.auto_suggest, auto_suggest.NavigableAutoSuggestFromHistory
+        )
+
+    kb.add("end", filter=has_focus(DEFAULT_BUFFER) & (ebivim | ~vi_insert_mode))(
+        auto_suggest.accept_in_vi_insert_mode
+    )
+    kb.add("c-e", filter=focused_insert_vi & ebivim)(
+        auto_suggest.accept_in_vi_insert_mode
+    )
+    kb.add("c-f", filter=focused_insert_vi)(auto_suggest.accept)
+    kb.add("escape", "f", filter=focused_insert_vi & ebivim)(auto_suggest.accept_word)
+    kb.add("c-right", filter=has_suggestion & has_focus(DEFAULT_BUFFER))(
+        auto_suggest.accept_token
+    )
+    kb.add(
+        "escape", filter=has_suggestion & has_focus(DEFAULT_BUFFER) & emacs_insert_mode
+    )(auto_suggest.discard)
+    kb.add(
+        "up",
+        filter=navigable_suggestions
+        & ~has_line_above
+        & has_suggestion
+        & has_focus(DEFAULT_BUFFER),
+    )(auto_suggest.swap_autosuggestion_up(shell.auto_suggest))
+    kb.add(
+        "down",
+        filter=navigable_suggestions
+        & ~has_line_below
+        & has_suggestion
+        & has_focus(DEFAULT_BUFFER),
+    )(auto_suggest.swap_autosuggestion_down(shell.auto_suggest))
+    kb.add(
+        "up", filter=has_line_above & navigable_suggestions & has_focus(DEFAULT_BUFFER)
+    )(auto_suggest.up_and_update_hint)
+    kb.add(
+        "down",
+        filter=has_line_below & navigable_suggestions & has_focus(DEFAULT_BUFFER),
+    )(auto_suggest.down_and_update_hint)
+    kb.add("right", filter=has_suggestion & has_focus(DEFAULT_BUFFER))(
+        auto_suggest.accept_character
+    )
+    kb.add("c-left", filter=has_suggestion & has_focus(DEFAULT_BUFFER))(
+        auto_suggest.accept_and_move_cursor_left
+    )
+    kb.add("c-down", filter=has_suggestion & has_focus(DEFAULT_BUFFER))(
+        auto_suggest.accept_and_keep_cursor
+    )
+    kb.add("backspace", filter=has_suggestion & has_focus(DEFAULT_BUFFER))(
+        auto_suggest.backspace_and_resume_hint
+    )
+
+    # Simple Control keybindings
+    key_cmd_dict = {
+        "c-a": nc.beginning_of_line,
+        "c-b": nc.backward_char,
+        "c-k": nc.kill_line,
+        "c-w": nc.backward_kill_word,
+        "c-y": nc.yank,
+        "c-_": nc.undo,
+    }
+
+    for key, cmd in key_cmd_dict.items():
+        kb.add(key, filter=focused_insert_vi & ebivim)(cmd)
+
+    # Alt and Combo Control keybindings
+    keys_cmd_dict = {
+        # Control Combos
+        ("c-x", "c-e"): nc.edit_and_execute,
+        ("c-x", "e"): nc.edit_and_execute,
+        # Alt
+        ("escape", "b"): nc.backward_word,
+        ("escape", "c"): nc.capitalize_word,
+        ("escape", "d"): nc.kill_word,
+        ("escape", "h"): nc.backward_kill_word,
+        ("escape", "l"): nc.downcase_word,
+        ("escape", "u"): nc.uppercase_word,
+        ("escape", "y"): nc.yank_pop,
+        ("escape", "."): nc.yank_last_arg,
+    }
+
+    for keys, cmd in keys_cmd_dict.items():
+        kb.add(*keys, filter=focused_insert_vi & ebivim)(cmd)
+
+    def get_input_mode(self):
+        app = get_app()
+        app.ttimeoutlen = shell.ttimeoutlen
+        app.timeoutlen = shell.timeoutlen
+
+        return self._input_mode
+
+    def set_input_mode(self, mode):
+        shape = {InputMode.NAVIGATION: 2, InputMode.REPLACE: 4}.get(mode, 6)
+        cursor = "\x1b[{} q".format(shape)
+
+        sys.stdout.write(cursor)
+        sys.stdout.flush()
+
+        self._input_mode = mode
+
+    if shell.editing_mode == "vi" and shell.modal_cursor:
+        ViState._input_mode = InputMode.INSERT  # type: ignore
+        ViState.input_mode = property(get_input_mode, set_input_mode)  # type: ignore
+    return kb
+
+
+def reformat_text_before_cursor(buffer, document, shell):
+    text = buffer.delete_before_cursor(len(document.text[: document.cursor_position]))
+    try:
+        formatted_text = shell.reformat_handler(text)
+        buffer.insert_text(formatted_text)
+    except Exception as e:
+        buffer.insert_text(text)
+
+
+def newline_or_execute_outer(shell):
+    def newline_or_execute(event):
+        """When the user presses return, insert a newline or execute the code."""
+        b = event.current_buffer
+        d = b.document
+
+        if b.complete_state:
+            cc = b.complete_state.current_completion
+            if cc:
+                b.apply_completion(cc)
+            else:
+                b.cancel_completion()
+            return
+
+        # If there's only one line, treat it as if the cursor is at the end.
+        # See https://github.com/ipython/ipython/issues/10425
+        if d.line_count == 1:
+            check_text = d.text
+        else:
+            check_text = d.text[: d.cursor_position]
+        status, indent = shell.check_complete(check_text)
+
+        # if all we have after the cursor is whitespace: reformat current text
+        # before cursor
+        after_cursor = d.text[d.cursor_position :]
+        reformatted = False
+        if not after_cursor.strip():
+            reformat_text_before_cursor(b, d, shell)
+            reformatted = True
+        if not (
+            d.on_last_line
+            or d.cursor_position_row >= d.line_count - d.empty_line_count_at_the_end()
+        ):
+            if shell.autoindent:
+                b.insert_text("\n" + indent)
+            else:
+                b.insert_text("\n")
+            return
+
+        if (status != "incomplete") and b.accept_handler:
+            if not reformatted:
+                reformat_text_before_cursor(b, d, shell)
+            b.validate_and_handle()
+        else:
+            if shell.autoindent:
+                b.insert_text("\n" + indent)
+            else:
+                b.insert_text("\n")
+
+    newline_or_execute.__qualname__ = "newline_or_execute"
+
+    return newline_or_execute
+
+
+def previous_history_or_previous_completion(event):
+    """
+    Control-P in vi edit mode on readline is history next, unlike default prompt toolkit.
+
+    If completer is open this still select previous completion.
+    """
+    event.current_buffer.auto_up()
+
+
+def next_history_or_next_completion(event):
+    """
+    Control-N in vi edit mode on readline is history previous, unlike default prompt toolkit.
+
+    If completer is open this still select next completion.
+    """
+    event.current_buffer.auto_down()
+
+
+def dismiss_completion(event):
+    """Dismiss completion"""
+    b = event.current_buffer
+    if b.complete_state:
+        b.cancel_completion()
+
+
+def reset_buffer(event):
+    """Reset buffer"""
+    b = event.current_buffer
+    if b.complete_state:
+        b.cancel_completion()
+    else:
+        b.reset()
+
+
+def reset_search_buffer(event):
+    """Reset search buffer"""
+    if event.current_buffer.document.text:
+        event.current_buffer.reset()
+    else:
+        event.app.layout.focus(DEFAULT_BUFFER)
+
+
+def suspend_to_bg(event):
+    """Suspend to background"""
+    event.app.suspend_to_background()
+
+
+def quit(event):
+    """
+    Quit application with ``SIGQUIT`` if supported or ``sys.exit`` otherwise.
+
+    On platforms that support SIGQUIT, send SIGQUIT to the current process.
+    On other platforms, just exit the process with a message.
+    """
+    sigquit = getattr(signal, "SIGQUIT", None)
+    if sigquit is not None:
+        os.kill(0, signal.SIGQUIT)
+    else:
+        sys.exit("Quit")
+
+
+def indent_buffer(event):
+    """Indent buffer"""
+    event.current_buffer.insert_text(" " * 4)
+
+
+@undoc
+def newline_with_copy_margin(event):
+    """
+    DEPRECATED since IPython 6.0
+
+    See :any:`newline_autoindent_outer` for a replacement.
+
+    Preserve margin and cursor position when using
+    Control-O to insert a newline in EMACS mode
+    """
+    warnings.warn(
+        "`newline_with_copy_margin(event)` is deprecated since IPython 6.0. "
+        "see `newline_autoindent_outer(shell)(event)` for a replacement.",
+        DeprecationWarning,
+        stacklevel=2,
+    )
+
+    b = event.current_buffer
+    cursor_start_pos = b.document.cursor_position_col
+    b.newline(copy_margin=True)
+    b.cursor_up(count=1)
+    cursor_end_pos = b.document.cursor_position_col
+    if cursor_start_pos != cursor_end_pos:
+        pos_diff = cursor_start_pos - cursor_end_pos
+        b.cursor_right(count=pos_diff)
+
+
+def newline_autoindent_outer(inputsplitter) -> Callable[..., None]:
+    """
+    Return a function suitable for inserting a indented newline after the cursor.
+
+    Fancier version of deprecated ``newline_with_copy_margin`` which should
+    compute the correct indentation of the inserted line. That is to say, indent
+    by 4 extra space after a function definition, class definition, context
+    manager... And dedent by 4 space after ``pass``, ``return``, ``raise ...``.
+    """
+
+    def newline_autoindent(event):
+        """Insert a newline after the cursor indented appropriately."""
+        b = event.current_buffer
+        d = b.document
+
+        if b.complete_state:
+            b.cancel_completion()
+        text = d.text[: d.cursor_position] + "\n"
+        _, indent = inputsplitter.check_complete(text)
+        b.insert_text("\n" + (" " * (indent or 0)), move_cursor=False)
+
+    newline_autoindent.__qualname__ = "newline_autoindent"
+
+    return newline_autoindent
+
+
+def open_input_in_editor(event):
+    """Open code from input in external editor"""
+    event.app.current_buffer.open_in_editor()
+
+
+if sys.platform == "win32":
+    from IPython.core.error import TryNext
+    from IPython.lib.clipboard import (
+        ClipboardEmpty,
+        tkinter_clipboard_get,
+        win32_clipboard_get,
+    )
+
+    @undoc
+    def win_paste(event):
+        try:
+            text = win32_clipboard_get()
+        except TryNext:
+            try:
+                text = tkinter_clipboard_get()
+            except (TryNext, ClipboardEmpty):
+                return
+        except ClipboardEmpty:
+            return
+        event.current_buffer.insert_text(text.replace("\t", " " * 4))
+
+else:
+
+    @undoc
+    def win_paste(event):
+        """Stub used when auto-generating shortcuts for documentation"""
+        pass

IPython/terminal/shortcuts/auto_match.py

@@ -0,0 +1,104 b''
+"""
+Utilities function for keybinding with prompt toolkit. 
+
+This will be bound to specific key press and filter modes, 
+like whether we are in edit mode, and whether the completer is open.
+"""
+import re
+from prompt_toolkit.key_binding import KeyPressEvent
+
+
+def parenthesis(event: KeyPressEvent):
+    """Auto-close parenthesis"""
+    event.current_buffer.insert_text("()")
+    event.current_buffer.cursor_left()
+
+
+def brackets(event: KeyPressEvent):
+    """Auto-close brackets"""
+    event.current_buffer.insert_text("[]")
+    event.current_buffer.cursor_left()
+
+
+def braces(event: KeyPressEvent):
+    """Auto-close braces"""
+    event.current_buffer.insert_text("{}")
+    event.current_buffer.cursor_left()
+
+
+def double_quote(event: KeyPressEvent):
+    """Auto-close double quotes"""
+    event.current_buffer.insert_text('""')
+    event.current_buffer.cursor_left()
+
+
+def single_quote(event: KeyPressEvent):
+    """Auto-close single quotes"""
+    event.current_buffer.insert_text("''")
+    event.current_buffer.cursor_left()
+
+
+def docstring_double_quotes(event: KeyPressEvent):
+    """Auto-close docstring (double quotes)"""
+    event.current_buffer.insert_text('""""')
+    event.current_buffer.cursor_left(3)
+
+
+def docstring_single_quotes(event: KeyPressEvent):
+    """Auto-close docstring (single quotes)"""
+    event.current_buffer.insert_text("''''")
+    event.current_buffer.cursor_left(3)
+
+
+def raw_string_parenthesis(event: KeyPressEvent):
+    """Auto-close parenthesis in raw strings"""
+    matches = re.match(
+        r".*(r|R)[\"'](-*)",
+        event.current_buffer.document.current_line_before_cursor,
+    )
+    dashes = matches.group(2) if matches else ""
+    event.current_buffer.insert_text("()" + dashes)
+    event.current_buffer.cursor_left(len(dashes) + 1)
+
+
+def raw_string_bracket(event: KeyPressEvent):
+    """Auto-close bracker in raw strings"""
+    matches = re.match(
+        r".*(r|R)[\"'](-*)",
+        event.current_buffer.document.current_line_before_cursor,
+    )
+    dashes = matches.group(2) if matches else ""
+    event.current_buffer.insert_text("[]" + dashes)
+    event.current_buffer.cursor_left(len(dashes) + 1)
+
+
+def raw_string_braces(event: KeyPressEvent):
+    """Auto-close braces in raw strings"""
+    matches = re.match(
+        r".*(r|R)[\"'](-*)",
+        event.current_buffer.document.current_line_before_cursor,
+    )
+    dashes = matches.group(2) if matches else ""
+    event.current_buffer.insert_text("{}" + dashes)
+    event.current_buffer.cursor_left(len(dashes) + 1)
+
+
+def skip_over(event: KeyPressEvent):
+    """Skip over automatically added parenthesis.
+
+    (rather than adding another parenthesis)"""
+    event.current_buffer.cursor_right()
+
+
+def delete_pair(event: KeyPressEvent):
+    """Delete auto-closed parenthesis"""
+    event.current_buffer.delete()
+    event.current_buffer.delete_before_cursor()
+
+
+auto_match_parens = {"(": parenthesis, "[": brackets, "{": braces}
+auto_match_parens_raw_string = {
+    "(": raw_string_parenthesis,
+    "[": raw_string_bracket,
+    "{": raw_string_braces,
+}

IPython/terminal/shortcuts/auto_suggest.py

@@ -0,0 +1,378 b''
+import re
+import tokenize
+from io import StringIO
+from typing import Callable, List, Optional, Union, Generator, Tuple, Sequence
+
+from prompt_toolkit.buffer import Buffer
+from prompt_toolkit.key_binding import KeyPressEvent
+from prompt_toolkit.key_binding.bindings import named_commands as nc
+from prompt_toolkit.auto_suggest import AutoSuggestFromHistory, Suggestion
+from prompt_toolkit.document import Document
+from prompt_toolkit.history import History
+from prompt_toolkit.shortcuts import PromptSession
+from prompt_toolkit.layout.processors import (
+    Processor,
+    Transformation,
+    TransformationInput,
+)
+
+from IPython.utils.tokenutil import generate_tokens
+
+
+def _get_query(document: Document):
+    return document.lines[document.cursor_position_row]
+
+
+class AppendAutoSuggestionInAnyLine(Processor):
+    """
+    Append the auto suggestion to lines other than the last (appending to the
+    last line is natively supported by the prompt toolkit).
+    """
+
+    def __init__(self, style: str = "class:auto-suggestion") -> None:
+        self.style = style
+
+    def apply_transformation(self, ti: TransformationInput) -> Transformation:
+        is_last_line = ti.lineno == ti.document.line_count - 1
+        is_active_line = ti.lineno == ti.document.cursor_position_row
+
+        if not is_last_line and is_active_line:
+            buffer = ti.buffer_control.buffer
+
+            if buffer.suggestion and ti.document.is_cursor_at_the_end_of_line:
+                suggestion = buffer.suggestion.text
+            else:
+                suggestion = ""
+
+            return Transformation(fragments=ti.fragments + [(self.style, suggestion)])
+        else:
+            return Transformation(fragments=ti.fragments)
+
+
+class NavigableAutoSuggestFromHistory(AutoSuggestFromHistory):
+    """
+    A subclass of AutoSuggestFromHistory that allow navigation to next/previous
+    suggestion from history. To do so it remembers the current position, but it
+    state need to carefully be cleared on the right events.
+    """
+
+    def __init__(
+        self,
+    ):
+        self.skip_lines = 0
+        self._connected_apps = []
+
+    def reset_history_position(self, _: Buffer):
+        self.skip_lines = 0
+
+    def disconnect(self):
+        for pt_app in self._connected_apps:
+            text_insert_event = pt_app.default_buffer.on_text_insert
+            text_insert_event.remove_handler(self.reset_history_position)
+
+    def connect(self, pt_app: PromptSession):
+        self._connected_apps.append(pt_app)
+        # note: `on_text_changed` could be used for a bit different behaviour
+        # on character deletion (i.e. reseting history position on backspace)
+        pt_app.default_buffer.on_text_insert.add_handler(self.reset_history_position)
+        pt_app.default_buffer.on_cursor_position_changed.add_handler(self._dismiss)
+
+    def get_suggestion(
+        self, buffer: Buffer, document: Document
+    ) -> Optional[Suggestion]:
+        text = _get_query(document)
+
+        if text.strip():
+            for suggestion, _ in self._find_next_match(
+                text, self.skip_lines, buffer.history
+            ):
+                return Suggestion(suggestion)
+
+        return None
+
+    def _dismiss(self, buffer, *args, **kwargs):
+        buffer.suggestion = None
+
+    def _find_match(
+        self, text: str, skip_lines: float, history: History, previous: bool
+    ) -> Generator[Tuple[str, float], None, None]:
+        """
+        text : str
+            Text content to find a match for, the user cursor is most of the
+            time at the end of this text.
+        skip_lines : float
+            number of items to skip in the search, this is used to indicate how
+            far in the list the user has navigated by pressing up or down.
+            The float type is used as the base value is +inf
+        history : History
+            prompt_toolkit History instance to fetch previous entries from.
+        previous : bool
+            Direction of the search, whether we are looking previous match
+            (True), or next match (False).
+
+        Yields
+        ------
+        Tuple with:
+        str:
+            current suggestion.
+        float:
+            will actually yield only ints, which is passed back via skip_lines,
+            which may be a +inf (float)
+
+
+        """
+        line_number = -1
+        for string in reversed(list(history.get_strings())):
+            for line in reversed(string.splitlines()):
+                line_number += 1
+                if not previous and line_number < skip_lines:
+                    continue
+                # do not return empty suggestions as these
+                # close the auto-suggestion overlay (and are useless)
+                if line.startswith(text) and len(line) > len(text):
+                    yield line[len(text) :], line_number
+                if previous and line_number >= skip_lines:
+                    return
+
+    def _find_next_match(
+        self, text: str, skip_lines: float, history: History
+    ) -> Generator[Tuple[str, float], None, None]:
+        return self._find_match(text, skip_lines, history, previous=False)
+
+    def _find_previous_match(self, text: str, skip_lines: float, history: History):
+        return reversed(
+            list(self._find_match(text, skip_lines, history, previous=True))
+        )
+
+    def up(self, query: str, other_than: str, history: History) -> None:
+        for suggestion, line_number in self._find_next_match(
+            query, self.skip_lines, history
+        ):
+            # if user has history ['very.a', 'very', 'very.b'] and typed 'very'
+            # we want to switch from 'very.b' to 'very.a' because a) if the
+            # suggestion equals current text, prompt-toolkit aborts suggesting
+            # b) user likely would not be interested in 'very' anyways (they
+            # already typed it).
+            if query + suggestion != other_than:
+                self.skip_lines = line_number
+                break
+        else:
+            # no matches found, cycle back to beginning
+            self.skip_lines = 0
+
+    def down(self, query: str, other_than: str, history: History) -> None:
+        for suggestion, line_number in self._find_previous_match(
+            query, self.skip_lines, history
+        ):
+            if query + suggestion != other_than:
+                self.skip_lines = line_number
+                break
+        else:
+            # no matches found, cycle to end
+            for suggestion, line_number in self._find_previous_match(
+                query, float("Inf"), history
+            ):
+                if query + suggestion != other_than:
+                    self.skip_lines = line_number
+                    break
+
+
+# Needed for to accept autosuggestions in vi insert mode
+def accept_in_vi_insert_mode(event: KeyPressEvent):
+    """Apply autosuggestion if at end of line."""
+    buffer = event.current_buffer
+    d = buffer.document
+    after_cursor = d.text[d.cursor_position :]
+    lines = after_cursor.split("\n")
+    end_of_current_line = lines[0].strip()
+    suggestion = buffer.suggestion
+    if (suggestion is not None) and (suggestion.text) and (end_of_current_line == ""):
+        buffer.insert_text(suggestion.text)
+    else:
+        nc.end_of_line(event)
+
+
+def accept(event: KeyPressEvent):
+    """Accept autosuggestion"""
+    buffer = event.current_buffer
+    suggestion = buffer.suggestion
+    if suggestion:
+        buffer.insert_text(suggestion.text)
+    else:
+        nc.forward_char(event)
+
+
+def discard(event: KeyPressEvent):
+    """Discard autosuggestion"""
+    buffer = event.current_buffer
+    buffer.suggestion = None
+
+
+def accept_word(event: KeyPressEvent):
+    """Fill partial autosuggestion by word"""
+    buffer = event.current_buffer
+    suggestion = buffer.suggestion
+    if suggestion:
+        t = re.split(r"(\S+\s+)", suggestion.text)
+        buffer.insert_text(next((x for x in t if x), ""))
+    else:
+        nc.forward_word(event)
+
+
+def accept_character(event: KeyPressEvent):
+    """Fill partial autosuggestion by character"""
+    b = event.current_buffer
+    suggestion = b.suggestion
+    if suggestion and suggestion.text:
+        b.insert_text(suggestion.text[0])
+
+
+def accept_and_keep_cursor(event: KeyPressEvent):
+    """Accept autosuggestion and keep cursor in place"""
+    buffer = event.current_buffer
+    old_position = buffer.cursor_position
+    suggestion = buffer.suggestion
+    if suggestion:
+        buffer.insert_text(suggestion.text)
+        buffer.cursor_position = old_position
+
+
+def accept_and_move_cursor_left(event: KeyPressEvent):
+    """Accept autosuggestion and move cursor left in place"""
+    accept_and_keep_cursor(event)
+    nc.backward_char(event)
+
+
+def _update_hint(buffer: Buffer):
+    if buffer.auto_suggest:
+        suggestion = buffer.auto_suggest.get_suggestion(buffer, buffer.document)
+        buffer.suggestion = suggestion
+
+
+def backspace_and_resume_hint(event: KeyPressEvent):
+    """Resume autosuggestions after deleting last character"""
+    current_buffer = event.current_buffer
+
+    def resume_hinting(buffer: Buffer):
+        _update_hint(buffer)
+        current_buffer.on_text_changed.remove_handler(resume_hinting)
+
+    current_buffer.on_text_changed.add_handler(resume_hinting)
+    nc.backward_delete_char(event)
+
+
+def up_and_update_hint(event: KeyPressEvent):
+    """Go up and update hint"""
+    current_buffer = event.current_buffer
+
+    current_buffer.auto_up(count=event.arg)
+    _update_hint(current_buffer)
+
+
+def down_and_update_hint(event: KeyPressEvent):
+    """Go down and update hint"""
+    current_buffer = event.current_buffer
+
+    current_buffer.auto_down(count=event.arg)
+    _update_hint(current_buffer)
+
+
+def accept_token(event: KeyPressEvent):
+    """Fill partial autosuggestion by token"""
+    b = event.current_buffer
+    suggestion = b.suggestion
+
+    if suggestion:
+        prefix = _get_query(b.document)
+        text = prefix + suggestion.text
+
+        tokens: List[Optional[str]] = [None, None, None]
+        substrings = [""]
+        i = 0
+
+        for token in generate_tokens(StringIO(text).readline):
+            if token.type == tokenize.NEWLINE:
+                index = len(text)
+            else:
+                index = text.index(token[1], len(substrings[-1]))
+            substrings.append(text[:index])
+            tokenized_so_far = substrings[-1]
+            if tokenized_so_far.startswith(prefix):
+                if i == 0 and len(tokenized_so_far) > len(prefix):
+                    tokens[0] = tokenized_so_far[len(prefix) :]
+                    substrings.append(tokenized_so_far)
+                    i += 1
+                tokens[i] = token[1]
+                if i == 2:
+                    break
+                i += 1
+
+        if tokens[0]:
+            to_insert: str
+            insert_text = substrings[-2]
+            if tokens[1] and len(tokens[1]) == 1:
+                insert_text = substrings[-1]
+            to_insert = insert_text[len(prefix) :]
+            b.insert_text(to_insert)
+            return
+
+    nc.forward_word(event)
+
+
+Provider = Union[AutoSuggestFromHistory, NavigableAutoSuggestFromHistory, None]
+
+
+def _swap_autosuggestion(
+    buffer: Buffer,
+    provider: NavigableAutoSuggestFromHistory,
+    direction_method: Callable,
+):
+    """
+    We skip most recent history entry (in either direction) if it equals the
+    current autosuggestion because if user cycles when auto-suggestion is shown
+    they most likely want something else than what was suggested (otherwise
+    they would have accepted the suggestion).
+    """
+    suggestion = buffer.suggestion
+    if not suggestion:
+        return
+
+    query = _get_query(buffer.document)
+    current = query + suggestion.text
+
+    direction_method(query=query, other_than=current, history=buffer.history)
+
+    new_suggestion = provider.get_suggestion(buffer, buffer.document)
+    buffer.suggestion = new_suggestion
+
+
+def swap_autosuggestion_up(provider: Provider):
+    def swap_autosuggestion_up(event: KeyPressEvent):
+        """Get next autosuggestion from history."""
+        if not isinstance(provider, NavigableAutoSuggestFromHistory):
+            return
+
+        return _swap_autosuggestion(
+            buffer=event.current_buffer, provider=provider, direction_method=provider.up
+        )
+
+    swap_autosuggestion_up.__name__ = "swap_autosuggestion_up"
+    return swap_autosuggestion_up
+
+
+def swap_autosuggestion_down(
+    provider: Union[AutoSuggestFromHistory, NavigableAutoSuggestFromHistory, None]
+):
+    def swap_autosuggestion_down(event: KeyPressEvent):
+        """Get previous autosuggestion from history."""
+        if not isinstance(provider, NavigableAutoSuggestFromHistory):
+            return
+
+        return _swap_autosuggestion(
+            buffer=event.current_buffer,
+            provider=provider,
+            direction_method=provider.down,
+        )
+
+    swap_autosuggestion_down.__name__ = "swap_autosuggestion_down"
+    return swap_autosuggestion_down

IPython/terminal/tests/test_shortcuts.py

@@ -0,0 +1,318 b''
+import pytest
+from IPython.terminal.shortcuts.auto_suggest import (
+    accept,
+    accept_in_vi_insert_mode,
+    accept_token,
+    accept_character,
+    accept_word,
+    accept_and_keep_cursor,
+    discard,
+    NavigableAutoSuggestFromHistory,
+    swap_autosuggestion_up,
+    swap_autosuggestion_down,
+)
+
+from prompt_toolkit.history import InMemoryHistory
+from prompt_toolkit.buffer import Buffer
+from prompt_toolkit.document import Document
+from prompt_toolkit.auto_suggest import AutoSuggestFromHistory
+
+from unittest.mock import patch, Mock
+
+
+def make_event(text, cursor, suggestion):
+    event = Mock()
+    event.current_buffer = Mock()
+    event.current_buffer.suggestion = Mock()
+    event.current_buffer.text = text
+    event.current_buffer.cursor_position = cursor
+    event.current_buffer.suggestion.text = suggestion
+    event.current_buffer.document = Document(text=text, cursor_position=cursor)
+    return event
+
+
[e-mail](
+    "text, suggestion, expected",
+    [
+        ("", "def out(tag: str, n=50):", "def out(tag: str, n=50):"),
+        ("def ", "out(tag: str, n=50):", "out(tag: str, n=50):"),
+    ],
+)
+def test_accept(text, suggestion, expected):
+    event = make_event(text, len(text), suggestion)
+    buffer = event.current_buffer
+    buffer.insert_text = Mock()
+    accept(event)
+    assert buffer.insert_text.called
+    assert buffer.insert_text.call_args[0] == (expected,)
+
+
[e-mail](
+    "text, suggestion",
+    [
+        ("", "def out(tag: str, n=50):"),
+        ("def ", "out(tag: str, n=50):"),
+    ],
+)
+def test_discard(text, suggestion):
+    event = make_event(text, len(text), suggestion)
+    buffer = event.current_buffer
+    buffer.insert_text = Mock()
+    discard(event)
+    assert not buffer.insert_text.called
+    assert buffer.suggestion is None
+
+
[e-mail](
+    "text, cursor, suggestion, called",
+    [
+        ("123456", 6, "123456789", True),
+        ("123456", 3, "123456789", False),
+        ("123456   \n789", 6, "123456789", True),
+    ],
+)
+def test_autosuggest_at_EOL(text, cursor, suggestion, called):
+    """
+    test that autosuggest is only applied at end of line.
+    """
+
+    event = make_event(text, cursor, suggestion)
+    event.current_buffer.insert_text = Mock()
+    accept_in_vi_insert_mode(event)
+    if called:
+        event.current_buffer.insert_text.assert_called()
+    else:
+        event.current_buffer.insert_text.assert_not_called()
+        # event.current_buffer.document.get_end_of_line_position.assert_called()
+
+
[e-mail](
+    "text, suggestion, expected",
+    [
+        ("", "def out(tag: str, n=50):", "def "),
+        ("d", "ef out(tag: str, n=50):", "ef "),
+        ("de ", "f out(tag: str, n=50):", "f "),
+        ("def", " out(tag: str, n=50):", " "),
+        ("def ", "out(tag: str, n=50):", "out("),
+        ("def o", "ut(tag: str, n=50):", "ut("),
+        ("def ou", "t(tag: str, n=50):", "t("),
+        ("def out", "(tag: str, n=50):", "("),
+        ("def out(", "tag: str, n=50):", "tag: "),
+        ("def out(t", "ag: str, n=50):", "ag: "),
+        ("def out(ta", "g: str, n=50):", "g: "),
+        ("def out(tag", ": str, n=50):", ": "),
+        ("def out(tag:", " str, n=50):", " "),
+        ("def out(tag: ", "str, n=50):", "str, "),
+        ("def out(tag: s", "tr, n=50):", "tr, "),
+        ("def out(tag: st", "r, n=50):", "r, "),
+        ("def out(tag: str", ", n=50):", ", n"),
+        ("def out(tag: str,", " n=50):", " n"),
+        ("def out(tag: str, ", "n=50):", "n="),
+        ("def out(tag: str, n", "=50):", "="),
+        ("def out(tag: str, n=", "50):", "50)"),
+        ("def out(tag: str, n=5", "0):", "0)"),
+        ("def out(tag: str, n=50", "):", "):"),
+        ("def out(tag: str, n=50)", ":", ":"),
+    ],
+)
+def test_autosuggest_token(text, suggestion, expected):
+    event = make_event(text, len(text), suggestion)
+    event.current_buffer.insert_text = Mock()
+    accept_token(event)
+    assert event.current_buffer.insert_text.called
+    assert event.current_buffer.insert_text.call_args[0] == (expected,)
+
+
[e-mail](
+    "text, suggestion, expected",
+    [
+        ("", "def out(tag: str, n=50):", "d"),
+        ("d", "ef out(tag: str, n=50):", "e"),
+        ("de ", "f out(tag: str, n=50):", "f"),
+        ("def", " out(tag: str, n=50):", " "),
+    ],
+)
+def test_accept_character(text, suggestion, expected):
+    event = make_event(text, len(text), suggestion)
+    event.current_buffer.insert_text = Mock()
+    accept_character(event)
+    assert event.current_buffer.insert_text.called
+    assert event.current_buffer.insert_text.call_args[0] == (expected,)
+
+
[e-mail](
+    "text, suggestion, expected",
+    [
+        ("", "def out(tag: str, n=50):", "def "),
+        ("d", "ef out(tag: str, n=50):", "ef "),
+        ("de", "f out(tag: str, n=50):", "f "),
+        ("def", " out(tag: str, n=50):", " "),
+        # (this is why we also have accept_token)
+        ("def ", "out(tag: str, n=50):", "out(tag: "),
+    ],
+)
+def test_accept_word(text, suggestion, expected):
+    event = make_event(text, len(text), suggestion)
+    event.current_buffer.insert_text = Mock()
+    accept_word(event)
+    assert event.current_buffer.insert_text.called
+    assert event.current_buffer.insert_text.call_args[0] == (expected,)
+
+
[e-mail](
+    "text, suggestion, expected, cursor",
+    [
+        ("", "def out(tag: str, n=50):", "def out(tag: str, n=50):", 0),
+        ("def ", "out(tag: str, n=50):", "out(tag: str, n=50):", 4),
+    ],
+)
+def test_accept_and_keep_cursor(text, suggestion, expected, cursor):
+    event = make_event(text, cursor, suggestion)
+    buffer = event.current_buffer
+    buffer.insert_text = Mock()
+    accept_and_keep_cursor(event)
+    assert buffer.insert_text.called
+    assert buffer.insert_text.call_args[0] == (expected,)
+    assert buffer.cursor_position == cursor
+
+
+def test_autosuggest_token_empty():
+    full = "def out(tag: str, n=50):"
+    event = make_event(full, len(full), "")
+    event.current_buffer.insert_text = Mock()
+
+    with patch(
+        "prompt_toolkit.key_binding.bindings.named_commands.forward_word"
+    ) as forward_word:
+        accept_token(event)
+        assert not event.current_buffer.insert_text.called
+        assert forward_word.called
+
+
+def test_other_providers():
+    """Ensure that swapping autosuggestions does not break with other providers"""
+    provider = AutoSuggestFromHistory()
+    up = swap_autosuggestion_up(provider)
+    down = swap_autosuggestion_down(provider)
+    event = Mock()
+    event.current_buffer = Buffer()
+    assert up(event) is None
+    assert down(event) is None
+
+
+async def test_navigable_provider():
+    provider = NavigableAutoSuggestFromHistory()
+    history = InMemoryHistory(history_strings=["very_a", "very", "very_b", "very_c"])
+    buffer = Buffer(history=history)
+
+    async for _ in history.load():
+        pass
+
+    buffer.cursor_position = 5
+    buffer.text = "very"
+
+    up = swap_autosuggestion_up(provider)
+    down = swap_autosuggestion_down(provider)
+
+    event = Mock()
+    event.current_buffer = buffer
+
+    def get_suggestion():
+        suggestion = provider.get_suggestion(buffer, buffer.document)
+        buffer.suggestion = suggestion
+        return suggestion
+
+    assert get_suggestion().text == "_c"
+
+    # should go up
+    up(event)
+    assert get_suggestion().text == "_b"
+
+    # should skip over 'very' which is identical to buffer content
+    up(event)
+    assert get_suggestion().text == "_a"
+
+    # should cycle back to beginning
+    up(event)
+    assert get_suggestion().text == "_c"
+
+    # should cycle back through end boundary
+    down(event)
+    assert get_suggestion().text == "_a"
+
+    down(event)
+    assert get_suggestion().text == "_b"
+
+    down(event)
+    assert get_suggestion().text == "_c"
+
+    down(event)
+    assert get_suggestion().text == "_a"
+
+
+async def test_navigable_provider_multiline_entries():
+    provider = NavigableAutoSuggestFromHistory()
+    history = InMemoryHistory(history_strings=["very_a\nvery_b", "very_c"])
+    buffer = Buffer(history=history)
+
+    async for _ in history.load():
+        pass
+
+    buffer.cursor_position = 5
+    buffer.text = "very"
+    up = swap_autosuggestion_up(provider)
+    down = swap_autosuggestion_down(provider)
+
+    event = Mock()
+    event.current_buffer = buffer
+
+    def get_suggestion():
+        suggestion = provider.get_suggestion(buffer, buffer.document)
+        buffer.suggestion = suggestion
+        return suggestion
+
+    assert get_suggestion().text == "_c"
+
+    up(event)
+    assert get_suggestion().text == "_b"
+
+    up(event)
+    assert get_suggestion().text == "_a"
+
+    down(event)
+    assert get_suggestion().text == "_b"
+
+    down(event)
+    assert get_suggestion().text == "_c"
+
+
+def create_session_mock():
+    session = Mock()
+    session.default_buffer = Buffer()
+    return session
+
+
+def test_navigable_provider_connection():
+    provider = NavigableAutoSuggestFromHistory()
+    provider.skip_lines = 1
+
+    session_1 = create_session_mock()
+    provider.connect(session_1)
+
+    assert provider.skip_lines == 1
+    session_1.default_buffer.on_text_insert.fire()
+    assert provider.skip_lines == 0
+
+    session_2 = create_session_mock()
+    provider.connect(session_2)
+    provider.skip_lines = 2
+
+    assert provider.skip_lines == 2
+    session_2.default_buffer.on_text_insert.fire()
+    assert provider.skip_lines == 0
+
+    provider.skip_lines = 3
+    provider.disconnect()
+    session_1.default_buffer.on_text_insert.fire()
+    session_2.default_buffer.on_text_insert.fire()
+    assert provider.skip_lines == 3

docs/source/_static/theme_overrides.css

@@ -0,0 +1,7 b''
+/*
+ Needed to revert problematic lack of wrapping in sphinx_rtd_theme, see:
+ https://github.com/readthedocs/sphinx_rtd_theme/issues/117
+*/
+.wy-table-responsive table.shortcuts td, .wy-table-responsive table.shortcuts th {
+  white-space: normal!important;
+}

.github/workflows/mypy.yml

@@ -29,11 +29,13 b' jobs:'
         pip install mypy pyflakes flake8
     - name: Lint with mypy
       run: |
+        set -e
         mypy -p IPython.terminal
         mypy -p IPython.core.magics
         mypy -p IPython.core.guarded_eval
         mypy -p IPython.core.completer
     - name: Lint with pyflakes
       run: |
+        set -e
         flake8  IPython/core/magics/script.py
         flake8  IPython/core/magics/packaging.py

IPython/__init__.py

@@ -63,7 +63,7 b' __version__ = release.version'
 version_info = release.version_info
 # list of CVEs that should have been patched in this release.
 # this is informational and should not be relied upon.
-__patched_cves__ = {"CVE-2022-21699"}
+__patched_cves__ = {"CVE-2022-21699", "CVE-2023-24816"}
 
 
 def embed_kernel(module=None, local_ns=None, **kwargs):

IPython/core/application.py

@@ -310,7 +310,7 b' class BaseIPythonApplication(Application):'
                 except OSError as e:
                     # this will not be EEXIST
                     self.log.error("couldn't create path %s: %s", path, e)
-            self.log.debug("IPYTHONDIR set to: %s" % new)
+            self.log.debug("IPYTHONDIR set to: %s", new)
 
     def load_config_file(self, suppress_errors=IPYTHON_SUPPRESS_CONFIG_ERRORS):
         """Load the config file.
@@ -400,7 +400,7 b' class BaseIPythonApplication(Application):'
                     self.log.fatal("Profile %r not found."%self.profile)
                     self.exit(1)
             else:
-                self.log.debug(f"Using existing profile dir: {p.location!r}")
+                self.log.debug("Using existing profile dir: %r", p.location)
         else:
             location = self.config.ProfileDir.location
             # location is fully specified
@@ -420,7 +420,7 b' class BaseIPythonApplication(Application):'
                     self.log.fatal("Profile directory %r not found."%location)
                     self.exit(1)
             else:
-                self.log.debug(f"Using existing profile dir: {p.location!r}")
+                self.log.debug("Using existing profile dir: %r", p.location)
             # if profile_dir is specified explicitly, set profile name
             dir_name = os.path.basename(p.location)
             if dir_name.startswith('profile_'):
@@ -467,7 +467,7 b' class BaseIPythonApplication(Application):'
         s = self.generate_config_file()
         config_file = Path(self.profile_dir.location) / self.config_file_name
         if self.overwrite or not config_file.exists():
-            self.log.warning("Generating default config file: %r" % (config_file))
+            self.log.warning("Generating default config file: %r", (config_file))
             config_file.write_text(s, encoding="utf-8")
 
     @catch_config_error

IPython/core/guarded_eval.py

@@ -609,8 +609,8 b' SUPPORTED_EXTERNAL_GETITEM = {'
 
 BUILTIN_GETITEM: Set[InstancesHaveGetItem] = {
     dict,
-    str,
-    bytes,
+    str,  # type: ignore[arg-type]
+    bytes,  # type: ignore[arg-type]
     list,
     tuple,
     collections.defaultdict,
@@ -619,7 +619,7 b' BUILTIN_GETITEM: Set[InstancesHaveGetItem] = {'
     collections.ChainMap,
     collections.UserDict,
     collections.UserList,
-    collections.UserString,
+    collections.UserString,  # type: ignore[arg-type]
     _DummyNamedTuple,
     _IdentitySubscript,
 }

IPython/core/interactiveshell.py

@@ -3013,7 +3013,7 b' class InteractiveShell(SingletonConfigurable):'
             runner = _pseudo_sync_runner
 
         try:
-            return runner(coro)
+            result = runner(coro)
         except BaseException as e:
             info = ExecutionInfo(
                 raw_cell, store_history, silent, shell_futures, cell_id
@@ -3021,6 +3021,7 b' class InteractiveShell(SingletonConfigurable):'
             result = ExecutionResult(info)
             result.error_in_exec = e
             self.showtraceback(running_compiled_code=True)
+        finally:
             return result
 
     def should_run_async(

IPython/core/logger.py

@@ -198,7 +198,16 b' which already exists. But you must first start the logging process with'
                 odata = u'\n'.join([u'#[Out]# %s' % s
                                    for s in data.splitlines()])
                 write(u'%s\n' % odata)
-            self.logfile.flush()
+            try:
+                self.logfile.flush()
+            except OSError:
+                print("Failed to flush the log file.")
+                print(
+                    f"Please check that {self.logfname} exists and have the right permissions."
+                )
+                print(
+                    "Also consider turning off the log with `%logstop` to avoid this warning."
+                )
 
     def logstop(self):
         """Fully stop logging and close log file.

IPython/core/magics/osm.py

@@ -468,9 +468,9 b' class OSMagics(Magics):'
         string.
 
         Usage:\\
-          %set_env var val: set value for var
-          %set_env var=val: set value for var
-          %set_env var=$val: set value for var, using python expansion if possible
+          :``%set_env var val``: set value for var
+          :``%set_env var=val``: set value for var
+          :``%set_env var=$val``: set value for var, using python expansion if possible
         """
         split = '=' if '=' in parameter_s else ' '
         bits = parameter_s.split(split, 1)

IPython/core/magics/pylab.py

@@ -54,7 +54,7 b' class PylabMagics(Magics):'
         If you are using the inline matplotlib backend in the IPython Notebook
         you can set which figure formats are enabled using the following::
 
-            In [1]: from IPython.display import set_matplotlib_formats
+            In [1]: from matplotlib_inline.backend_inline import set_matplotlib_formats
 
             In [2]: set_matplotlib_formats('pdf', 'svg')
 
@@ -65,9 +65,9 b' class PylabMagics(Magics):'
 
             In [3]: %config InlineBackend.print_figure_kwargs = {'bbox_inches':None}
 
-        In addition, see the docstring of
-        `IPython.display.set_matplotlib_formats` and
-        `IPython.display.set_matplotlib_close` for more information on
+        In addition, see the docstrings of
+        `matplotlib_inline.backend_inline.set_matplotlib_formats` and
+        `matplotlib_inline.backend_inline.set_matplotlib_close` for more information on
         changing additional behaviors of the inline backend.
 
         Examples

IPython/core/release.py

@@ -16,7 +16,7 b''
 # release.  'dev' as a _version_extra string means this is a development
 # version
 _version_major = 8
-_version_minor = 9
+_version_minor = 11
 _version_patch = 0
 _version_extra = ".dev"
 # _version_extra = "rc1"

IPython/core/shellapp.py

@@ -278,7 +278,7 b' class InteractiveShellApp(Configurable):'
             )
             for ext in extensions:
                 try:
-                    self.log.info("Loading IPython extension: %s" % ext)
+                    self.log.info("Loading IPython extension: %s", ext)
                     self.shell.extension_manager.load_extension(ext)
                 except:
                     if self.reraise_ipython_extension_failures:

IPython/core/tests/test_interactiveshell.py

@@ -17,6 +17,7 b' import shutil'
 import sys
 import tempfile
 import unittest
+import pytest
 from unittest import mock
 
 from os.path import join
@@ -635,10 +636,23 b' class TestSystemRaw(ExitCodeChecks):'
             )
         self.assertEqual(ip.user_ns["_exit_code"], -signal.SIGINT)
 
-    def test_magic_warnings(self):
-        for magic_cmd in ("pip", "conda", "cd"):
-            with self.assertWarnsRegex(Warning, "You executed the system command"):
-                ip.system_raw(magic_cmd)
+
[e-mail]("magic_cmd", ["pip", "conda", "cd"])
+def test_magic_warnings(magic_cmd):
+    if sys.platform == "win32":
+        to_mock = "os.system"
+        expected_arg, expected_kwargs = magic_cmd, dict()
+    else:
+        to_mock = "subprocess.call"
+        expected_arg, expected_kwargs = magic_cmd, dict(
+            shell=True, executable=os.environ.get("SHELL", None)
+        )
+
+    with mock.patch(to_mock, return_value=0) as mock_sub:
+        with pytest.warns(Warning, match=r"You executed the system command"):
+            ip.system_raw(magic_cmd)
+        mock_sub.assert_called_once_with(expected_arg, **expected_kwargs)
+
 
 # TODO: Exit codes are currently ignored on Windows.
 class TestSystemPipedExitCode(ExitCodeChecks):
@@ -1089,9 +1103,12 b' def test_run_cell_asyncio_run():'
 
 
 def test_should_run_async():
-    assert not ip.should_run_async("a = 5")
-    assert ip.should_run_async("await x")
-    assert ip.should_run_async("import asyncio; await asyncio.sleep(1)")
+    assert not ip.should_run_async("a = 5", transformed_cell="a = 5")
+    assert ip.should_run_async("await x", transformed_cell="await x")
+    assert ip.should_run_async(
+        "import asyncio; await asyncio.sleep(1)",
+        transformed_cell="import asyncio; await asyncio.sleep(1)",
+    )
 
 
 def test_set_custom_completer():
@@ -1110,3 +1127,49 b' def test_set_custom_completer():'
 
     # clean up
     ip.Completer.custom_matchers.pop()
+
+
+class TestShowTracebackAttack(unittest.TestCase):
+    """Test that the interactive shell is resilient against the client attack of
+    manipulating the showtracebacks method. These attacks shouldn't result in an
+    unhandled exception in the kernel."""
+
+    def setUp(self):
+        self.orig_showtraceback = interactiveshell.InteractiveShell.showtraceback
+
+    def tearDown(self):
+        interactiveshell.InteractiveShell.showtraceback = self.orig_showtraceback
+
+    def test_set_show_tracebacks_none(self):
+        """Test the case of the client setting showtracebacks to None"""
+
+        result = ip.run_cell(
+            """
+            import IPython.core.interactiveshell
+            IPython.core.interactiveshell.InteractiveShell.showtraceback = None
+
+            assert False, "This should not raise an exception"
+        """
+        )
+        print(result)
+
+        assert result.result is None
+        assert isinstance(result.error_in_exec, TypeError)
+        assert str(result.error_in_exec) == "'NoneType' object is not callable"
+
+    def test_set_show_tracebacks_noop(self):
+        """Test the case of the client setting showtracebacks to a no op lambda"""
+
+        result = ip.run_cell(
+            """
+            import IPython.core.interactiveshell
+            IPython.core.interactiveshell.InteractiveShell.showtraceback = lambda *args, **kwargs: None
+
+            assert False, "This should not raise an exception"
+        """
+        )
+        print(result)
+
+        assert result.result is None
+        assert isinstance(result.error_in_exec, AssertionError)
+        assert str(result.error_in_exec) == "This should not raise an exception"

IPython/lib/latextools.py

@@ -160,38 +160,37 b" def latex_to_png_dvipng(s, wrap, color='Black', scale=1.0):"
         with workdir.joinpath(tmpfile).open("w", encoding="utf8") as f:
             f.writelines(genelatex(s, wrap))
 
-        with open(os.devnull, 'wb') as devnull:
-            subprocess.check_call(
-                ["latex", "-halt-on-error", "-interaction", "batchmode", tmpfile],
-                cwd=workdir,
-                stdout=devnull,
-                stderr=devnull,
-                startupinfo=startupinfo,
-            )
-
-            resolution = round(150*scale)
-            subprocess.check_call(
-                [
-                    "dvipng",
-                    "-T",
-                    "tight",
-                    "-D",
-                    str(resolution),
-                    "-z",
-                    "9",
-                    "-bg",
-                    "Transparent",
-                    "-o",
-                    outfile,
-                    dvifile,
-                    "-fg",
-                    color,
-                ],
-                cwd=workdir,
-                stdout=devnull,
-                stderr=devnull,
-                startupinfo=startupinfo,
-            )
+        subprocess.check_call(
+            ["latex", "-halt-on-error", "-interaction", "batchmode", tmpfile],
+            cwd=workdir,
+            stdout=subprocess.DEVNULL,
+            stderr=subprocess.DEVNULL,
+            startupinfo=startupinfo,
+        )
+
+        resolution = round(150 * scale)
+        subprocess.check_call(
+            [
+                "dvipng",
+                "-T",
+                "tight",
+                "-D",
+                str(resolution),
+                "-z",
+                "9",
+                "-bg",
+                "Transparent",
+                "-o",
+                outfile,
+                dvifile,
+                "-fg",
+                color,
+            ],
+            cwd=workdir,
+            stdout=subprocess.DEVNULL,
+            stderr=subprocess.DEVNULL,
+            startupinfo=startupinfo,
+        )
 
         with workdir.joinpath(outfile).open("rb") as f:
             return f.read()

IPython/terminal/interactiveshell.py

@@ -4,6 +4,7 b' import asyncio'
 import os
 import sys
 from warnings import warn
+from typing import Union as UnionType
 
 from IPython.core.async_helpers import get_asyncio_loop
 from IPython.core.interactiveshell import InteractiveShell, InteractiveShellABC
@@ -49,6 +50,10 b' from .pt_inputhooks import get_inputhook_name_and_func'
 from .prompts import Prompts, ClassicPrompts, RichPromptDisplayHook
 from .ptutils import IPythonPTCompleter, IPythonPTLexer
 from .shortcuts import create_ipython_shortcuts
+from .shortcuts.auto_suggest import (
+    NavigableAutoSuggestFromHistory,
+    AppendAutoSuggestionInAnyLine,
+)
 
 PTK3 = ptk_version.startswith('3.')
 
@@ -183,7 +188,10 b' class TerminalInteractiveShell(InteractiveShell):'
                                      'menus, decrease for short and wide.'
                             ).tag(config=True)
 
-    pt_app = None
+    pt_app: UnionType[PromptSession, None] = None
+    auto_suggest: UnionType[
+        AutoSuggestFromHistory, NavigableAutoSuggestFromHistory, None
+    ] = None
     debugger_history = None
 
     debugger_history_file = Unicode(
@@ -376,18 +384,27 b' class TerminalInteractiveShell(InteractiveShell):'
     ).tag(config=True)
 
     autosuggestions_provider = Unicode(
-        "AutoSuggestFromHistory",
+        "NavigableAutoSuggestFromHistory",
         help="Specifies from which source automatic suggestions are provided. "
-        "Can be set to `'AutoSuggestFromHistory`' or `None` to disable"
-        "automatic suggestions. Default is `'AutoSuggestFromHistory`'.",
+        "Can be set to ``'NavigableAutoSuggestFromHistory'`` (:kbd:`up` and "
+        ":kbd:`down` swap suggestions), ``'AutoSuggestFromHistory'``, "
+        " or ``None`` to disable automatic suggestions. "
+        "Default is `'NavigableAutoSuggestFromHistory`'.",
         allow_none=True,
     ).tag(config=True)
 
     def _set_autosuggestions(self, provider):
+        # disconnect old handler
+        if self.auto_suggest and isinstance(
+            self.auto_suggest, NavigableAutoSuggestFromHistory
+        ):
+            self.auto_suggest.disconnect()
         if provider is None:
             self.auto_suggest = None
         elif provider == "AutoSuggestFromHistory":
             self.auto_suggest = AutoSuggestFromHistory()
+        elif provider == "NavigableAutoSuggestFromHistory":
+            self.auto_suggest = NavigableAutoSuggestFromHistory()
         else:
             raise ValueError("No valid provider.")
         if self.pt_app:
@@ -462,6 +479,8 b' class TerminalInteractiveShell(InteractiveShell):'
             tempfile_suffix=".py",
             **self._extra_prompt_options()
         )
+        if isinstance(self.auto_suggest, NavigableAutoSuggestFromHistory):
+            self.auto_suggest.connect(self.pt_app)
 
     def _make_style_from_name_or_cls(self, name_or_cls):
         """
@@ -560,23 +579,39 b' class TerminalInteractiveShell(InteractiveShell):'
             get_message = get_message()
 
         options = {
-                'complete_in_thread': False,
-                'lexer':IPythonPTLexer(),
-                'reserve_space_for_menu':self.space_for_menu,
-                'message': get_message,
-                'prompt_continuation': (
-                    lambda width, lineno, is_soft_wrap:
-                        PygmentsTokens(self.prompts.continuation_prompt_tokens(width))),
-                'multiline': True,
-                'complete_style': self.pt_complete_style,
-
+            "complete_in_thread": False,
+            "lexer": IPythonPTLexer(),
+            "reserve_space_for_menu": self.space_for_menu,
+            "message": get_message,
+            "prompt_continuation": (
+                lambda width, lineno, is_soft_wrap: PygmentsTokens(
+                    self.prompts.continuation_prompt_tokens(width)
+                )
+            ),
+            "multiline": True,
+            "complete_style": self.pt_complete_style,
+            "input_processors": [
                 # Highlight matching brackets, but only when this setting is
                 # enabled, and only when the DEFAULT_BUFFER has the focus.
-                'input_processors': [ConditionalProcessor(
-                        processor=HighlightMatchingBracketProcessor(chars='[](){}'),
-                        filter=HasFocus(DEFAULT_BUFFER) & ~IsDone() &
-                            Condition(lambda: self.highlight_matching_brackets))],
-                }
+                ConditionalProcessor(
+                    processor=HighlightMatchingBracketProcessor(chars="[](){}"),
+                    filter=HasFocus(DEFAULT_BUFFER)
+                    & ~IsDone()
+                    & Condition(lambda: self.highlight_matching_brackets),
+                ),
+                # Show auto-suggestion in lines other than the last line.
+                ConditionalProcessor(
+                    processor=AppendAutoSuggestionInAnyLine(),
+                    filter=HasFocus(DEFAULT_BUFFER)
+                    & ~IsDone()
+                    & Condition(
+                        lambda: isinstance(
+                            self.auto_suggest, NavigableAutoSuggestFromHistory
+                        )
+                    ),
+                ),
+            ],
+        }
         if not PTK3:
             options['inputhook'] = self.inputhook
 
@@ -647,7 +682,7 b' class TerminalInteractiveShell(InteractiveShell):'
                 self.alias_manager.soft_define_alias(cmd, cmd)
 
 
-    def __init__(self, *args, **kwargs):
+    def __init__(self, *args, **kwargs) -> None:
         super(TerminalInteractiveShell, self).__init__(*args, **kwargs)
         self._set_autosuggestions(self.autosuggestions_provider)
         self.init_prompt_toolkit_cli()

IPython/terminal/ipapp.py

@@ -156,7 +156,7 b" frontend_flags['i'] = ("
 flags.update(frontend_flags)
 
 aliases = dict(base_aliases)
-aliases.update(shell_aliases)
+aliases.update(shell_aliases)  # type: ignore[arg-type]
 
 #-----------------------------------------------------------------------------
 # Main classes and functions
@@ -180,7 +180,7 b' class LocateIPythonApp(BaseIPythonApplication):'
 class TerminalIPythonApp(BaseIPythonApplication, InteractiveShellApp):
     name = u'ipython'
     description = usage.cl_usage
-    crash_handler_class = IPAppCrashHandler
+    crash_handler_class = IPAppCrashHandler  # typing: ignore[assignment]
     examples = _examples
 
     flags = flags

IPython/terminal/tests/test_interactivshell.py

@@ -7,11 +7,25 b' import sys'
 import unittest
 import os
 
+from prompt_toolkit.auto_suggest import AutoSuggestFromHistory
+
 from IPython.core.inputtransformer import InputTransformer
 from IPython.testing import tools as tt
 from IPython.utils.capture import capture_output
 
 from IPython.terminal.ptutils import _elide, _adjust_completion_text_based_on_context
+from IPython.terminal.shortcuts.auto_suggest import NavigableAutoSuggestFromHistory
+
+
+class TestAutoSuggest(unittest.TestCase):
+    def test_changing_provider(self):
+        ip = get_ipython()
+        ip.autosuggestions_provider = None
+        self.assertEqual(ip.auto_suggest, None)
+        ip.autosuggestions_provider = "AutoSuggestFromHistory"
+        self.assertIsInstance(ip.auto_suggest, AutoSuggestFromHistory)
+        ip.autosuggestions_provider = "NavigableAutoSuggestFromHistory"
+        self.assertIsInstance(ip.auto_suggest, NavigableAutoSuggestFromHistory)
 
 
 class TestElide(unittest.TestCase):
@@ -24,10 +38,10 b' class TestElide(unittest.TestCase):'
         )
 
         test_string = os.sep.join(["", 10 * "a", 10 * "b", 10 * "c", ""])
-        expect_stirng = (
+        expect_string = (
             os.sep + "a" + "\N{HORIZONTAL ELLIPSIS}" + "b" + os.sep + 10 * "c"
         )
-        self.assertEqual(_elide(test_string, ""), expect_stirng)
+        self.assertEqual(_elide(test_string, ""), expect_string)
 
     def test_elide_typed_normal(self):
         self.assertEqual(

IPython/utils/io.py

@@ -18,11 +18,6 b' from warnings import warn'
 from IPython.utils.decorators import undoc
 from .capture import CapturedIO, capture_output
 
-# setup stdin/stdout/stderr to sys.stdin/sys.stdout/sys.stderr
-devnull = open(os.devnull, "w", encoding="utf-8")
-atexit.register(devnull.close)
-
-
 class Tee(object):
     """A class to duplicate an output stream to stdout/err.
 

IPython/utils/terminal.py

@@ -91,30 +91,14 b" if os.name == 'posix':"
         _set_term_title = _set_term_title_xterm
         _restore_term_title = _restore_term_title_xterm
 elif sys.platform == 'win32':
-    try:
-        import ctypes
-
-        SetConsoleTitleW = ctypes.windll.kernel32.SetConsoleTitleW
-        SetConsoleTitleW.argtypes = [ctypes.c_wchar_p]
-    
-        def _set_term_title(title):
-            """Set terminal title using ctypes to access the Win32 APIs."""
-            SetConsoleTitleW(title)
-    except ImportError:
-        def _set_term_title(title):
-            """Set terminal title using the 'title' command."""
-            global ignore_termtitle
-
-            try:
-                # Cannot be on network share when issuing system commands
-                curr = os.getcwd()
-                os.chdir("C:")
-                ret = os.system("title " + title)
-            finally:
-                os.chdir(curr)
-            if ret:
-                # non-zero return code signals error, don't try again
-                ignore_termtitle = True
+    import ctypes
+
+    SetConsoleTitleW = ctypes.windll.kernel32.SetConsoleTitleW
+    SetConsoleTitleW.argtypes = [ctypes.c_wchar_p]
+
+    def _set_term_title(title):
+        """Set terminal title using ctypes to access the Win32 APIs."""
+        SetConsoleTitleW(title)
 
 
 def set_term_title(title):

docs/autogen_shortcuts.py

@@ -1,45 +1,98 b''
+from dataclasses import dataclass
+from inspect import getsource
 from pathlib import Path
+from typing import cast, Callable, List, Union
+from html import escape as html_escape
+import re
+
+from prompt_toolkit.keys import KEY_ALIASES
+from prompt_toolkit.key_binding import KeyBindingsBase
+from prompt_toolkit.filters import Filter, Condition
+from prompt_toolkit.shortcuts import PromptSession
 
 from IPython.terminal.shortcuts import create_ipython_shortcuts
 
-def name(c):
-    s = c.__class__.__name__
-    if s == '_Invert':
-        return '(Not: %s)' % name(c.filter)
-    if s in log_filters.keys():
-        return '(%s: %s)' % (log_filters[s], ', '.join(name(x) for x in c.filters))
-    return log_filters[s] if s in log_filters.keys() else s
 
+@dataclass
+class Shortcut:
+    #: a sequence of keys (each element on the list corresponds to pressing one or more keys)
+    keys_sequence: list[str]
+    filter: str
 
-def sentencize(s):
-    """Extract first sentence
-    """
-    s = s.replace('\n', ' ').strip().split('.')
-    s = s[0] if len(s) else s
-    try:
-        return " ".join(s.split())
-    except AttributeError:
-        return s
 
+@dataclass
+class Handler:
+    description: str
+    identifier: str
 
-def most_common(lst, n=3):
-    """Most common elements occurring more then `n` times
-    """
-    from collections import Counter
 
-    c = Counter(lst)
-    return [k for (k, v) in c.items() if k and v > n]
+@dataclass
+class Binding:
+    handler: Handler
+    shortcut: Shortcut
 
 
-def multi_filter_str(flt):
-    """Yield readable conditional filter
-    """
-    assert hasattr(flt, 'filters'), 'Conditional filter required'
-    yield name(flt)
+class _NestedFilter(Filter):
+    """Protocol reflecting non-public prompt_toolkit's `_AndList` and `_OrList`."""
+
+    filters: List[Filter]
+
+
+class _Invert(Filter):
+    """Protocol reflecting non-public prompt_toolkit's `_Invert`."""
+
+    filter: Filter
+
+
+conjunctions_labels = {"_AndList": "and", "_OrList": "or"}
 
+ATOMIC_CLASSES = {"Never", "Always", "Condition"}
+
+
+def format_filter(
+    filter_: Union[Filter, _NestedFilter, Condition, _Invert],
+    is_top_level=True,
+    skip=None,
+) -> str:
+    """Create easily readable description of the filter."""
+    s = filter_.__class__.__name__
+    if s == "Condition":
+        func = cast(Condition, filter_).func
+        name = func.__name__
+        if name == "<lambda>":
+            source = getsource(func)
+            return source.split("=")[0].strip()
+        return func.__name__
+    elif s == "_Invert":
+        operand = cast(_Invert, filter_).filter
+        if operand.__class__.__name__ in ATOMIC_CLASSES:
+            return f"not {format_filter(operand, is_top_level=False)}"
+        return f"not ({format_filter(operand, is_top_level=False)})"
+    elif s in conjunctions_labels:
+        filters = cast(_NestedFilter, filter_).filters
+        conjunction = conjunctions_labels[s]
+        glue = f" {conjunction} "
+        result = glue.join(format_filter(x, is_top_level=False) for x in filters)
+        if len(filters) > 1 and not is_top_level:
+            result = f"({result})"
+        return result
+    elif s in ["Never", "Always"]:
+        return s.lower()
+    else:
+        raise ValueError(f"Unknown filter type: {filter_}")
+
+
+def sentencize(s) -> str:
+    """Extract first sentence"""
+    s = re.split(r"\.\W", s.replace("\n", " ").strip())
+    s = s[0] if len(s) else ""
+    if not s.endswith("."):
+        s += "."
+    try:
+        return " ".join(s.split())
+    except AttributeError:
+        return s
 
-log_filters = {'_AndList': 'And', '_OrList': 'Or'}
-log_invert =  {'_Invert'}
 
 class _DummyTerminal:
     """Used as a buffer to get prompt_toolkit bindings
@@ -48,49 +101,121 b' class _DummyTerminal:'
     input_transformer_manager = None
     display_completions = None
     editing_mode = "emacs"
+    auto_suggest = None
 
 
-ipy_bindings = create_ipython_shortcuts(_DummyTerminal()).bindings
-
-dummy_docs = []  # ignore bindings without proper documentation
-
-common_docs = most_common([kb.handler.__doc__ for kb in ipy_bindings])
-if common_docs:
-    dummy_docs.extend(common_docs)
+def create_identifier(handler: Callable):
+    parts = handler.__module__.split(".")
+    name = handler.__name__
+    package = parts[0]
+    if len(parts) > 1:
+        final_module = parts[-1]
+        return f"{package}:{final_module}.{name}"
+    else:
+        return f"{package}:{name}"
+
+
+def bindings_from_prompt_toolkit(prompt_bindings: KeyBindingsBase) -> List[Binding]:
+    """Collect bindings to a simple format that does not depend on prompt-toolkit internals"""
+    bindings: List[Binding] = []
+
+    for kb in prompt_bindings.bindings:
+        bindings.append(
+            Binding(
+                handler=Handler(
+                    description=kb.handler.__doc__ or "",
+                    identifier=create_identifier(kb.handler),
+                ),
+                shortcut=Shortcut(
+                    keys_sequence=[
+                        str(k.value) if hasattr(k, "value") else k for k in kb.keys
+                    ],
+                    filter=format_filter(kb.filter, skip={"has_focus_filter"}),
+                ),
+            )
+        )
+    return bindings
+
+
+INDISTINGUISHABLE_KEYS = {**KEY_ALIASES, **{v: k for k, v in KEY_ALIASES.items()}}
+
+
+def format_prompt_keys(keys: str, add_alternatives=True) -> str:
+    """Format prompt toolkit key with modifier into an RST representation."""
+
+    def to_rst(key):
+        escaped = key.replace("\\", "\\\\")
+        return f":kbd:`{escaped}`"
+
+    keys_to_press: list[str]
+
+    prefixes = {
+        "c-s-": [to_rst("ctrl"), to_rst("shift")],
+        "s-c-": [to_rst("ctrl"), to_rst("shift")],
+        "c-": [to_rst("ctrl")],
+        "s-": [to_rst("shift")],
+    }
+
+    for prefix, modifiers in prefixes.items():
+        if keys.startswith(prefix):
+            remainder = keys[len(prefix) :]
+            keys_to_press = [*modifiers, to_rst(remainder)]
+            break
+    else:
+        keys_to_press = [to_rst(keys)]
 
-dummy_docs = list(set(dummy_docs))
+    result = " + ".join(keys_to_press)
 
-single_filter = {}
-multi_filter =  {}
-for kb in ipy_bindings:
-    doc = kb.handler.__doc__
-    if not doc or doc in dummy_docs:
-        continue
+    if keys in INDISTINGUISHABLE_KEYS and add_alternatives:
+        alternative = INDISTINGUISHABLE_KEYS[keys]
 
-    shortcut = ' '.join([k if isinstance(k, str) else k.name for k in kb.keys])
-    shortcut += shortcut.endswith('\\') and '\\' or ''
-    if hasattr(kb.filter, 'filters'):
-        flt = ' '.join(multi_filter_str(kb.filter))
-        multi_filter[(shortcut, flt)] = sentencize(doc)
-    else:
-        single_filter[(shortcut, name(kb.filter))] = sentencize(doc)
+        result = (
+            result
+            + " (or "
+            + format_prompt_keys(alternative, add_alternatives=False)
+            + ")"
+        )
 
+    return result
 
 if __name__ == '__main__':
     here = Path(__file__).parent
     dest = here / "source" / "config" / "shortcuts"
 
-    def sort_key(item):
-        k, v = item
-        shortcut, flt = k
-        return (str(shortcut), str(flt))
-
-    for filters, output_filename in [
-        (single_filter, "single_filtered"),
-        (multi_filter, "multi_filtered"),
-    ]:
-        with (dest / "{}.csv".format(output_filename)).open(
-            "w", encoding="utf-8"
-        ) as csv:
-            for (shortcut, flt), v in sorted(filters.items(), key=sort_key):
-                csv.write(":kbd:`{}`\t{}\t{}\n".format(shortcut, flt, v))
+    ipy_bindings = create_ipython_shortcuts(_DummyTerminal(), for_all_platforms=True)
+
+    session = PromptSession(key_bindings=ipy_bindings)
+    prompt_bindings = session.app.key_bindings
+
+    assert prompt_bindings
+    # Ensure that we collected the default shortcuts
+    assert len(prompt_bindings.bindings) > len(ipy_bindings.bindings)
+
+    bindings = bindings_from_prompt_toolkit(prompt_bindings)
+
+    def sort_key(binding: Binding):
+        return binding.handler.identifier, binding.shortcut.filter
+
+    filters = []
+    with (dest / "table.tsv").open("w", encoding="utf-8") as csv:
+        for binding in sorted(bindings, key=sort_key):
+            sequence = ", ".join(
+                [format_prompt_keys(keys) for keys in binding.shortcut.keys_sequence]
+            )
+            if binding.shortcut.filter == "always":
+                condition_label = "-"
+            else:
+                # we cannot fit all the columns as the filters got too complex over time
+                condition_label = "ⓘ"
+
+            csv.write(
+                "\t".join(
+                    [
+                        sequence,
+                        sentencize(binding.handler.description)
+                        + f" :raw-html:`<br>` `{binding.handler.identifier}`",
+                        f':raw-html:`<span title="{html_escape(binding.shortcut.filter)}" style="cursor: help">{condition_label}</span>`',
+                    ]
+                )
+                + "\n"
+            )

docs/environment.yml

@@ -3,14 +3,14 b' channels:'
  - conda-forge
  - defaults
 dependencies:
-  - python=3.8
-  - setuptools>=18.5
+  - python=3.10
+  - setuptools
   - sphinx>=4.2
-  - sphinx_rtd_theme>=1.0
+  - sphinx_rtd_theme
   - numpy
-  - nose
   - testpath
   - matplotlib
+  - pip
   - pip:
     - docrepr
     - prompt_toolkit

docs/source/conf.py

@@ -183,8 +183,7 b" today_fmt = '%B %d, %Y'"
 
 # Exclude these glob-style patterns when looking for source files. They are
 # relative to the source/ directory.
-exclude_patterns = []
-
+exclude_patterns = ["**.ipynb_checkpoints"]
 
 # If true, '()' will be appended to :func: etc. cross-reference text.
 #add_function_parentheses = True
@@ -211,7 +210,6 b" default_role = 'literal'"
 # given in html_static_path.
 # html_style = 'default.css'
 
-
 # The name for this set of Sphinx documents.  If None, it defaults to
 # "<project> v<release> documentation".
 #html_title = None
@@ -327,6 +325,10 b' texinfo_documents = ['
 modindex_common_prefix = ['IPython.']
 
 
+def setup(app):
+    app.add_css_file("theme_overrides.css")
+
+
 # Cleanup
 # -------
 # delete release info to avoid pickling errors from sphinx

docs/source/config/shortcuts/index.rst

@@ -4,28 +4,23 b' IPython shortcuts'
 
 Available shortcuts in an IPython terminal.
 
-.. warning::
+.. note::
 
-  This list is automatically generated, and may not hold all available
-  shortcuts. In particular, it may depend on the version of ``prompt_toolkit``
-  installed during the generation of this page.
+  This list is automatically generated. Key bindings defined in ``prompt_toolkit`` may differ
+  between installations depending on the ``prompt_toolkit`` version.
 
 
-Single Filtered shortcuts
-=========================
-
-.. csv-table::
-    :header: Shortcut,Filter,Description
-    :widths: 30, 30, 100
-    :delim: tab
-    :file: single_filtered.csv
+* Comma-separated keys, e.g. :kbd:`Esc`, :kbd:`f`, indicate a sequence which can be activated by pressing the listed keys in succession.
+* Plus-separated keys, e.g. :kbd:`Esc` + :kbd:`f` indicate a combination which requires pressing all keys simultaneously.
+* Hover over the ⓘ icon in the filter column to see when the shortcut is active.g
 
+.. role:: raw-html(raw)
+   :format: html
 
-Multi Filtered shortcuts
-========================
 
 .. csv-table::
-    :header: Shortcut,Filter,Description
-    :widths: 30, 30, 100
+    :header: Shortcut,Description and identifier,Filter
     :delim: tab
-    :file: multi_filtered.csv
+    :class: shortcuts
+    :file: table.tsv
+    :widths: 20 75 5

docs/source/whatsnew/version8.rst

@@ -2,6 +2,91 b''
  8.x Series
 ============
 
+
+.. _version 8.10.0:
+   
+IPython 8.10
+------------
+
+Out of schedule release of IPython with minor fixes to patch a potential CVE-2023-24816.
+This is a really low severity CVE that you most likely are not affected by unless:
+
+ - You are on windows.
+ - You have a custom build of Python without ``_ctypes``
+ - You cd or start IPython or Jupyter in untrusted directory which names may be
+   valid shell commands.
+
+You can read more on `the advisory
+<https://github.com/ipython/ipython/security/advisories/GHSA-29gw-9793-fvw7>`__. 
+
+In addition to fixing this CVE we also fix a couple of outstanding bugs and issues.
+
+As usual you can find the full list of PRs on GitHub under `the 8.10 milestone
+<https://github.com/ipython/ipython/milestone/112?closed=1>`__.
+
+In Particular:
+
+ - bump minimum numpy to `>=1.21` version following NEP29. :ghpull:`13930`
+ - fix for compatibility with MyPy 1.0. :ghpull:`13933`
+ - fix nbgrader stalling when IPython's ``showtraceback`` function is
+   monkeypatched. :ghpull:`13934`
+
+
+
+As this release also contains those minimal changes in addition to fixing the
+CVE I decided to bump the minor version anyway.
+
+This will not affect the normal release schedule, so IPython 8.11 is due in
+about 2 weeks.
+
+.. _version 8.9.0:
+
+IPython 8.9.0
+-------------
+
+Second release of IPython in 2023, last Friday of the month, we are back on
+track. This is a small release with a few bug-fixes, and improvements, mostly
+with respect to terminal shortcuts.
+
+
+The biggest improvement for 8.9 is a drastic amelioration of the
+auto-suggestions sponsored by D.E. Shaw and implemented by the more and more
+active contributor `@krassowski <https://github.com/krassowski>`.
+
+- ``right`` accepts a single character from suggestion
+- ``ctrl+right`` accepts a semantic token (macos default shortcuts take
+  precedence and need to be disabled to make this work)
+- ``backspace`` deletes a character and resumes hinting autosuggestions
+- ``ctrl-left`` accepts suggestion and moves cursor left one character.
+- ``backspace`` deletes a character and resumes hinting autosuggestions
+- ``down`` moves to suggestion to later in history when no lines are present below the cursors.
+- ``up`` moves to suggestion from earlier in history when no lines are present above the cursor.
+
+This is best described by the Gif posted by `@krassowski
+<https://github.com/krassowski>`, and in the PR itself :ghpull:`13888`.
+
+.. image:: ../_images/autosuggest.gif
+
+Please report any feedback in order for us to improve the user experience.
+In particular we are also working on making the shortcuts configurable.
+
+If you are interested in better terminal shortcuts, I also invite you to
+participate in issue `13879
+<https://github.com/ipython/ipython/issues/13879>`__.
+
+
+As we follow `NEP29
+<https://numpy.org/neps/nep-0029-deprecation_policy.html>`__, next version of
+IPython will officially stop supporting numpy 1.20, and will stop supporting
+Python 3.8 after April release.
+
+As usual you can find the full list of PRs on GitHub under `the 8.9 milestone
+<https://github.com/ipython/ipython/milestone/111?closed=1>`__.
+
+
+Thanks to the `D. E. Shaw group <https://deshaw.com/>`__ for sponsoring
+work on IPython and related libraries.
+
 .. _version 8.8.0:
 
 IPython 8.8.0
@@ -11,11 +96,11 b' First release of IPython in 2023 as there was no release at the end of'
 December.
 
 This is an unusually big release (relatively speaking) with more than 15 Pull
-Requests merge.
+Requests merged.
 
 Of particular interest are:
 
- - :ghpull:`13852` that replace the greedy completer and improve
+ - :ghpull:`13852` that replaces the greedy completer and improves
    completion, in particular for dictionary keys.
  - :ghpull:`13858` that adds ``py.typed`` to ``setup.cfg`` to make sure it is
    bundled in wheels.
@@ -24,7 +109,7 b' Of particular interest are:'
    believe this also needs a recent version of Traitlets.
  - :ghpull:`13865` makes the ``inspector`` class of `InteractiveShell`
    configurable.
- - :ghpull:`13880` that remove minor-version entrypoints as the minor version
+ - :ghpull:`13880` that removes minor-version entrypoints as the minor version
    entry points that would be included in the wheel would be the one of the
    Python version that was used to build the ``whl`` file.
 
@@ -48,8 +133,8 b' IPython 8.7.0'
 
 
 Small release of IPython with a couple of bug fixes and new features for this
-month. Next month is end of year, it is unclear if there will be a release close
-the new year's eve, or if the next release will be at end of January.
+month. Next month is the end of year, it is unclear if there will be a release
+close to the new year's eve, or if the next release will be at the end of January.
 
 Here are a few of the relevant fixes,
 as usual you can find the full list of PRs on GitHub under `the 8.7 milestone
@@ -73,29 +158,29 b' IPython 8.6.0'
 
 Back to a more regular release schedule (at least I try), as Friday is
 already over by more than 24h hours. This is a slightly bigger release with a
-few new features that contain no less then 25 PRs.
+few new features that contain no less than 25 PRs.
 
 We'll notably found a couple of non negligible changes:
 
 The ``install_ext`` and related functions have been removed after being
 deprecated for years. You can use pip to install extensions. ``pip`` did not
-exists when ``install_ext`` was introduced. You can still load local extensions
+exist when ``install_ext`` was introduced. You can still load local extensions
 without installing them. Just set your ``sys.path`` for example. :ghpull:`13744`
 
-IPython now have extra entry points that that the major *and minor* version of
-python. For some of you this mean that you can do a quick ``ipython3.10`` to
+IPython now has extra entry points that use the major *and minor* version of
+python. For some of you this means that you can do a quick ``ipython3.10`` to
 launch IPython from the Python 3.10 interpreter, while still using Python 3.11
 as your main Python. :ghpull:`13743`
 
-The completer matcher API have been improved. See :ghpull:`13745`. This should
+The completer matcher API has been improved. See :ghpull:`13745`. This should
 improve the type inference and improve dict keys completions in many use case.
-Tanks ``@krassowski`` for all the works, and the D.E. Shaw group for sponsoring
+Thanks ``@krassowski`` for all the work, and the D.E. Shaw group for sponsoring
 it.
 
 The color of error nodes in tracebacks can now be customized. See
-:ghpull:`13756`. This is a private attribute until someone find the time to
-properly add a configuration option. Note that with Python 3.11 that also show
-the relevant nodes in traceback, it would be good to leverage this informations
+:ghpull:`13756`. This is a private attribute until someone finds the time to
+properly add a configuration option. Note that with Python 3.11 that also shows
+the relevant nodes in traceback, it would be good to leverage this information
 (plus the "did you mean" info added on attribute errors). But that's likely work
 I won't have time to do before long, so contributions welcome.
 
@@ -108,7 +193,7 b' This mostly occurs in teaching context when incorrect values get passed around.'
 
 
 The ``?``, ``??``, and corresponding ``pinfo``, ``pinfo2`` magics can now find
-objects insides arrays. That is to say, the following now works::
+objects inside arrays. That is to say, the following now works::
 
 
    >>> def my_func(*arg, **kwargs):pass
@@ -117,7 +202,7 b' objects insides arrays. That is to say, the following now works::'
 
 
 If ``container`` define a custom ``getitem``, this __will__ trigger the custom
-method. So don't put side effects in your ``getitems``. Thanks the D.E. Shaw
+method. So don't put side effects in your ``getitems``. Thanks to the D.E. Shaw
 group for the request and sponsoring the work.
 
 
@@ -143,17 +228,17 b' an bug fixes.'
 Many thanks to everybody who contributed PRs for your patience in review and
 merges.
 
-Here is a non exhaustive list of changes that have been implemented for IPython
+Here is a non-exhaustive list of changes that have been implemented for IPython
 8.5.0. As usual you can find the full list of issues and PRs tagged with `the
 8.5 milestone
 <https://github.com/ipython/ipython/pulls?q=is%3Aclosed+milestone%3A8.5+>`__.
 
- - Added shortcut for accepting auto suggestion. The End key shortcut for
+ - Added a shortcut for accepting auto suggestion. The End key shortcut for
    accepting auto-suggestion This binding works in Vi mode too, provided
    ``TerminalInteractiveShell.emacs_bindings_in_vi_insert_mode`` is set to be
    ``True`` :ghpull:`13566`.
 
- - No popup in window for latex generation w hen generating latex (e.g. via
+ - No popup in window for latex generation when generating latex (e.g. via
    `_latex_repr_`) no popup window is shows under Windows. :ghpull:`13679`
 
  - Fixed error raised when attempting to tab-complete an input string with
@@ -269,12 +354,12 b' IPython 8.3.0'
 
 
  - :ghpull:`13600`, ``pre_run_*``-hooks will now have a ``cell_id`` attribute on
-   the info object when frontend provide it. This has been backported to 7.33
+   the info object when frontend provides it. This has been backported to 7.33
 
  - :ghpull:`13624`, fixed :kbd:`End` key being broken after accepting an
    auto-suggestion.
 
- - :ghpull:`13657` fix issue where history from different sessions would be mixed.
+ - :ghpull:`13657` fixed an issue where history from different sessions would be mixed.
 
 .. _version 8.2.0:
 
@@ -292,8 +377,8 b' IPython 8.2 mostly bring bugfixes to IPython.'
  - Fixes to ``ultratb`` ipdb support when used outside of IPython. :ghpull:`13498`
 
 
-I am still trying to fix and investigate :ghissue:`13598`, which seem to be
-random, and would appreciate help if you find  reproducible minimal case. I've
+I am still trying to fix and investigate :ghissue:`13598`, which seems to be
+random, and would appreciate help if you find a reproducible minimal case. I've
 tried to make various changes to the codebase to mitigate it, but a proper fix
 will be difficult without understanding the cause.
 
@@ -322,7 +407,7 b' IPython 8.1.0'
 -------------
 
 IPython 8.1 is the first minor release after 8.0 and fixes a number of bugs and
-Update a few behavior that were problematic with the 8.0 as with many new major
+updates a few behaviors that were problematic with the 8.0 as with many new major
 release.
 
 Note that beyond the changes listed here, IPython 8.1.0 also contains all the
@@ -373,8 +458,8 b' We want to remind users that IPython is part of the Jupyter organisations, and'
 thus governed by a Code of Conduct. Some of the behavior we have seen on GitHub is not acceptable.
 Abuse and non-respectful comments on discussion will not be tolerated.
 
-Many thanks to all the contributors to this release, many of the above fixed issue and
-new features where done by first time contributors, showing there is still
+Many thanks to all the contributors to this release, many of the above fixed issues and
+new features were done by first time contributors, showing there is still
 plenty of easy contribution possible in IPython
 . You can find all individual contributions
 to this milestone `on github <https://github.com/ipython/ipython/milestone/91>`__.
@@ -435,7 +520,7 b' IPython 8.0'
 
 IPython 8.0 is bringing a large number of new features and improvements to both the
 user of the terminal and of the kernel via Jupyter. The removal of compatibility
-with older version of Python is also the opportunity to do a couple of
+with an older version of Python is also the opportunity to do a couple of
 performance improvements in particular with respect to startup time.
 The 8.x branch started diverging from its predecessor around IPython 7.12
 (January 2020).
@@ -444,7 +529,7 b' This release contains 250+ pull requests, in addition to many of the features'
 and backports that have made it to the 7.x branch. Please see the
 `8.0 milestone <https://github.com/ipython/ipython/milestone/73?closed=1>`__ for the full list of pull requests.
 
-Please feel free to send pull requests to updates those notes after release,
+Please feel free to send pull requests to update those notes after release,
 I have likely forgotten a few things reviewing 250+ PRs.
 
 Dependencies changes/downstream packaging
@@ -459,8 +544,8 b' looking for help to do so.'
  - minimal Python is now 3.8
  - ``nose`` is not a testing requirement anymore
  - ``pytest`` replaces nose.
- - ``iptest``/``iptest3`` cli entrypoints do not exists anymore.
- - minimum officially support ``numpy`` version has been bumped, but this should
+ - ``iptest``/``iptest3`` cli entrypoints do not exist anymore.
+ - the minimum officially ​supported ``numpy`` version has been bumped, but this should
    not have much effect on packaging.
 
 

setup.cfg

@@ -37,7 +37,7 b' install_requires ='
     matplotlib-inline
     pexpect>4.3; sys_platform != "win32"
     pickleshare
-    prompt_toolkit>=3.0.11,<3.1.0
+    prompt_toolkit>=3.0.30,<3.1.0
     pygments>=2.4.0
     stack_data
     traitlets>=5
@@ -79,7 +79,7 b' test_extra ='
     curio
     matplotlib!=3.2.0
     nbformat
-    numpy>=1.20
+    numpy>=1.21
     pandas
     trio
 all =