Implement traversal of autosuggestions and by-character fill

IPython/terminal/shortcuts/auto_suggest.py

@@ -0,0 +1,255
+import re
+import tokenize
+from io import StringIO
+from typing import Callable, List, Optional, Union
+
+from prompt_toolkit.buffer import Buffer
+from prompt_toolkit.key_binding import KeyPressEvent
+from prompt_toolkit.key_binding.bindings import named_commands as nc
+from prompt_toolkit.auto_suggest import AutoSuggestFromHistory, Suggestion
+from prompt_toolkit.document import Document
+from prompt_toolkit.history import History
+from prompt_toolkit.shortcuts import PromptSession
+
+from IPython.utils.tokenutil import generate_tokens
+
+
+def _get_query(document: Document):
+    return document.text.rsplit("\n", 1)[-1]
+
+
+class NavigableAutoSuggestFromHistory(AutoSuggestFromHistory):
+    """ """
+
+    def __init__(
+        self,
+    ):
+        self.skip_lines = 0
+        self._connected_apps = []
+
+    def reset_history_position(self, _: Buffer):
+        self.skip_lines = 0
+
+    def disconnect(self):
+        for pt_app in self._connected_apps:
+            text_insert_event = pt_app.default_buffer.on_text_insert
+            text_insert_event.remove_handler(self.reset_history_position)
+
+    def connect(self, pt_app: PromptSession):
+        self._connected_apps.append(pt_app)
+        pt_app.default_buffer.on_text_insert.add_handler(self.reset_history_position)
+
+    def get_suggestion(
+        self, buffer: Buffer, document: Document
+    ) -> Optional[Suggestion]:
+        text = _get_query(document)
+
+        if text.strip():
+            for suggestion, _ in self._find_next_match(
+                text, self.skip_lines, buffer.history
+            ):
+                return Suggestion(suggestion)
+
+        return None
+
+    def _find_match(
+        self, text: str, skip_lines: float, history: History, previous: bool
+    ):
+        line_number = -1
+
+        for string in reversed(list(history.get_strings())):
+            for line in reversed(string.splitlines()):
+                line_number += 1
+                if not previous and line_number < skip_lines:
+                    continue
+                # do not return empty suggestions as these
+                # close the auto-suggestion overlay (and are useless)
+                if line.startswith(text) and len(line) > len(text):
+                    yield line[len(text) :], line_number
+                if previous and line_number >= skip_lines:
+                    return
+
+    def _find_next_match(self, text: str, skip_lines: float, history: History):
+        return self._find_match(text, skip_lines, history, previous=False)
+
+    def _find_previous_match(self, text: str, skip_lines: float, history: History):
+        return reversed(
+            list(self._find_match(text, skip_lines, history, previous=True))
+        )
+
+    def up(self, query: str, other_than: str, history: History):
+        for suggestion, line_number in self._find_next_match(
+            query, self.skip_lines, history
+        ):
+            # if user has history ['very.a', 'very', 'very.b'] and typed 'very'
+            # we want to switch from 'very.b' to 'very.a' because a) if they
+            # suggestion equals current text, prompt-toolit aborts suggesting
+            # b) user likely would not be interested in 'very' anyways (they
+            # already typed it).
+            if query + suggestion != other_than:
+                self.skip_lines = line_number
+                break
+        else:
+            # no matches found, cycle back to beginning
+            self.skip_lines = 0
+
+    def down(self, query: str, other_than: str, history: History):
+        for suggestion, line_number in self._find_previous_match(
+            query, self.skip_lines, history
+        ):
+            if query + suggestion != other_than:
+                self.skip_lines = line_number
+                break
+        else:
+            # no matches found, cycle to end
+            for suggestion, line_number in self._find_previous_match(
+                query, float("Inf"), history
+            ):
+                if query + suggestion != other_than:
+                    self.skip_lines = line_number
+                    break
+
+
+# Needed for to accept autosuggestions in vi insert mode
+def accept_in_vi_insert_mode(event: KeyPressEvent):
+    """Apply autosuggestion if at end of line."""
+    b = event.current_buffer
+    d = b.document
+    after_cursor = d.text[d.cursor_position :]
+    lines = after_cursor.split("\n")
+    end_of_current_line = lines[0].strip()
+    suggestion = b.suggestion
+    if (suggestion is not None) and (suggestion.text) and (end_of_current_line == ""):
+        b.insert_text(suggestion.text)
+    else:
+        nc.end_of_line(event)
+
+
+def accept(event: KeyPressEvent):
+    """Accept autosuggestion"""
+    b = event.current_buffer
+    suggestion = b.suggestion
+    if suggestion:
+        b.insert_text(suggestion.text)
+    else:
+        nc.forward_char(event)
+
+
+def accept_word(event: KeyPressEvent):
+    """Fill partial autosuggestion by word"""
+    b = event.current_buffer
+    suggestion = b.suggestion
+    if suggestion:
+        t = re.split(r"(\S+\s+)", suggestion.text)
+        b.insert_text(next((x for x in t if x), ""))
+    else:
+        nc.forward_word(event)
+
+
+def accept_character(event: KeyPressEvent):
+    """Fill partial autosuggestion by character"""
+    b = event.current_buffer
+    suggestion = b.suggestion
+    if suggestion and suggestion.text:
+        b.insert_text(suggestion.text[0])
+
+
+def accept_token(event: KeyPressEvent):
+    """Fill partial autosuggestion by token"""
+    b = event.current_buffer
+    suggestion = b.suggestion
+
+    if suggestion:
+        prefix = _get_query(b.document)
+        text = prefix + suggestion.text
+
+        tokens: List[Optional[str]] = [None, None, None]
+        substrings = [""]
+        i = 0
+
+        for token in generate_tokens(StringIO(text).readline):
+            if token.type == tokenize.NEWLINE:
+                index = len(text)
+            else:
+                index = text.index(token[1], len(substrings[-1]))
+            substrings.append(text[:index])
+            tokenized_so_far = substrings[-1]
+            if tokenized_so_far.startswith(prefix):
+                if i == 0 and len(tokenized_so_far) > len(prefix):
+                    tokens[0] = tokenized_so_far[len(prefix) :]
+                    substrings.append(tokenized_so_far)
+                    i += 1
+                tokens[i] = token[1]
+                if i == 2:
+                    break
+                i += 1
+
+        if tokens[0]:
+            to_insert: str
+            insert_text = substrings[-2]
+            if tokens[1] and len(tokens[1]) == 1:
+                insert_text = substrings[-1]
+            to_insert = insert_text[len(prefix) :]
+            b.insert_text(to_insert)
+            return
+
+    nc.forward_word(event)
+
+
+Provider = Union[AutoSuggestFromHistory, NavigableAutoSuggestFromHistory, None]
+
+
+def _swap_autosuggestion(
+    buffer: Buffer,
+    provider: NavigableAutoSuggestFromHistory,
+    direction_method: Callable,
+):
+    """
+    We skip most recent history entry (in either direction) if it equals the
+    current autosuggestion because if user cycles when auto-suggestion is shown
+    they most likely want something else than what was suggested (othewrise
+    they would have accepted the suggestion).
+    """
+    suggestion = buffer.suggestion
+    if not suggestion:
+        return
+
+    query = _get_query(buffer.document)
+    current = query + suggestion.text
+
+    direction_method(query=query, other_than=current, history=buffer.history)
+
+    new_suggestion = provider.get_suggestion(buffer, buffer.document)
+    buffer.suggestion = new_suggestion
+
+
+def swap_autosuggestion_up(provider: Provider):
+    def swap_autosuggestion_up(event: KeyPressEvent):
+        """Get next autosuggestion from history."""
+        if not isinstance(provider, NavigableAutoSuggestFromHistory):
+            return
+
+        return _swap_autosuggestion(
+            buffer=event.current_buffer, provider=provider, direction_method=provider.up
+        )
+
+    swap_autosuggestion_up.__name__ = "swap_autosuggestion_up"
+    return swap_autosuggestion_up
+
+
+def swap_autosuggestion_down(
+    provider: Union[AutoSuggestFromHistory, NavigableAutoSuggestFromHistory, None]
+):
+    def swap_autosuggestion_down(event: KeyPressEvent):
+        """Get previous autosuggestion from history."""
+        if not isinstance(provider, NavigableAutoSuggestFromHistory):
+            return
+
+        return _swap_autosuggestion(
+            buffer=event.current_buffer,
+            provider=provider,
+            direction_method=provider.down,
+        )
+
+    swap_autosuggestion_down.__name__ = "swap_autosuggestion_down"
+    return swap_autosuggestion_down

IPython/terminal/interactiveshell.py

@@ -4,6 +4,7 import asyncio
 import os
 import sys
 from warnings import warn
+from typing import Union as UnionType
 
 from IPython.core.async_helpers import get_asyncio_loop
 from IPython.core.interactiveshell import InteractiveShell, InteractiveShellABC
@@ -49,6 +50,7 from .pt_inputhooks import get_inputhook_name_and_func
 from .prompts import Prompts, ClassicPrompts, RichPromptDisplayHook
 from .ptutils import IPythonPTCompleter, IPythonPTLexer
 from .shortcuts import create_ipython_shortcuts
+from .shortcuts.auto_suggest import NavigableAutoSuggestFromHistory
 
 PTK3 = ptk_version.startswith('3.')
 
@@ -183,7 +185,7 class TerminalInteractiveShell(InteractiveShell):
                                      'menus, decrease for short and wide.'
                             ).tag(config=True)
 
-    pt_app = None
+    pt_app: UnionType[PromptSession, None] = None
     debugger_history = None
 
     debugger_history_file = Unicode(
@@ -376,18 +378,25 class TerminalInteractiveShell(InteractiveShell):
     ).tag(config=True)
 
     autosuggestions_provider = Unicode(
-        "AutoSuggestFromHistory",
+        "NavigableAutoSuggestFromHistory",
         help="Specifies from which source automatic suggestions are provided. "
-        "Can be set to `'AutoSuggestFromHistory`' or `None` to disable"
-        "automatic suggestions. Default is `'AutoSuggestFromHistory`'.",
+        "Can be set to ``'NavigableAutoSuggestFromHistory'`` (:kbd:`up` and "
+        ":kbd:`down` swap suggestions), ``'AutoSuggestFromHistory'``, "
+        " or ``None`` to disable automatic suggestions. "
+        "Default is `'NavigableAutoSuggestFromHistory`'.",
         allow_none=True,
     ).tag(config=True)
 
     def _set_autosuggestions(self, provider):
+        # disconnect old handler
+        if self.auto_suggest and isinstance(self.auto_suggest, NavigableAutoSuggestFromHistory):
+            self.auto_suggest.disconnect()
         if provider is None:
             self.auto_suggest = None
         elif provider == "AutoSuggestFromHistory":
             self.auto_suggest = AutoSuggestFromHistory()
+        elif provider == "NavigableAutoSuggestFromHistory":
+            self.auto_suggest = NavigableAutoSuggestFromHistory()
         else:
             raise ValueError("No valid provider.")
         if self.pt_app:
@@ -462,6 +471,8 class TerminalInteractiveShell(InteractiveShell):
             tempfile_suffix=".py",
             **self._extra_prompt_options()
         )
+        if isinstance(self.auto_suggest, NavigableAutoSuggestFromHistory):
+            self.auto_suggest.connect(self.pt_app)
 
     def _make_style_from_name_or_cls(self, name_or_cls):
         """
@@ -649,6 +660,7 class TerminalInteractiveShell(InteractiveShell):
 
     def __init__(self, *args, **kwargs):
         super(TerminalInteractiveShell, self).__init__(*args, **kwargs)
+        self.auto_suggest: UnionType[AutoSuggestFromHistory, NavigableAutoSuggestFromHistory, None] = None
         self._set_autosuggestions(self.autosuggestions_provider)
         self.init_prompt_toolkit_cli()
         self.init_term_title()

IPython/terminal/shortcuts/__init__.py

@@ -34,12 +34,24 from prompt_toolkit.key_binding.vi_state import InputMode, ViState
 from prompt_toolkit.layout.layout import FocusableElement
 
 from IPython.utils.decorators import undoc
-from . import auto_match as match, autosuggestions
+from . import auto_match as match, auto_suggest
 
 
 __all__ = ["create_ipython_shortcuts"]
 
 
+try:
+    # only added in 3.0.30
+    from prompt_toolkit.filters import has_suggestion
+except ImportError:
+
+    @undoc
+    @Condition
+    def has_suggestion():
+        buffer = get_app().current_buffer
+        return buffer.suggestion is not None and buffer.suggestion.text != ""
+
+
 @undoc
 @Condition
 def cursor_in_leading_ws():
@@ -324,16 +336,27 def create_ipython_shortcuts(shell, for_all_platforms: bool = False):
 
     # autosuggestions
     kb.add("end", filter=has_focus(DEFAULT_BUFFER) & (ebivim | ~vi_insert_mode))(
-        autosuggestions.accept_in_vi_insert_mode
+        auto_suggest.accept_in_vi_insert_mode
     )
     kb.add("c-e", filter=focused_insert_vi & ebivim)(
-        autosuggestions.accept_in_vi_insert_mode
+        auto_suggest.accept_in_vi_insert_mode
+    )
+    kb.add("c-f", filter=focused_insert_vi)(auto_suggest.accept)
+    kb.add("escape", "f", filter=focused_insert_vi & ebivim)(auto_suggest.accept_word)
+    kb.add("c-right", filter=has_suggestion & has_focus(DEFAULT_BUFFER))(
+        auto_suggest.accept_token
+    )
+    from functools import partial
+
+    kb.add("up", filter=has_suggestion & has_focus(DEFAULT_BUFFER))(
+        auto_suggest.swap_autosuggestion_up(shell.auto_suggest)
+    )
+    kb.add("down", filter=has_suggestion & has_focus(DEFAULT_BUFFER))(
+        auto_suggest.swap_autosuggestion_down(shell.auto_suggest)
     )
-    kb.add("c-f", filter=focused_insert_vi)(autosuggestions.accept)
-    kb.add("escape", "f", filter=focused_insert_vi & ebivim)(
-        autosuggestions.accept_word
+    kb.add("right", filter=has_suggestion & has_focus(DEFAULT_BUFFER))(
+        auto_suggest.accept_character
     )
-    kb.add("c-right", filter=has_focus(DEFAULT_BUFFER))(autosuggestions.accept_token)
 
     # Simple Control keybindings
     key_cmd_dict = {

IPython/terminal/tests/test_shortcuts.py

@@ -1,5 +1,5
 import pytest
-from IPython.terminal.shortcuts.autosuggestions import (
+from IPython.terminal.shortcuts.auto_suggest import (
     accept_in_vi_insert_mode,
     accept_token,
 )

docs/autogen_shortcuts.py

@@ -101,6 +101,7 class _DummyTerminal:
     input_transformer_manager = None
     display_completions = None
     editing_mode = "emacs"
+    auto_suggest = None
 
 
 def create_identifier(handler: Callable):