Merge branch 'main' into create_app_session_for_debugger_prompt

IPython/terminal/shortcuts/__init__.py

@@ -0,0 +1,670 b''
+"""
+Module to define and register Terminal IPython shortcuts with
+:mod:`prompt_toolkit`
+"""
+
+# Copyright (c) IPython Development Team.
+# Distributed under the terms of the Modified BSD License.
+
+import os
+import re
+import signal
+import sys
+import warnings
+from typing import Callable, Dict, Union
+
+from prompt_toolkit.application.current import get_app
+from prompt_toolkit.enums import DEFAULT_BUFFER, SEARCH_BUFFER
+from prompt_toolkit.filters import Condition, emacs_insert_mode, has_completions
+from prompt_toolkit.filters import has_focus as has_focus_impl
+from prompt_toolkit.filters import (
+    has_selection,
+    has_suggestion,
+    vi_insert_mode,
+    vi_mode,
+)
+from prompt_toolkit.key_binding import KeyBindings
+from prompt_toolkit.key_binding.bindings import named_commands as nc
+from prompt_toolkit.key_binding.bindings.completion import (
+    display_completions_like_readline,
+)
+from prompt_toolkit.key_binding.vi_state import InputMode, ViState
+from prompt_toolkit.layout.layout import FocusableElement
+
+from IPython.terminal.shortcuts import auto_match as match
+from IPython.terminal.shortcuts import auto_suggest
+from IPython.utils.decorators import undoc
+
+__all__ = ["create_ipython_shortcuts"]
+
+
+@undoc
+@Condition
+def cursor_in_leading_ws():
+    before = get_app().current_buffer.document.current_line_before_cursor
+    return (not before) or before.isspace()
+
+
+def has_focus(value: FocusableElement):
+    """Wrapper around has_focus adding a nice `__name__` to tester function"""
+    tester = has_focus_impl(value).func
+    tester.__name__ = f"is_focused({value})"
+    return Condition(tester)
+
+
+@undoc
+@Condition
+def has_line_below() -> bool:
+    document = get_app().current_buffer.document
+    return document.cursor_position_row < len(document.lines) - 1
+
+
+@undoc
+@Condition
+def has_line_above() -> bool:
+    document = get_app().current_buffer.document
+    return document.cursor_position_row != 0
+
+
+def create_ipython_shortcuts(shell, for_all_platforms: bool = False) -> KeyBindings:
+    """Set up the prompt_toolkit keyboard shortcuts for IPython.
+
+    Parameters
+    ----------
+    shell: InteractiveShell
+        The current IPython shell Instance
+    for_all_platforms: bool (default false)
+        This parameter is mostly used in generating the documentation
+        to create the shortcut binding for all the platforms, and export
+        them.
+
+    Returns
+    -------
+    KeyBindings
+        the keybinding instance for prompt toolkit.
+
+    """
+    # Warning: if possible, do NOT define handler functions in the locals
+    # scope of this function, instead define functions in the global
+    # scope, or a separate module, and include a user-friendly docstring
+    # describing the action.
+
+    kb = KeyBindings()
+    insert_mode = vi_insert_mode | emacs_insert_mode
+
+    if getattr(shell, "handle_return", None):
+        return_handler = shell.handle_return(shell)
+    else:
+        return_handler = newline_or_execute_outer(shell)
+
+    kb.add("enter", filter=(has_focus(DEFAULT_BUFFER) & ~has_selection & insert_mode))(
+        return_handler
+    )
+
+    @Condition
+    def ebivim():
+        return shell.emacs_bindings_in_vi_insert_mode
+
+    @kb.add(
+        "escape",
+        "enter",
+        filter=(has_focus(DEFAULT_BUFFER) & ~has_selection & insert_mode & ebivim),
+    )
+    def reformat_and_execute(event):
+        """Reformat code and execute it"""
+        reformat_text_before_cursor(
+            event.current_buffer, event.current_buffer.document, shell
+        )
+        event.current_buffer.validate_and_handle()
+
+    kb.add("c-\\")(quit)
+
+    kb.add("c-p", filter=(vi_insert_mode & has_focus(DEFAULT_BUFFER)))(
+        previous_history_or_previous_completion
+    )
+
+    kb.add("c-n", filter=(vi_insert_mode & has_focus(DEFAULT_BUFFER)))(
+        next_history_or_next_completion
+    )
+
+    kb.add("c-g", filter=(has_focus(DEFAULT_BUFFER) & has_completions))(
+        dismiss_completion
+    )
+
+    kb.add("c-c", filter=has_focus(DEFAULT_BUFFER))(reset_buffer)
+
+    kb.add("c-c", filter=has_focus(SEARCH_BUFFER))(reset_search_buffer)
+
+    supports_suspend = Condition(lambda: hasattr(signal, "SIGTSTP"))
+    kb.add("c-z", filter=supports_suspend)(suspend_to_bg)
+
+    # Ctrl+I == Tab
+    kb.add(
+        "tab",
+        filter=(
+            has_focus(DEFAULT_BUFFER)
+            & ~has_selection
+            & insert_mode
+            & cursor_in_leading_ws
+        ),
+    )(indent_buffer)
+    kb.add("c-o", filter=(has_focus(DEFAULT_BUFFER) & emacs_insert_mode))(
+        newline_autoindent_outer(shell.input_transformer_manager)
+    )
+
+    kb.add("f2", filter=has_focus(DEFAULT_BUFFER))(open_input_in_editor)
+
+    @Condition
+    def auto_match():
+        return shell.auto_match
+
+    def all_quotes_paired(quote, buf):
+        paired = True
+        i = 0
+        while i < len(buf):
+            c = buf[i]
+            if c == quote:
+                paired = not paired
+            elif c == "\\":
+                i += 1
+            i += 1
+        return paired
+
+    focused_insert = (vi_insert_mode | emacs_insert_mode) & has_focus(DEFAULT_BUFFER)
+    _preceding_text_cache: Dict[Union[str, Callable], Condition] = {}
+    _following_text_cache: Dict[Union[str, Callable], Condition] = {}
+
+    def preceding_text(pattern: Union[str, Callable]):
+        if pattern in _preceding_text_cache:
+            return _preceding_text_cache[pattern]
+
+        if callable(pattern):
+
+            def _preceding_text():
+                app = get_app()
+                before_cursor = app.current_buffer.document.current_line_before_cursor
+                # mypy can't infer if(callable): https://github.com/python/mypy/issues/3603
+                return bool(pattern(before_cursor))  # type: ignore[operator]
+
+        else:
+            m = re.compile(pattern)
+
+            def _preceding_text():
+                app = get_app()
+                before_cursor = app.current_buffer.document.current_line_before_cursor
+                return bool(m.match(before_cursor))
+
+            _preceding_text.__name__ = f"preceding_text({pattern!r})"
+
+        condition = Condition(_preceding_text)
+        _preceding_text_cache[pattern] = condition
+        return condition
+
+    def following_text(pattern):
+        try:
+            return _following_text_cache[pattern]
+        except KeyError:
+            pass
+        m = re.compile(pattern)
+
+        def _following_text():
+            app = get_app()
+            return bool(m.match(app.current_buffer.document.current_line_after_cursor))
+
+        _following_text.__name__ = f"following_text({pattern!r})"
+
+        condition = Condition(_following_text)
+        _following_text_cache[pattern] = condition
+        return condition
+
+    @Condition
+    def not_inside_unclosed_string():
+        app = get_app()
+        s = app.current_buffer.document.text_before_cursor
+        # remove escaped quotes
+        s = s.replace('\\"', "").replace("\\'", "")
+        # remove triple-quoted string literals
+        s = re.sub(r"(?:\"\"\"[\s\S]*\"\"\"|'''[\s\S]*''')", "", s)
+        # remove single-quoted string literals
+        s = re.sub(r"""(?:"[^"]*["\n]|'[^']*['\n])""", "", s)
+        return not ('"' in s or "'" in s)
+
+    # auto match
+    for key, cmd in match.auto_match_parens.items():
+        kb.add(key, filter=focused_insert & auto_match & following_text(r"[,)}\]]|$"))(
+            cmd
+        )
+
+    # raw string
+    for key, cmd in match.auto_match_parens_raw_string.items():
+        kb.add(
+            key,
+            filter=focused_insert & auto_match & preceding_text(r".*(r|R)[\"'](-*)$"),
+        )(cmd)
+
+    kb.add(
+        '"',
+        filter=focused_insert
+        & auto_match
+        & not_inside_unclosed_string
+        & preceding_text(lambda line: all_quotes_paired('"', line))
+        & following_text(r"[,)}\]]|$"),
+    )(match.double_quote)
+
+    kb.add(
+        "'",
+        filter=focused_insert
+        & auto_match
+        & not_inside_unclosed_string
+        & preceding_text(lambda line: all_quotes_paired("'", line))
+        & following_text(r"[,)}\]]|$"),
+    )(match.single_quote)
+
+    kb.add(
+        '"',
+        filter=focused_insert
+        & auto_match
+        & not_inside_unclosed_string
+        & preceding_text(r'^.*""$'),
+    )(match.docstring_double_quotes)
+
+    kb.add(
+        "'",
+        filter=focused_insert
+        & auto_match
+        & not_inside_unclosed_string
+        & preceding_text(r"^.*''$"),
+    )(match.docstring_single_quotes)
+
+    # just move cursor
+    kb.add(")", filter=focused_insert & auto_match & following_text(r"^\)"))(
+        match.skip_over
+    )
+    kb.add("]", filter=focused_insert & auto_match & following_text(r"^\]"))(
+        match.skip_over
+    )
+    kb.add("}", filter=focused_insert & auto_match & following_text(r"^\}"))(
+        match.skip_over
+    )
+    kb.add('"', filter=focused_insert & auto_match & following_text('^"'))(
+        match.skip_over
+    )
+    kb.add("'", filter=focused_insert & auto_match & following_text("^'"))(
+        match.skip_over
+    )
+
+    kb.add(
+        "backspace",
+        filter=focused_insert
+        & preceding_text(r".*\($")
+        & auto_match
+        & following_text(r"^\)"),
+    )(match.delete_pair)
+    kb.add(
+        "backspace",
+        filter=focused_insert
+        & preceding_text(r".*\[$")
+        & auto_match
+        & following_text(r"^\]"),
+    )(match.delete_pair)
+    kb.add(
+        "backspace",
+        filter=focused_insert
+        & preceding_text(r".*\{$")
+        & auto_match
+        & following_text(r"^\}"),
+    )(match.delete_pair)
+    kb.add(
+        "backspace",
+        filter=focused_insert
+        & preceding_text('.*"$')
+        & auto_match
+        & following_text('^"'),
+    )(match.delete_pair)
+    kb.add(
+        "backspace",
+        filter=focused_insert
+        & preceding_text(r".*'$")
+        & auto_match
+        & following_text(r"^'"),
+    )(match.delete_pair)
+
+    if shell.display_completions == "readlinelike":
+        kb.add(
+            "c-i",
+            filter=(
+                has_focus(DEFAULT_BUFFER)
+                & ~has_selection
+                & insert_mode
+                & ~cursor_in_leading_ws
+            ),
+        )(display_completions_like_readline)
+
+    if sys.platform == "win32" or for_all_platforms:
+        kb.add("c-v", filter=(has_focus(DEFAULT_BUFFER) & ~vi_mode))(win_paste)
+
+    focused_insert_vi = has_focus(DEFAULT_BUFFER) & vi_insert_mode
+
+    # autosuggestions
+    @Condition
+    def navigable_suggestions():
+        return isinstance(
+            shell.auto_suggest, auto_suggest.NavigableAutoSuggestFromHistory
+        )
+
+    kb.add("end", filter=has_focus(DEFAULT_BUFFER) & (ebivim | ~vi_insert_mode))(
+        auto_suggest.accept_in_vi_insert_mode
+    )
+    kb.add("c-e", filter=focused_insert_vi & ebivim)(
+        auto_suggest.accept_in_vi_insert_mode
+    )
+    kb.add("c-f", filter=focused_insert_vi)(auto_suggest.accept)
+    kb.add("escape", "f", filter=focused_insert_vi & ebivim)(auto_suggest.accept_word)
+    kb.add("c-right", filter=has_suggestion & has_focus(DEFAULT_BUFFER))(
+        auto_suggest.accept_token
+    )
+    kb.add(
+        "escape", filter=has_suggestion & has_focus(DEFAULT_BUFFER) & emacs_insert_mode
+    )(auto_suggest.discard)
+    kb.add(
+        "up",
+        filter=navigable_suggestions
+        & ~has_line_above
+        & has_suggestion
+        & has_focus(DEFAULT_BUFFER),
+    )(auto_suggest.swap_autosuggestion_up(shell.auto_suggest))
+    kb.add(
+        "down",
+        filter=navigable_suggestions
+        & ~has_line_below
+        & has_suggestion
+        & has_focus(DEFAULT_BUFFER),
+    )(auto_suggest.swap_autosuggestion_down(shell.auto_suggest))
+    kb.add(
+        "up", filter=has_line_above & navigable_suggestions & has_focus(DEFAULT_BUFFER)
+    )(auto_suggest.up_and_update_hint)
+    kb.add(
+        "down",
+        filter=has_line_below & navigable_suggestions & has_focus(DEFAULT_BUFFER),
+    )(auto_suggest.down_and_update_hint)
+    kb.add("right", filter=has_suggestion & has_focus(DEFAULT_BUFFER))(
+        auto_suggest.accept_character
+    )
+    kb.add("c-left", filter=has_suggestion & has_focus(DEFAULT_BUFFER))(
+        auto_suggest.accept_and_move_cursor_left
+    )
+    kb.add("c-down", filter=has_suggestion & has_focus(DEFAULT_BUFFER))(
+        auto_suggest.accept_and_keep_cursor
+    )
+    kb.add("backspace", filter=has_suggestion & has_focus(DEFAULT_BUFFER))(
+        auto_suggest.backspace_and_resume_hint
+    )
+
+    # Simple Control keybindings
+    key_cmd_dict = {
+        "c-a": nc.beginning_of_line,
+        "c-b": nc.backward_char,
+        "c-k": nc.kill_line,
+        "c-w": nc.backward_kill_word,
+        "c-y": nc.yank,
+        "c-_": nc.undo,
+    }
+
+    for key, cmd in key_cmd_dict.items():
+        kb.add(key, filter=focused_insert_vi & ebivim)(cmd)
+
+    # Alt and Combo Control keybindings
+    keys_cmd_dict = {
+        # Control Combos
+        ("c-x", "c-e"): nc.edit_and_execute,
+        ("c-x", "e"): nc.edit_and_execute,
+        # Alt
+        ("escape", "b"): nc.backward_word,
+        ("escape", "c"): nc.capitalize_word,
+        ("escape", "d"): nc.kill_word,
+        ("escape", "h"): nc.backward_kill_word,
+        ("escape", "l"): nc.downcase_word,
+        ("escape", "u"): nc.uppercase_word,
+        ("escape", "y"): nc.yank_pop,
+        ("escape", "."): nc.yank_last_arg,
+    }
+
+    for keys, cmd in keys_cmd_dict.items():
+        kb.add(*keys, filter=focused_insert_vi & ebivim)(cmd)
+
+    def get_input_mode(self):
+        app = get_app()
+        app.ttimeoutlen = shell.ttimeoutlen
+        app.timeoutlen = shell.timeoutlen
+
+        return self._input_mode
+
+    def set_input_mode(self, mode):
+        shape = {InputMode.NAVIGATION: 2, InputMode.REPLACE: 4}.get(mode, 6)
+        cursor = "\x1b[{} q".format(shape)
+
+        sys.stdout.write(cursor)
+        sys.stdout.flush()
+
+        self._input_mode = mode
+
+    if shell.editing_mode == "vi" and shell.modal_cursor:
+        ViState._input_mode = InputMode.INSERT  # type: ignore
+        ViState.input_mode = property(get_input_mode, set_input_mode)  # type: ignore
+    return kb
+
+
+def reformat_text_before_cursor(buffer, document, shell):
+    text = buffer.delete_before_cursor(len(document.text[: document.cursor_position]))
+    try:
+        formatted_text = shell.reformat_handler(text)
+        buffer.insert_text(formatted_text)
+    except Exception as e:
+        buffer.insert_text(text)
+
+
+def newline_or_execute_outer(shell):
+    def newline_or_execute(event):
+        """When the user presses return, insert a newline or execute the code."""
+        b = event.current_buffer
+        d = b.document
+
+        if b.complete_state:
+            cc = b.complete_state.current_completion
+            if cc:
+                b.apply_completion(cc)
+            else:
+                b.cancel_completion()
+            return
+
+        # If there's only one line, treat it as if the cursor is at the end.
+        # See https://github.com/ipython/ipython/issues/10425
+        if d.line_count == 1:
+            check_text = d.text
+        else:
+            check_text = d.text[: d.cursor_position]
+        status, indent = shell.check_complete(check_text)
+
+        # if all we have after the cursor is whitespace: reformat current text
+        # before cursor
+        after_cursor = d.text[d.cursor_position :]
+        reformatted = False
+        if not after_cursor.strip():
+            reformat_text_before_cursor(b, d, shell)
+            reformatted = True
+        if not (
+            d.on_last_line
+            or d.cursor_position_row >= d.line_count - d.empty_line_count_at_the_end()
+        ):
+            if shell.autoindent:
+                b.insert_text("\n" + indent)
+            else:
+                b.insert_text("\n")
+            return
+
+        if (status != "incomplete") and b.accept_handler:
+            if not reformatted:
+                reformat_text_before_cursor(b, d, shell)
+            b.validate_and_handle()
+        else:
+            if shell.autoindent:
+                b.insert_text("\n" + indent)
+            else:
+                b.insert_text("\n")
+
+    newline_or_execute.__qualname__ = "newline_or_execute"
+
+    return newline_or_execute
+
+
+def previous_history_or_previous_completion(event):
+    """
+    Control-P in vi edit mode on readline is history next, unlike default prompt toolkit.
+
+    If completer is open this still select previous completion.
+    """
+    event.current_buffer.auto_up()
+
+
+def next_history_or_next_completion(event):
+    """
+    Control-N in vi edit mode on readline is history previous, unlike default prompt toolkit.
+
+    If completer is open this still select next completion.
+    """
+    event.current_buffer.auto_down()
+
+
+def dismiss_completion(event):
+    """Dismiss completion"""
+    b = event.current_buffer
+    if b.complete_state:
+        b.cancel_completion()
+
+
+def reset_buffer(event):
+    """Reset buffer"""
+    b = event.current_buffer
+    if b.complete_state:
+        b.cancel_completion()
+    else:
+        b.reset()
+
+
+def reset_search_buffer(event):
+    """Reset search buffer"""
+    if event.current_buffer.document.text:
+        event.current_buffer.reset()
+    else:
+        event.app.layout.focus(DEFAULT_BUFFER)
+
+
+def suspend_to_bg(event):
+    """Suspend to background"""
+    event.app.suspend_to_background()
+
+
+def quit(event):
+    """
+    Quit application with ``SIGQUIT`` if supported or ``sys.exit`` otherwise.
+
+    On platforms that support SIGQUIT, send SIGQUIT to the current process.
+    On other platforms, just exit the process with a message.
+    """
+    sigquit = getattr(signal, "SIGQUIT", None)
+    if sigquit is not None:
+        os.kill(0, signal.SIGQUIT)
+    else:
+        sys.exit("Quit")
+
+
+def indent_buffer(event):
+    """Indent buffer"""
+    event.current_buffer.insert_text(" " * 4)
+
+
+@undoc
+def newline_with_copy_margin(event):
+    """
+    DEPRECATED since IPython 6.0
+
+    See :any:`newline_autoindent_outer` for a replacement.
+
+    Preserve margin and cursor position when using
+    Control-O to insert a newline in EMACS mode
+    """
+    warnings.warn(
+        "`newline_with_copy_margin(event)` is deprecated since IPython 6.0. "
+        "see `newline_autoindent_outer(shell)(event)` for a replacement.",
+        DeprecationWarning,
+        stacklevel=2,
+    )
+
+    b = event.current_buffer
+    cursor_start_pos = b.document.cursor_position_col
+    b.newline(copy_margin=True)
+    b.cursor_up(count=1)
+    cursor_end_pos = b.document.cursor_position_col
+    if cursor_start_pos != cursor_end_pos:
+        pos_diff = cursor_start_pos - cursor_end_pos
+        b.cursor_right(count=pos_diff)
+
+
+def newline_autoindent_outer(inputsplitter) -> Callable[..., None]:
+    """
+    Return a function suitable for inserting a indented newline after the cursor.
+
+    Fancier version of deprecated ``newline_with_copy_margin`` which should
+    compute the correct indentation of the inserted line. That is to say, indent
+    by 4 extra space after a function definition, class definition, context
+    manager... And dedent by 4 space after ``pass``, ``return``, ``raise ...``.
+    """
+
+    def newline_autoindent(event):
+        """Insert a newline after the cursor indented appropriately."""
+        b = event.current_buffer
+        d = b.document
+
+        if b.complete_state:
+            b.cancel_completion()
+        text = d.text[: d.cursor_position] + "\n"
+        _, indent = inputsplitter.check_complete(text)
+        b.insert_text("\n" + (" " * (indent or 0)), move_cursor=False)
+
+    newline_autoindent.__qualname__ = "newline_autoindent"
+
+    return newline_autoindent
+
+
+def open_input_in_editor(event):
+    """Open code from input in external editor"""
+    event.app.current_buffer.open_in_editor()
+
+
+if sys.platform == "win32":
+    from IPython.core.error import TryNext
+    from IPython.lib.clipboard import (
+        ClipboardEmpty,
+        tkinter_clipboard_get,
+        win32_clipboard_get,
+    )
+
+    @undoc
+    def win_paste(event):
+        try:
+            text = win32_clipboard_get()
+        except TryNext:
+            try:
+                text = tkinter_clipboard_get()
+            except (TryNext, ClipboardEmpty):
+                return
+        except ClipboardEmpty:
+            return
+        event.current_buffer.insert_text(text.replace("\t", " " * 4))
+
+else:
+
+    @undoc
+    def win_paste(event):
+        """Stub used when auto-generating shortcuts for documentation"""
+        pass

IPython/terminal/shortcuts/auto_match.py

@@ -0,0 +1,104 b''
+"""
+Utilities function for keybinding with prompt toolkit. 
+
+This will be bound to specific key press and filter modes, 
+like whether we are in edit mode, and whether the completer is open.
+"""
+import re
+from prompt_toolkit.key_binding import KeyPressEvent
+
+
+def parenthesis(event: KeyPressEvent):
+    """Auto-close parenthesis"""
+    event.current_buffer.insert_text("()")
+    event.current_buffer.cursor_left()
+
+
+def brackets(event: KeyPressEvent):
+    """Auto-close brackets"""
+    event.current_buffer.insert_text("[]")
+    event.current_buffer.cursor_left()
+
+
+def braces(event: KeyPressEvent):
+    """Auto-close braces"""
+    event.current_buffer.insert_text("{}")
+    event.current_buffer.cursor_left()
+
+
+def double_quote(event: KeyPressEvent):
+    """Auto-close double quotes"""
+    event.current_buffer.insert_text('""')
+    event.current_buffer.cursor_left()
+
+
+def single_quote(event: KeyPressEvent):
+    """Auto-close single quotes"""
+    event.current_buffer.insert_text("''")
+    event.current_buffer.cursor_left()
+
+
+def docstring_double_quotes(event: KeyPressEvent):
+    """Auto-close docstring (double quotes)"""
+    event.current_buffer.insert_text('""""')
+    event.current_buffer.cursor_left(3)
+
+
+def docstring_single_quotes(event: KeyPressEvent):
+    """Auto-close docstring (single quotes)"""
+    event.current_buffer.insert_text("''''")
+    event.current_buffer.cursor_left(3)
+
+
+def raw_string_parenthesis(event: KeyPressEvent):
+    """Auto-close parenthesis in raw strings"""
+    matches = re.match(
+        r".*(r|R)[\"'](-*)",
+        event.current_buffer.document.current_line_before_cursor,
+    )
+    dashes = matches.group(2) if matches else ""
+    event.current_buffer.insert_text("()" + dashes)
+    event.current_buffer.cursor_left(len(dashes) + 1)
+
+
+def raw_string_bracket(event: KeyPressEvent):
+    """Auto-close bracker in raw strings"""
+    matches = re.match(
+        r".*(r|R)[\"'](-*)",
+        event.current_buffer.document.current_line_before_cursor,
+    )
+    dashes = matches.group(2) if matches else ""
+    event.current_buffer.insert_text("[]" + dashes)
+    event.current_buffer.cursor_left(len(dashes) + 1)
+
+
+def raw_string_braces(event: KeyPressEvent):
+    """Auto-close braces in raw strings"""
+    matches = re.match(
+        r".*(r|R)[\"'](-*)",
+        event.current_buffer.document.current_line_before_cursor,
+    )
+    dashes = matches.group(2) if matches else ""
+    event.current_buffer.insert_text("{}" + dashes)
+    event.current_buffer.cursor_left(len(dashes) + 1)
+
+
+def skip_over(event: KeyPressEvent):
+    """Skip over automatically added parenthesis.
+
+    (rather than adding another parenthesis)"""
+    event.current_buffer.cursor_right()
+
+
+def delete_pair(event: KeyPressEvent):
+    """Delete auto-closed parenthesis"""
+    event.current_buffer.delete()
+    event.current_buffer.delete_before_cursor()
+
+
+auto_match_parens = {"(": parenthesis, "[": brackets, "{": braces}
+auto_match_parens_raw_string = {
+    "(": raw_string_parenthesis,
+    "[": raw_string_bracket,
+    "{": raw_string_braces,
+}

IPython/terminal/shortcuts/auto_suggest.py

@@ -0,0 +1,378 b''
+import re
+import tokenize
+from io import StringIO
+from typing import Callable, List, Optional, Union, Generator, Tuple, Sequence
+
+from prompt_toolkit.buffer import Buffer
+from prompt_toolkit.key_binding import KeyPressEvent
+from prompt_toolkit.key_binding.bindings import named_commands as nc
+from prompt_toolkit.auto_suggest import AutoSuggestFromHistory, Suggestion
+from prompt_toolkit.document import Document
+from prompt_toolkit.history import History
+from prompt_toolkit.shortcuts import PromptSession
+from prompt_toolkit.layout.processors import (
+    Processor,
+    Transformation,
+    TransformationInput,
+)
+
+from IPython.utils.tokenutil import generate_tokens
+
+
+def _get_query(document: Document):
+    return document.lines[document.cursor_position_row]
+
+
+class AppendAutoSuggestionInAnyLine(Processor):
+    """
+    Append the auto suggestion to lines other than the last (appending to the
+    last line is natively supported by the prompt toolkit).
+    """
+
+    def __init__(self, style: str = "class:auto-suggestion") -> None:
+        self.style = style
+
+    def apply_transformation(self, ti: TransformationInput) -> Transformation:
+        is_last_line = ti.lineno == ti.document.line_count - 1
+        is_active_line = ti.lineno == ti.document.cursor_position_row
+
+        if not is_last_line and is_active_line:
+            buffer = ti.buffer_control.buffer
+
+            if buffer.suggestion and ti.document.is_cursor_at_the_end_of_line:
+                suggestion = buffer.suggestion.text
+            else:
+                suggestion = ""
+
+            return Transformation(fragments=ti.fragments + [(self.style, suggestion)])
+        else:
+            return Transformation(fragments=ti.fragments)
+
+
+class NavigableAutoSuggestFromHistory(AutoSuggestFromHistory):
+    """
+    A subclass of AutoSuggestFromHistory that allow navigation to next/previous
+    suggestion from history. To do so it remembers the current position, but it
+    state need to carefully be cleared on the right events.
+    """
+
+    def __init__(
+        self,
+    ):
+        self.skip_lines = 0
+        self._connected_apps = []
+
+    def reset_history_position(self, _: Buffer):
+        self.skip_lines = 0
+
+    def disconnect(self):
+        for pt_app in self._connected_apps:
+            text_insert_event = pt_app.default_buffer.on_text_insert
+            text_insert_event.remove_handler(self.reset_history_position)
+
+    def connect(self, pt_app: PromptSession):
+        self._connected_apps.append(pt_app)
+        # note: `on_text_changed` could be used for a bit different behaviour
+        # on character deletion (i.e. reseting history position on backspace)
+        pt_app.default_buffer.on_text_insert.add_handler(self.reset_history_position)
+        pt_app.default_buffer.on_cursor_position_changed.add_handler(self._dismiss)
+
+    def get_suggestion(
+        self, buffer: Buffer, document: Document
+    ) -> Optional[Suggestion]:
+        text = _get_query(document)
+
+        if text.strip():
+            for suggestion, _ in self._find_next_match(
+                text, self.skip_lines, buffer.history
+            ):
+                return Suggestion(suggestion)
+
+        return None
+
+    def _dismiss(self, buffer, *args, **kwargs):
+        buffer.suggestion = None
+
+    def _find_match(
+        self, text: str, skip_lines: float, history: History, previous: bool
+    ) -> Generator[Tuple[str, float], None, None]:
+        """
+        text : str
+            Text content to find a match for, the user cursor is most of the
+            time at the end of this text.
+        skip_lines : float
+            number of items to skip in the search, this is used to indicate how
+            far in the list the user has navigated by pressing up or down.
+            The float type is used as the base value is +inf
+        history : History
+            prompt_toolkit History instance to fetch previous entries from.
+        previous : bool
+            Direction of the search, whether we are looking previous match
+            (True), or next match (False).
+
+        Yields
+        ------
+        Tuple with:
+        str:
+            current suggestion.
+        float:
+            will actually yield only ints, which is passed back via skip_lines,
+            which may be a +inf (float)
+
+
+        """
+        line_number = -1
+        for string in reversed(list(history.get_strings())):
+            for line in reversed(string.splitlines()):
+                line_number += 1
+                if not previous and line_number < skip_lines:
+                    continue
+                # do not return empty suggestions as these
+                # close the auto-suggestion overlay (and are useless)
+                if line.startswith(text) and len(line) > len(text):
+                    yield line[len(text) :], line_number
+                if previous and line_number >= skip_lines:
+                    return
+
+    def _find_next_match(
+        self, text: str, skip_lines: float, history: History
+    ) -> Generator[Tuple[str, float], None, None]:
+        return self._find_match(text, skip_lines, history, previous=False)
+
+    def _find_previous_match(self, text: str, skip_lines: float, history: History):
+        return reversed(
+            list(self._find_match(text, skip_lines, history, previous=True))
+        )
+
+    def up(self, query: str, other_than: str, history: History) -> None:
+        for suggestion, line_number in self._find_next_match(
+            query, self.skip_lines, history
+        ):
+            # if user has history ['very.a', 'very', 'very.b'] and typed 'very'
+            # we want to switch from 'very.b' to 'very.a' because a) if the
+            # suggestion equals current text, prompt-toolkit aborts suggesting
+            # b) user likely would not be interested in 'very' anyways (they
+            # already typed it).
+            if query + suggestion != other_than:
+                self.skip_lines = line_number
+                break
+        else:
+            # no matches found, cycle back to beginning
+            self.skip_lines = 0
+
+    def down(self, query: str, other_than: str, history: History) -> None:
+        for suggestion, line_number in self._find_previous_match(
+            query, self.skip_lines, history
+        ):
+            if query + suggestion != other_than:
+                self.skip_lines = line_number
+                break
+        else:
+            # no matches found, cycle to end
+            for suggestion, line_number in self._find_previous_match(
+                query, float("Inf"), history
+            ):
+                if query + suggestion != other_than:
+                    self.skip_lines = line_number
+                    break
+
+
+# Needed for to accept autosuggestions in vi insert mode
+def accept_in_vi_insert_mode(event: KeyPressEvent):
+    """Apply autosuggestion if at end of line."""
+    buffer = event.current_buffer
+    d = buffer.document
+    after_cursor = d.text[d.cursor_position :]
+    lines = after_cursor.split("\n")
+    end_of_current_line = lines[0].strip()
+    suggestion = buffer.suggestion
+    if (suggestion is not None) and (suggestion.text) and (end_of_current_line == ""):
+        buffer.insert_text(suggestion.text)
+    else:
+        nc.end_of_line(event)
+
+
+def accept(event: KeyPressEvent):
+    """Accept autosuggestion"""
+    buffer = event.current_buffer
+    suggestion = buffer.suggestion
+    if suggestion:
+        buffer.insert_text(suggestion.text)
+    else:
+        nc.forward_char(event)
+
+
+def discard(event: KeyPressEvent):
+    """Discard autosuggestion"""
+    buffer = event.current_buffer
+    buffer.suggestion = None
+
+
+def accept_word(event: KeyPressEvent):
+    """Fill partial autosuggestion by word"""
+    buffer = event.current_buffer
+    suggestion = buffer.suggestion
+    if suggestion:
+        t = re.split(r"(\S+\s+)", suggestion.text)
+        buffer.insert_text(next((x for x in t if x), ""))
+    else:
+        nc.forward_word(event)
+
+
+def accept_character(event: KeyPressEvent):
+    """Fill partial autosuggestion by character"""
+    b = event.current_buffer
+    suggestion = b.suggestion
+    if suggestion and suggestion.text:
+        b.insert_text(suggestion.text[0])
+
+
+def accept_and_keep_cursor(event: KeyPressEvent):
+    """Accept autosuggestion and keep cursor in place"""
+    buffer = event.current_buffer
+    old_position = buffer.cursor_position
+    suggestion = buffer.suggestion
+    if suggestion:
+        buffer.insert_text(suggestion.text)
+        buffer.cursor_position = old_position
+
+
+def accept_and_move_cursor_left(event: KeyPressEvent):
+    """Accept autosuggestion and move cursor left in place"""
+    accept_and_keep_cursor(event)
+    nc.backward_char(event)
+
+
+def _update_hint(buffer: Buffer):
+    if buffer.auto_suggest:
+        suggestion = buffer.auto_suggest.get_suggestion(buffer, buffer.document)
+        buffer.suggestion = suggestion
+
+
+def backspace_and_resume_hint(event: KeyPressEvent):
+    """Resume autosuggestions after deleting last character"""
+    current_buffer = event.current_buffer
+
+    def resume_hinting(buffer: Buffer):
+        _update_hint(buffer)
+        current_buffer.on_text_changed.remove_handler(resume_hinting)
+
+    current_buffer.on_text_changed.add_handler(resume_hinting)
+    nc.backward_delete_char(event)
+
+
+def up_and_update_hint(event: KeyPressEvent):
+    """Go up and update hint"""
+    current_buffer = event.current_buffer
+
+    current_buffer.auto_up(count=event.arg)
+    _update_hint(current_buffer)
+
+
+def down_and_update_hint(event: KeyPressEvent):
+    """Go down and update hint"""
+    current_buffer = event.current_buffer
+
+    current_buffer.auto_down(count=event.arg)
+    _update_hint(current_buffer)
+
+
+def accept_token(event: KeyPressEvent):
+    """Fill partial autosuggestion by token"""
+    b = event.current_buffer
+    suggestion = b.suggestion
+
+    if suggestion:
+        prefix = _get_query(b.document)
+        text = prefix + suggestion.text
+
+        tokens: List[Optional[str]] = [None, None, None]
+        substrings = [""]
+        i = 0
+
+        for token in generate_tokens(StringIO(text).readline):
+            if token.type == tokenize.NEWLINE:
+                index = len(text)
+            else:
+                index = text.index(token[1], len(substrings[-1]))
+            substrings.append(text[:index])
+            tokenized_so_far = substrings[-1]
+            if tokenized_so_far.startswith(prefix):
+                if i == 0 and len(tokenized_so_far) > len(prefix):
+                    tokens[0] = tokenized_so_far[len(prefix) :]
+                    substrings.append(tokenized_so_far)
+                    i += 1
+                tokens[i] = token[1]
+                if i == 2:
+                    break
+                i += 1
+
+        if tokens[0]:
+            to_insert: str
+            insert_text = substrings[-2]
+            if tokens[1] and len(tokens[1]) == 1:
+                insert_text = substrings[-1]
+            to_insert = insert_text[len(prefix) :]
+            b.insert_text(to_insert)
+            return
+
+    nc.forward_word(event)
+
+
+Provider = Union[AutoSuggestFromHistory, NavigableAutoSuggestFromHistory, None]
+
+
+def _swap_autosuggestion(
+    buffer: Buffer,
+    provider: NavigableAutoSuggestFromHistory,
+    direction_method: Callable,
+):
+    """
+    We skip most recent history entry (in either direction) if it equals the
+    current autosuggestion because if user cycles when auto-suggestion is shown
+    they most likely want something else than what was suggested (otherwise
+    they would have accepted the suggestion).
+    """
+    suggestion = buffer.suggestion
+    if not suggestion:
+        return
+
+    query = _get_query(buffer.document)
+    current = query + suggestion.text
+
+    direction_method(query=query, other_than=current, history=buffer.history)
+
+    new_suggestion = provider.get_suggestion(buffer, buffer.document)
+    buffer.suggestion = new_suggestion
+
+
+def swap_autosuggestion_up(provider: Provider):
+    def swap_autosuggestion_up(event: KeyPressEvent):
+        """Get next autosuggestion from history."""
+        if not isinstance(provider, NavigableAutoSuggestFromHistory):
+            return
+
+        return _swap_autosuggestion(
+            buffer=event.current_buffer, provider=provider, direction_method=provider.up
+        )
+
+    swap_autosuggestion_up.__name__ = "swap_autosuggestion_up"
+    return swap_autosuggestion_up
+
+
+def swap_autosuggestion_down(
+    provider: Union[AutoSuggestFromHistory, NavigableAutoSuggestFromHistory, None]
+):
+    def swap_autosuggestion_down(event: KeyPressEvent):
+        """Get previous autosuggestion from history."""
+        if not isinstance(provider, NavigableAutoSuggestFromHistory):
+            return
+
+        return _swap_autosuggestion(
+            buffer=event.current_buffer,
+            provider=provider,
+            direction_method=provider.down,
+        )
+
+    swap_autosuggestion_down.__name__ = "swap_autosuggestion_down"
+    return swap_autosuggestion_down

.github/workflows/mypy.yml

@@ -1,39 +1,41 b''
 name: Run MyPy
 
 on:
   push:
     branches: [ main, 7.x]
   pull_request:
     branches: [ main, 7.x]
 
 permissions:
   contents: read
 
 jobs:
   build:
 
     runs-on: ubuntu-latest
     strategy:
       matrix:
         python-version: ["3.x"]
 
     steps:
     - uses: actions/checkout@v3
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v4
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
         pip install mypy pyflakes flake8
     - name: Lint with mypy
       run: |
+        set -e
         mypy -p IPython.terminal
         mypy -p IPython.core.magics
         mypy -p IPython.core.guarded_eval
         mypy -p IPython.core.completer
     - name: Lint with pyflakes
       run: |
+        set -e
         flake8  IPython/core/magics/script.py
         flake8  IPython/core/magics/packaging.py

IPython/core/application.py

@@ -1,489 +1,488 b''
 # encoding: utf-8
 """
 An application for IPython.
 
 All top-level applications should use the classes in this module for
 handling configuration and creating configurables.
 
 The job of an :class:`Application` is to create the master configuration
 object and then create the configurable objects, passing the config to them.
 """
 
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 import atexit
 from copy import deepcopy
 import logging
 import os
 import shutil
 import sys
 
 from pathlib import Path
 
 from traitlets.config.application import Application, catch_config_error
 from traitlets.config.loader import ConfigFileNotFound, PyFileConfigLoader
 from IPython.core import release, crashhandler
 from IPython.core.profiledir import ProfileDir, ProfileDirError
 from IPython.paths import get_ipython_dir, get_ipython_package_dir
 from IPython.utils.path import ensure_dir_exists
 from traitlets import (
     List, Unicode, Type, Bool, Set, Instance, Undefined,
     default, observe,
 )
 
 if os.name == "nt":
     programdata = os.environ.get("PROGRAMDATA", None)
     if programdata is not None:
         SYSTEM_CONFIG_DIRS = [str(Path(programdata) / "ipython")]
     else:  # PROGRAMDATA is not defined by default on XP.
         SYSTEM_CONFIG_DIRS = []
 else:
     SYSTEM_CONFIG_DIRS = [
         "/usr/local/etc/ipython",
         "/etc/ipython",
     ]
 
 
 ENV_CONFIG_DIRS = []
 _env_config_dir = os.path.join(sys.prefix, 'etc', 'ipython')
 if _env_config_dir not in SYSTEM_CONFIG_DIRS:
     # only add ENV_CONFIG if sys.prefix is not already included
     ENV_CONFIG_DIRS.append(_env_config_dir)
 
 
 _envvar = os.environ.get('IPYTHON_SUPPRESS_CONFIG_ERRORS')
 if _envvar in {None, ''}:
     IPYTHON_SUPPRESS_CONFIG_ERRORS = None
 else:
     if _envvar.lower() in {'1','true'}:
         IPYTHON_SUPPRESS_CONFIG_ERRORS = True
     elif _envvar.lower() in {'0','false'} :
         IPYTHON_SUPPRESS_CONFIG_ERRORS = False
     else:
         sys.exit("Unsupported value for environment variable: 'IPYTHON_SUPPRESS_CONFIG_ERRORS' is set to '%s' which is none of  {'0', '1', 'false', 'true', ''}."% _envvar )
 
 # aliases and flags
 
 base_aliases = {}
 if isinstance(Application.aliases, dict):
     # traitlets 5
     base_aliases.update(Application.aliases)
 base_aliases.update(
     {
         "profile-dir": "ProfileDir.location",
         "profile": "BaseIPythonApplication.profile",
         "ipython-dir": "BaseIPythonApplication.ipython_dir",
         "log-level": "Application.log_level",
         "config": "BaseIPythonApplication.extra_config_file",
     }
 )
 
 base_flags = dict()
 if isinstance(Application.flags, dict):
     # traitlets 5
     base_flags.update(Application.flags)
 base_flags.update(
     dict(
         debug=(
             {"Application": {"log_level": logging.DEBUG}},
             "set log level to logging.DEBUG (maximize logging output)",
         ),
         quiet=(
             {"Application": {"log_level": logging.CRITICAL}},
             "set log level to logging.CRITICAL (minimize logging output)",
         ),
         init=(
             {
                 "BaseIPythonApplication": {
                     "copy_config_files": True,
                     "auto_create": True,
                 }
             },
             """Initialize profile with default config files.  This is equivalent
             to running `ipython profile create <profile>` prior to startup.
             """,
         ),
     )
 )
 
 
 class ProfileAwareConfigLoader(PyFileConfigLoader):
     """A Python file config loader that is aware of IPython profiles."""
     def load_subconfig(self, fname, path=None, profile=None):
         if profile is not None:
             try:
                 profile_dir = ProfileDir.find_profile_dir_by_name(
                         get_ipython_dir(),
                         profile,
                 )
             except ProfileDirError:
                 return
             path = profile_dir.location
         return super(ProfileAwareConfigLoader, self).load_subconfig(fname, path=path)
 
 class BaseIPythonApplication(Application):
-
-    name = u'ipython'
-    description = Unicode(u'IPython: an enhanced interactive Python shell.')
+    name = "ipython"
+    description = "IPython: an enhanced interactive Python shell."
     version = Unicode(release.version)
 
     aliases = base_aliases
     flags = base_flags
     classes = List([ProfileDir])
     
     # enable `load_subconfig('cfg.py', profile='name')`
     python_config_loader_class = ProfileAwareConfigLoader
 
     # Track whether the config_file has changed,
     # because some logic happens only if we aren't using the default.
     config_file_specified = Set()
 
     config_file_name = Unicode()
     @default('config_file_name')
     def _config_file_name_default(self):
         return self.name.replace('-','_') + u'_config.py'
     @observe('config_file_name')
     def _config_file_name_changed(self, change):
         if change['new'] != change['old']:
             self.config_file_specified.add(change['new'])
 
     # The directory that contains IPython's builtin profiles.
     builtin_profile_dir = Unicode(
         os.path.join(get_ipython_package_dir(), u'config', u'profile', u'default')
     )
     
     config_file_paths = List(Unicode())
     @default('config_file_paths')
     def _config_file_paths_default(self):
         return []
 
     extra_config_file = Unicode(
     help="""Path to an extra config file to load.
     
     If specified, load this config file in addition to any other IPython config.
     """).tag(config=True)
     @observe('extra_config_file')
     def _extra_config_file_changed(self, change):
         old = change['old']
         new = change['new']
         try:
             self.config_files.remove(old)
         except ValueError:
             pass
         self.config_file_specified.add(new)
         self.config_files.append(new)
 
     profile = Unicode(u'default',
         help="""The IPython profile to use."""
     ).tag(config=True)
     
     @observe('profile')
     def _profile_changed(self, change):
         self.builtin_profile_dir = os.path.join(
                 get_ipython_package_dir(), u'config', u'profile', change['new']
         )
 
     add_ipython_dir_to_sys_path = Bool(
         False,
         """Should the IPython profile directory be added to sys path ?
 
         This option was non-existing before IPython 8.0, and ipython_dir was added to
         sys path to allow import of extensions present there. This was historical
         baggage from when pip did not exist. This now default to false,
         but can be set to true for legacy reasons.
         """,
     ).tag(config=True)
 
     ipython_dir = Unicode(
         help="""
         The name of the IPython directory. This directory is used for logging
         configuration (through profiles), history storage, etc. The default
         is usually $HOME/.ipython. This option can also be specified through
         the environment variable IPYTHONDIR.
         """
     ).tag(config=True)
     @default('ipython_dir')
     def _ipython_dir_default(self):
         d = get_ipython_dir()
         self._ipython_dir_changed({
             'name': 'ipython_dir',
             'old': d,
             'new': d,
         })
         return d
     
     _in_init_profile_dir = False
     profile_dir = Instance(ProfileDir, allow_none=True)
     @default('profile_dir')
     def _profile_dir_default(self):
         # avoid recursion
         if self._in_init_profile_dir:
             return
         # profile_dir requested early, force initialization
         self.init_profile_dir()
         return self.profile_dir
 
     overwrite = Bool(False,
         help="""Whether to overwrite existing config files when copying"""
     ).tag(config=True)
     auto_create = Bool(False,
         help="""Whether to create profile dir if it doesn't exist"""
     ).tag(config=True)
 
     config_files = List(Unicode())
     @default('config_files')
     def _config_files_default(self):
         return [self.config_file_name]
 
     copy_config_files = Bool(False,
         help="""Whether to install the default config files into the profile dir.
         If a new profile is being created, and IPython contains config files for that
         profile, then they will be staged into the new directory.  Otherwise,
         default config files will be automatically generated.
         """).tag(config=True)
     
     verbose_crash = Bool(False,
         help="""Create a massive crash report when IPython encounters what may be an
         internal error.  The default is to append a short message to the
         usual traceback""").tag(config=True)
 
     # The class to use as the crash handler.
     crash_handler_class = Type(crashhandler.CrashHandler)
 
     @catch_config_error
     def __init__(self, **kwargs):
         super(BaseIPythonApplication, self).__init__(**kwargs)
         # ensure current working directory exists
         try:
             os.getcwd()
         except:
             # exit if cwd doesn't exist
             self.log.error("Current working directory doesn't exist.")
             self.exit(1)
 
     #-------------------------------------------------------------------------
     # Various stages of Application creation
     #-------------------------------------------------------------------------
     
     def init_crash_handler(self):
         """Create a crash handler, typically setting sys.excepthook to it."""
         self.crash_handler = self.crash_handler_class(self)
         sys.excepthook = self.excepthook
         def unset_crashhandler():
             sys.excepthook = sys.__excepthook__
         atexit.register(unset_crashhandler)
     
     def excepthook(self, etype, evalue, tb):
         """this is sys.excepthook after init_crashhandler
 
         set self.verbose_crash=True to use our full crashhandler, instead of
         a regular traceback with a short message (crash_handler_lite)
         """
         
         if self.verbose_crash:
             return self.crash_handler(etype, evalue, tb)
         else:
             return crashhandler.crash_handler_lite(etype, evalue, tb)
 
     @observe('ipython_dir')
     def _ipython_dir_changed(self, change):
         old = change['old']
         new = change['new']
         if old is not Undefined:
             str_old = os.path.abspath(old)
             if str_old in sys.path:
                 sys.path.remove(str_old)
         if self.add_ipython_dir_to_sys_path:
             str_path = os.path.abspath(new)
             sys.path.append(str_path)
             ensure_dir_exists(new)
             readme = os.path.join(new, "README")
             readme_src = os.path.join(
                 get_ipython_package_dir(), "config", "profile", "README"
             )
             if not os.path.exists(readme) and os.path.exists(readme_src):
                 shutil.copy(readme_src, readme)
             for d in ("extensions", "nbextensions"):
                 path = os.path.join(new, d)
                 try:
                     ensure_dir_exists(path)
                 except OSError as e:
                     # this will not be EEXIST
                     self.log.error("couldn't create path %s: %s", path, e)
-            self.log.debug("IPYTHONDIR set to: %s" % new)
+            self.log.debug("IPYTHONDIR set to: %s", new)
 
     def load_config_file(self, suppress_errors=IPYTHON_SUPPRESS_CONFIG_ERRORS):
         """Load the config file.
 
         By default, errors in loading config are handled, and a warning
         printed on screen. For testing, the suppress_errors option is set
         to False, so errors will make tests fail.
 
         `suppress_errors` default value is to be `None` in which case the
         behavior default to the one of `traitlets.Application`.
 
         The default value can be set :
            - to `False` by setting 'IPYTHON_SUPPRESS_CONFIG_ERRORS' environment variable to '0', or 'false' (case insensitive).
            - to `True` by setting 'IPYTHON_SUPPRESS_CONFIG_ERRORS' environment variable to '1' or 'true' (case insensitive).
            - to `None` by setting 'IPYTHON_SUPPRESS_CONFIG_ERRORS' environment variable to '' (empty string) or leaving it unset.
 
         Any other value are invalid, and will make IPython exit with a non-zero return code.
         """
 
 
         self.log.debug("Searching path %s for config files", self.config_file_paths)
         base_config = 'ipython_config.py'
         self.log.debug("Attempting to load config file: %s" %
                        base_config)
         try:
             if suppress_errors is not None:
                 old_value = Application.raise_config_file_errors
                 Application.raise_config_file_errors = not suppress_errors;
             Application.load_config_file(
                 self,
                 base_config,
                 path=self.config_file_paths
             )
         except ConfigFileNotFound:
             # ignore errors loading parent
             self.log.debug("Config file %s not found", base_config)
             pass
         if suppress_errors is not None:
             Application.raise_config_file_errors = old_value
         
         for config_file_name in self.config_files:
             if not config_file_name or config_file_name == base_config:
                 continue
             self.log.debug("Attempting to load config file: %s" %
                            self.config_file_name)
             try:
                 Application.load_config_file(
                     self,
                     config_file_name,
                     path=self.config_file_paths
                 )
             except ConfigFileNotFound:
                 # Only warn if the default config file was NOT being used.
                 if config_file_name in self.config_file_specified:
                     msg = self.log.warning
                 else:
                     msg = self.log.debug
                 msg("Config file not found, skipping: %s", config_file_name)
             except Exception:
                 # For testing purposes.
                 if not suppress_errors:
                     raise
                 self.log.warning("Error loading config file: %s" %
                               self.config_file_name, exc_info=True)
 
     def init_profile_dir(self):
         """initialize the profile dir"""
         self._in_init_profile_dir = True
         if self.profile_dir is not None:
             # already ran
             return
         if 'ProfileDir.location' not in self.config:
             # location not specified, find by profile name
             try:
                 p = ProfileDir.find_profile_dir_by_name(self.ipython_dir, self.profile, self.config)
             except ProfileDirError:
                 # not found, maybe create it (always create default profile)
                 if self.auto_create or self.profile == 'default':
                     try:
                         p = ProfileDir.create_profile_dir_by_name(self.ipython_dir, self.profile, self.config)
                     except ProfileDirError:
                         self.log.fatal("Could not create profile: %r"%self.profile)
                         self.exit(1)
                     else:
                         self.log.info("Created profile dir: %r"%p.location)
                 else:
                     self.log.fatal("Profile %r not found."%self.profile)
                     self.exit(1)
             else:
-                self.log.debug(f"Using existing profile dir: {p.location!r}")
+                self.log.debug("Using existing profile dir: %r", p.location)
         else:
             location = self.config.ProfileDir.location
             # location is fully specified
             try:
                 p = ProfileDir.find_profile_dir(location, self.config)
             except ProfileDirError:
                 # not found, maybe create it
                 if self.auto_create:
                     try:
                         p = ProfileDir.create_profile_dir(location, self.config)
                     except ProfileDirError:
                         self.log.fatal("Could not create profile directory: %r"%location)
                         self.exit(1)
                     else:
                         self.log.debug("Creating new profile dir: %r"%location)
                 else:
                     self.log.fatal("Profile directory %r not found."%location)
                     self.exit(1)
             else:
-                self.log.debug(f"Using existing profile dir: {p.location!r}")
+                self.log.debug("Using existing profile dir: %r", p.location)
             # if profile_dir is specified explicitly, set profile name
             dir_name = os.path.basename(p.location)
             if dir_name.startswith('profile_'):
                 self.profile = dir_name[8:]
 
         self.profile_dir = p
         self.config_file_paths.append(p.location)
         self._in_init_profile_dir = False
 
     def init_config_files(self):
         """[optionally] copy default config files into profile dir."""
         self.config_file_paths.extend(ENV_CONFIG_DIRS)
         self.config_file_paths.extend(SYSTEM_CONFIG_DIRS)
         # copy config files
         path = Path(self.builtin_profile_dir)
         if self.copy_config_files:
             src = self.profile
 
             cfg = self.config_file_name
             if path and (path / cfg).exists():
                 self.log.warning(
                     "Staging %r from %s into %r [overwrite=%s]"
                     % (cfg, src, self.profile_dir.location, self.overwrite)
                 )
                 self.profile_dir.copy_config_file(cfg, path=path, overwrite=self.overwrite)
             else:
                 self.stage_default_config_file()
         else:
             # Still stage *bundled* config files, but not generated ones
             # This is necessary for `ipython profile=sympy` to load the profile
             # on the first go
             files = path.glob("*.py")
             for fullpath in files:
                 cfg = fullpath.name
                 if self.profile_dir.copy_config_file(cfg, path=path, overwrite=False):
                     # file was copied
                     self.log.warning("Staging bundled %s from %s into %r"%(
                             cfg, self.profile, self.profile_dir.location)
                     )
 
 
     def stage_default_config_file(self):
         """auto generate default config file, and stage it into the profile."""
         s = self.generate_config_file()
         config_file = Path(self.profile_dir.location) / self.config_file_name
         if self.overwrite or not config_file.exists():
-            self.log.warning("Generating default config file: %r" % (config_file))
+            self.log.warning("Generating default config file: %r", (config_file))
             config_file.write_text(s, encoding="utf-8")
 
     @catch_config_error
     def initialize(self, argv=None):
         # don't hook up crash handler before parsing command-line
         self.parse_command_line(argv)
         self.init_crash_handler()
         if self.subapp is not None:
             # stop here if subapp is taking over
             return
         # save a copy of CLI config to re-load after config files
         # so that it has highest priority
         cl_config = deepcopy(self.config)
         self.init_profile_dir()
         self.init_config_files()
         self.load_config_file()
         # enforce cl-opts override configfile opts:
         self.update_config(cl_config)

IPython/core/displayhook.py

@@ -1,325 +1,331 b''
 # -*- coding: utf-8 -*-
 """Displayhook for IPython.
 
 This defines a callable class that IPython uses for `sys.displayhook`.
 """
 
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 import builtins as builtin_mod
 import sys
 import io as _io
 import tokenize
 
 from traitlets.config.configurable import Configurable
 from traitlets import Instance, Float
 from warnings import warn
 
 # TODO: Move the various attributes (cache_size, [others now moved]). Some
 # of these are also attributes of InteractiveShell. They should be on ONE object
 # only and the other objects should ask that one object for their values.
 
 class DisplayHook(Configurable):
     """The custom IPython displayhook to replace sys.displayhook.
 
     This class does many things, but the basic idea is that it is a callable
     that gets called anytime user code returns a value.
     """
 
     shell = Instance('IPython.core.interactiveshell.InteractiveShellABC',
                      allow_none=True)
     exec_result = Instance('IPython.core.interactiveshell.ExecutionResult',
                            allow_none=True)
     cull_fraction = Float(0.2)
 
     def __init__(self, shell=None, cache_size=1000, **kwargs):
         super(DisplayHook, self).__init__(shell=shell, **kwargs)
         cache_size_min = 3
         if cache_size <= 0:
             self.do_full_cache = 0
             cache_size = 0
         elif cache_size < cache_size_min:
             self.do_full_cache = 0
             cache_size = 0
             warn('caching was disabled (min value for cache size is %s).' %
                  cache_size_min,stacklevel=3)
         else:
             self.do_full_cache = 1
 
         self.cache_size = cache_size
 
         # we need a reference to the user-level namespace
         self.shell = shell
         
         self._,self.__,self.___ = '','',''
 
         # these are deliberately global:
         to_user_ns = {'_':self._,'__':self.__,'___':self.___}
         self.shell.user_ns.update(to_user_ns)
 
     @property
     def prompt_count(self):
         return self.shell.execution_count
 
     #-------------------------------------------------------------------------
     # Methods used in __call__. Override these methods to modify the behavior
     # of the displayhook.
     #-------------------------------------------------------------------------
 
     def check_for_underscore(self):
         """Check if the user has set the '_' variable by hand."""
         # If something injected a '_' variable in __builtin__, delete
         # ipython's automatic one so we don't clobber that.  gettext() in
         # particular uses _, so we need to stay away from it.
         if '_' in builtin_mod.__dict__:
             try:
                 user_value = self.shell.user_ns['_']
                 if user_value is not self._:
                     return
                 del self.shell.user_ns['_']
             except KeyError:
                 pass
 
     def quiet(self):
         """Should we silence the display hook because of ';'?"""
         # do not print output if input ends in ';'
         
         try:
             cell = self.shell.history_manager.input_hist_parsed[-1]
         except IndexError:
             # some uses of ipshellembed may fail here
             return False
         
-        sio = _io.StringIO(cell)
+        return self.semicolon_at_end_of_expression(cell)
+
+    @staticmethod
+    def semicolon_at_end_of_expression(expression):
+        """Parse Python expression and detects whether last token is ';'"""
+
+        sio = _io.StringIO(expression)
         tokens = list(tokenize.generate_tokens(sio.readline))
 
         for token in reversed(tokens):
             if token[0] in (tokenize.ENDMARKER, tokenize.NL, tokenize.NEWLINE, tokenize.COMMENT):
                 continue
             if (token[0] == tokenize.OP) and (token[1] == ';'):
                 return True
             else:
                 return False
 
     def start_displayhook(self):
         """Start the displayhook, initializing resources."""
         pass
 
     def write_output_prompt(self):
         """Write the output prompt.
 
         The default implementation simply writes the prompt to
         ``sys.stdout``.
         """
         # Use write, not print which adds an extra space.
         sys.stdout.write(self.shell.separate_out)
         outprompt = 'Out[{}]: '.format(self.shell.execution_count)
         if self.do_full_cache:
             sys.stdout.write(outprompt)
 
     def compute_format_data(self, result):
         """Compute format data of the object to be displayed.
 
         The format data is a generalization of the :func:`repr` of an object.
         In the default implementation the format data is a :class:`dict` of
         key value pair where the keys are valid MIME types and the values
         are JSON'able data structure containing the raw data for that MIME
         type. It is up to frontends to determine pick a MIME to to use and
         display that data in an appropriate manner.
 
         This method only computes the format data for the object and should
         NOT actually print or write that to a stream.
 
         Parameters
         ----------
         result : object
             The Python object passed to the display hook, whose format will be
             computed.
 
         Returns
         -------
         (format_dict, md_dict) : dict
             format_dict is a :class:`dict` whose keys are valid MIME types and values are
             JSON'able raw data for that MIME type. It is recommended that
             all return values of this should always include the "text/plain"
             MIME type representation of the object.
             md_dict is a :class:`dict` with the same MIME type keys
             of metadata associated with each output.
 
         """
         return self.shell.display_formatter.format(result)
 
     # This can be set to True by the write_output_prompt method in a subclass
     prompt_end_newline = False
 
     def write_format_data(self, format_dict, md_dict=None) -> None:
         """Write the format data dict to the frontend.
 
         This default version of this method simply writes the plain text
         representation of the object to ``sys.stdout``. Subclasses should
         override this method to send the entire `format_dict` to the
         frontends.
 
         Parameters
         ----------
         format_dict : dict
             The format dict for the object passed to `sys.displayhook`.
         md_dict : dict (optional)
             The metadata dict to be associated with the display data.
         """
         if 'text/plain' not in format_dict:
             # nothing to do
             return
         # We want to print because we want to always make sure we have a
         # newline, even if all the prompt separators are ''. This is the
         # standard IPython behavior.
         result_repr = format_dict['text/plain']
         if '\n' in result_repr:
             # So that multi-line strings line up with the left column of
             # the screen, instead of having the output prompt mess up
             # their first line.
             # We use the prompt template instead of the expanded prompt
             # because the expansion may add ANSI escapes that will interfere
             # with our ability to determine whether or not we should add
             # a newline.
             if not self.prompt_end_newline:
                 # But avoid extraneous empty lines.
                 result_repr = '\n' + result_repr
 
         try:
             print(result_repr)
         except UnicodeEncodeError:
             # If a character is not supported by the terminal encoding replace
             # it with its \u or \x representation
             print(result_repr.encode(sys.stdout.encoding,'backslashreplace').decode(sys.stdout.encoding))
 
     def update_user_ns(self, result):
         """Update user_ns with various things like _, __, _1, etc."""
 
         # Avoid recursive reference when displaying _oh/Out
         if self.cache_size and result is not self.shell.user_ns['_oh']:
             if len(self.shell.user_ns['_oh']) >= self.cache_size and self.do_full_cache:
                 self.cull_cache()
 
             # Don't overwrite '_' and friends if '_' is in __builtin__
             # (otherwise we cause buggy behavior for things like gettext). and
             # do not overwrite _, __ or ___ if one of these has been assigned
             # by the user.
             update_unders = True
             for unders in ['_'*i for i in range(1,4)]:
                 if not unders in self.shell.user_ns:
                     continue
                 if getattr(self, unders) is not self.shell.user_ns.get(unders):
                     update_unders = False
 
             self.___ = self.__
             self.__ = self._
             self._ = result
 
             if ('_' not in builtin_mod.__dict__) and (update_unders):
                 self.shell.push({'_':self._,
                                  '__':self.__,
                                 '___':self.___}, interactive=False)
 
             # hackish access to top-level  namespace to create _1,_2... dynamically
             to_main = {}
             if self.do_full_cache:
                 new_result = '_%s' % self.prompt_count
                 to_main[new_result] = result
                 self.shell.push(to_main, interactive=False)
                 self.shell.user_ns['_oh'][self.prompt_count] = result
 
     def fill_exec_result(self, result):
         if self.exec_result is not None:
             self.exec_result.result = result
 
     def log_output(self, format_dict):
         """Log the output."""
         if 'text/plain' not in format_dict:
             # nothing to do
             return
         if self.shell.logger.log_output:
             self.shell.logger.log_write(format_dict['text/plain'], 'output')
         self.shell.history_manager.output_hist_reprs[self.prompt_count] = \
                                                     format_dict['text/plain']
 
     def finish_displayhook(self):
         """Finish up all displayhook activities."""
         sys.stdout.write(self.shell.separate_out2)
         sys.stdout.flush()
 
     def __call__(self, result=None):
         """Printing with history cache management.
 
         This is invoked every time the interpreter needs to print, and is
         activated by setting the variable sys.displayhook to it.
         """
         self.check_for_underscore()
         if result is not None and not self.quiet():
             self.start_displayhook()
             self.write_output_prompt()
             format_dict, md_dict = self.compute_format_data(result)
             self.update_user_ns(result)
             self.fill_exec_result(result)
             if format_dict:
                 self.write_format_data(format_dict, md_dict)
                 self.log_output(format_dict)
             self.finish_displayhook()
 
     def cull_cache(self):
         """Output cache is full, cull the oldest entries"""
         oh = self.shell.user_ns.get('_oh', {})
         sz = len(oh)
         cull_count = max(int(sz * self.cull_fraction), 2)
         warn('Output cache limit (currently {sz} entries) hit.\n'
              'Flushing oldest {cull_count} entries.'.format(sz=sz, cull_count=cull_count))
         
         for i, n in enumerate(sorted(oh)):
             if i >= cull_count:
                 break
             self.shell.user_ns.pop('_%i' % n, None)
             oh.pop(n, None)
         
 
     def flush(self):
         if not self.do_full_cache:
             raise ValueError("You shouldn't have reached the cache flush "
                              "if full caching is not enabled!")
         # delete auto-generated vars from global namespace
 
         for n in range(1,self.prompt_count + 1):
             key = '_'+repr(n)
             try:
                 del self.shell.user_ns[key]
             except: pass
         # In some embedded circumstances, the user_ns doesn't have the
         # '_oh' key set up.
         oh = self.shell.user_ns.get('_oh', None)
         if oh is not None:
             oh.clear()
 
         # Release our own references to objects:
         self._, self.__, self.___ = '', '', ''
 
         if '_' not in builtin_mod.__dict__:
             self.shell.user_ns.update({'_':self._,'__':self.__,'___':self.___})
         import gc
         # TODO: Is this really needed?
         # IronPython blocks here forever
         if sys.platform != "cli":
             gc.collect()
 
 
 class CapturingDisplayHook(object):
     def __init__(self, shell, outputs=None):
         self.shell = shell
         if outputs is None:
             outputs = []
         self.outputs = outputs
 
     def __call__(self, result=None):
         if result is None:
             return
         format_dict, md_dict = self.shell.display_formatter.format(result)
         self.outputs.append({ 'data': format_dict, 'metadata': md_dict })

IPython/core/logger.py

@@ -1,218 +1,227 b''
 """Logger class for IPython's logging facilities.
 """
 
 #*****************************************************************************
 #       Copyright (C) 2001 Janko Hauser <[e-mail]> and
 #       Copyright (C) 2001-2006 Fernando Perez <[e-mail]>
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #*****************************************************************************
 
 #****************************************************************************
 # Modules and globals
 
 # Python standard modules
 import glob
 import io
 import os
 import time
 
 
 #****************************************************************************
 # FIXME: This class isn't a mixin anymore, but it still needs attributes from
 # ipython and does input cache management.  Finish cleanup later...
 
 class Logger(object):
     """A Logfile class with different policies for file creation"""
 
     def __init__(self, home_dir, logfname='Logger.log', loghead=u'',
                  logmode='over'):
 
         # this is the full ipython instance, we need some attributes from it
         # which won't exist until later. What a mess, clean up later...
         self.home_dir = home_dir
 
         self.logfname = logfname
         self.loghead = loghead
         self.logmode = logmode
         self.logfile = None
 
         # Whether to log raw or processed input
         self.log_raw_input = False
 
         # whether to also log output
         self.log_output = False
 
         # whether to put timestamps before each log entry
         self.timestamp = False
 
         # activity control flags
         self.log_active = False
 
     # logmode is a validated property
     def _set_mode(self,mode):
         if mode not in ['append','backup','global','over','rotate']:
             raise ValueError('invalid log mode %s given' % mode)
         self._logmode = mode
 
     def _get_mode(self):
         return self._logmode
 
     logmode = property(_get_mode,_set_mode)
 
     def logstart(self, logfname=None, loghead=None, logmode=None,
                  log_output=False, timestamp=False, log_raw_input=False):
         """Generate a new log-file with a default header.
 
         Raises RuntimeError if the log has already been started"""
 
         if self.logfile is not None:
             raise RuntimeError('Log file is already active: %s' %
                                self.logfname)
 
         # The parameters can override constructor defaults
         if logfname is not None: self.logfname = logfname
         if loghead is not None: self.loghead = loghead
         if logmode is not None: self.logmode = logmode
 
         # Parameters not part of the constructor
         self.timestamp = timestamp
         self.log_output = log_output
         self.log_raw_input = log_raw_input
 
         # init depending on the log mode requested
         isfile = os.path.isfile
         logmode = self.logmode
 
         if logmode == 'append':
             self.logfile = io.open(self.logfname, 'a', encoding='utf-8')
 
         elif logmode == 'backup':
             if isfile(self.logfname):
                 backup_logname = self.logfname+'~'
                 # Manually remove any old backup, since os.rename may fail
                 # under Windows.
                 if isfile(backup_logname):
                     os.remove(backup_logname)
                 os.rename(self.logfname,backup_logname)
             self.logfile = io.open(self.logfname, 'w', encoding='utf-8')
 
         elif logmode == 'global':
             self.logfname = os.path.join(self.home_dir,self.logfname)
             self.logfile = io.open(self.logfname, 'a', encoding='utf-8')
 
         elif logmode == 'over':
             if isfile(self.logfname):
                 os.remove(self.logfname)
             self.logfile = io.open(self.logfname,'w', encoding='utf-8')
 
         elif logmode == 'rotate':
             if isfile(self.logfname):
                 if isfile(self.logfname+'.001~'):
                     old = glob.glob(self.logfname+'.*~')
                     old.sort()
                     old.reverse()
                     for f in old:
                         root, ext = os.path.splitext(f)
                         num = int(ext[1:-1])+1
                         os.rename(f, root+'.'+repr(num).zfill(3)+'~')
                 os.rename(self.logfname, self.logfname+'.001~')
             self.logfile = io.open(self.logfname, 'w', encoding='utf-8')
 
         if logmode != 'append':
             self.logfile.write(self.loghead)
 
         self.logfile.flush()
         self.log_active = True
 
     def switch_log(self,val):
         """Switch logging on/off. val should be ONLY a boolean."""
 
         if val not in [False,True,0,1]:
             raise ValueError('Call switch_log ONLY with a boolean argument, '
                              'not with: %s' % val)
 
         label = {0:'OFF',1:'ON',False:'OFF',True:'ON'}
 
         if self.logfile is None:
             print("""
 Logging hasn't been started yet (use logstart for that).
 
 %logon/%logoff are for temporarily starting and stopping logging for a logfile
 which already exists. But you must first start the logging process with
 %logstart (optionally giving a logfile name).""")
 
         else:
             if self.log_active == val:
                 print('Logging is already',label[val])
             else:
                 print('Switching logging',label[val])
                 self.log_active = not self.log_active
                 self.log_active_out = self.log_active
 
     def logstate(self):
         """Print a status message about the logger."""
         if self.logfile is None:
             print('Logging has not been activated.')
         else:
             state = self.log_active and 'active' or 'temporarily suspended'
             print('Filename       :', self.logfname)
             print('Mode           :', self.logmode)
             print('Output logging :', self.log_output)
             print('Raw input log  :', self.log_raw_input)
             print('Timestamping   :', self.timestamp)
             print('State          :', state)
 
     def log(self, line_mod, line_ori):
         """Write the sources to a log.
 
         Inputs:
 
         - line_mod: possibly modified input, such as the transformations made
           by input prefilters or input handlers of various kinds. This should
           always be valid Python.
 
         - line_ori: unmodified input line from the user. This is not
           necessarily valid Python.
         """
 
         # Write the log line, but decide which one according to the
         # log_raw_input flag, set when the log is started.
         if self.log_raw_input:
             self.log_write(line_ori)
         else:
             self.log_write(line_mod)
 
     def log_write(self, data, kind='input'):
         """Write data to the log file, if active"""
 
         #print 'data: %r' % data # dbg
         if self.log_active and data:
             write = self.logfile.write
             if kind=='input':
                 if self.timestamp:
                     write(time.strftime('# %a, %d %b %Y %H:%M:%S\n', time.localtime()))
                 write(data)
             elif kind=='output' and self.log_output:
                 odata = u'\n'.join([u'#[Out]# %s' % s
                                    for s in data.splitlines()])
                 write(u'%s\n' % odata)
-            self.logfile.flush()
+            try:
+                self.logfile.flush()
+            except OSError:
+                print("Failed to flush the log file.")
+                print(
+                    f"Please check that {self.logfname} exists and have the right permissions."
+                )
+                print(
+                    "Also consider turning off the log with `%logstop` to avoid this warning."
+                )
 
     def logstop(self):
         """Fully stop logging and close log file.
 
         In order to start logging again, a new logstart() call needs to be
         made, possibly (though not necessarily) with a new filename, mode and
         other options."""
 
         if self.logfile is not None:
             self.logfile.close()
             self.logfile = None
         else:
             print("Logging hadn't been started.")
         self.log_active = False
 
     # For backwards compatibility, in case anyone was using this.
     close_log = logstop

IPython/core/magic.py

@@ -1,746 +1,757 b''
 # encoding: utf-8
 """Magic functions for InteractiveShell.
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2001 Janko Hauser <[e-mail]> and
 #  Copyright (C) 2001 Fernando Perez <[e-mail]>
 #  Copyright (C) 2008 The IPython Development Team
 
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 import os
 import re
 import sys
 from getopt import getopt, GetoptError
 
 from traitlets.config.configurable import Configurable
 from . import oinspect
 from .error import UsageError
 from .inputtransformer2 import ESC_MAGIC, ESC_MAGIC2
 from ..utils.ipstruct import Struct
 from ..utils.process import arg_split
 from ..utils.text import dedent
 from traitlets import Bool, Dict, Instance, observe
 from logging import error
 
 #-----------------------------------------------------------------------------
 # Globals
 #-----------------------------------------------------------------------------
 
 # A dict we'll use for each class that has magics, used as temporary storage to
 # pass information between the @line/cell_magic method decorators and the
 # @magics_class class decorator, because the method decorators have no
 # access to the class when they run.  See for more details:
 # http://stackoverflow.com/questions/2366713/can-a-python-decorator-of-an-instance-method-access-the-class
 
 magics = dict(line={}, cell={})
 
 magic_kinds = ('line', 'cell')
 magic_spec = ('line', 'cell', 'line_cell')
 magic_escapes = dict(line=ESC_MAGIC, cell=ESC_MAGIC2)
 
 #-----------------------------------------------------------------------------
 # Utility classes and functions
 #-----------------------------------------------------------------------------
 
 class Bunch: pass
 
 
 def on_off(tag):
     """Return an ON/OFF string for a 1/0 input. Simple utility function."""
     return ['OFF','ON'][tag]
 
 
 def compress_dhist(dh):
     """Compress a directory history into a new one with at most 20 entries.
 
     Return a new list made from the first and last 10 elements of dhist after
     removal of duplicates.
     """
     head, tail = dh[:-10], dh[-10:]
 
     newhead = []
     done = set()
     for h in head:
         if h in done:
             continue
         newhead.append(h)
         done.add(h)
 
     return newhead + tail
 
 
 def needs_local_scope(func):
     """Decorator to mark magic functions which need to local scope to run."""
     func.needs_local_scope = True
     return func
 
 #-----------------------------------------------------------------------------
 # Class and method decorators for registering magics
 #-----------------------------------------------------------------------------
 
 def magics_class(cls):
     """Class decorator for all subclasses of the main Magics class.
 
     Any class that subclasses Magics *must* also apply this decorator, to
     ensure that all the methods that have been decorated as line/cell magics
     get correctly registered in the class instance.  This is necessary because
     when method decorators run, the class does not exist yet, so they
     temporarily store their information into a module global.  Application of
     this class decorator copies that global data to the class instance and
     clears the global.
 
     Obviously, this mechanism is not thread-safe, which means that the
     *creation* of subclasses of Magic should only be done in a single-thread
     context.  Instantiation of the classes has no restrictions.  Given that
     these classes are typically created at IPython startup time and before user
     application code becomes active, in practice this should not pose any
     problems.
     """
     cls.registered = True
     cls.magics = dict(line = magics['line'],
                       cell = magics['cell'])
     magics['line'] = {}
     magics['cell'] = {}
     return cls
 
 
 def record_magic(dct, magic_kind, magic_name, func):
     """Utility function to store a function as a magic of a specific kind.
 
     Parameters
     ----------
     dct : dict
         A dictionary with 'line' and 'cell' subdicts.
     magic_kind : str
         Kind of magic to be stored.
     magic_name : str
         Key to store the magic as.
     func : function
         Callable object to store.
     """
     if magic_kind == 'line_cell':
         dct['line'][magic_name] = dct['cell'][magic_name] = func
     else:
         dct[magic_kind][magic_name] = func
 
 
 def validate_type(magic_kind):
     """Ensure that the given magic_kind is valid.
 
     Check that the given magic_kind is one of the accepted spec types (stored
     in the global `magic_spec`), raise ValueError otherwise.
     """
     if magic_kind not in magic_spec:
         raise ValueError('magic_kind must be one of %s, %s given' %
                          magic_kinds, magic_kind)
 
 
 # The docstrings for the decorator below will be fairly similar for the two
 # types (method and function), so we generate them here once and reuse the
 # templates below.
 _docstring_template = \
 """Decorate the given {0} as {1} magic.
 
 The decorator can be used with or without arguments, as follows.
 
 i) without arguments: it will create a {1} magic named as the {0} being
 decorated::
 
     @deco
     def foo(...)
 
 will create a {1} magic named `foo`.
 
 ii) with one string argument: which will be used as the actual name of the
 resulting magic::
 
     @deco('bar')
     def foo(...)
 
 will create a {1} magic named `bar`.
 
 To register a class magic use ``Interactiveshell.register_magic(class or instance)``.
 """
 
 # These two are decorator factories.  While they are conceptually very similar,
 # there are enough differences in the details that it's simpler to have them
 # written as completely standalone functions rather than trying to share code
 # and make a single one with convoluted logic.
 
 def _method_magic_marker(magic_kind):
     """Decorator factory for methods in Magics subclasses.
     """
 
     validate_type(magic_kind)
 
     # This is a closure to capture the magic_kind.  We could also use a class,
     # but it's overkill for just that one bit of state.
     def magic_deco(arg):
         if callable(arg):
             # "Naked" decorator call (just @foo, no args)
             func = arg
             name = func.__name__
             retval = arg
             record_magic(magics, magic_kind, name, name)
         elif isinstance(arg, str):
             # Decorator called with arguments (@foo('bar'))
             name = arg
             def mark(func, *a, **kw):
                 record_magic(magics, magic_kind, name, func.__name__)
                 return func
             retval = mark
         else:
             raise TypeError("Decorator can only be called with "
                             "string or function")
         return retval
 
     # Ensure the resulting decorator has a usable docstring
     magic_deco.__doc__ = _docstring_template.format('method', magic_kind)
     return magic_deco
 
 
 def _function_magic_marker(magic_kind):
     """Decorator factory for standalone functions.
     """
     validate_type(magic_kind)
     
     # This is a closure to capture the magic_kind.  We could also use a class,
     # but it's overkill for just that one bit of state.
     def magic_deco(arg):
         # Find get_ipython() in the caller's namespace
         caller = sys._getframe(1)
         for ns in ['f_locals', 'f_globals', 'f_builtins']:
             get_ipython = getattr(caller, ns).get('get_ipython')
             if get_ipython is not None:
                 break
         else:
             raise NameError('Decorator can only run in context where '
                             '`get_ipython` exists')
 
         ip = get_ipython()
 
         if callable(arg):
             # "Naked" decorator call (just @foo, no args)
             func = arg
             name = func.__name__
             ip.register_magic_function(func, magic_kind, name)
             retval = arg
         elif isinstance(arg, str):
             # Decorator called with arguments (@foo('bar'))
             name = arg
             def mark(func, *a, **kw):
                 ip.register_magic_function(func, magic_kind, name)
                 return func
             retval = mark
         else:
             raise TypeError("Decorator can only be called with "
                              "string or function")
         return retval
 
     # Ensure the resulting decorator has a usable docstring
     ds = _docstring_template.format('function', magic_kind)
 
     ds += dedent("""
     Note: this decorator can only be used in a context where IPython is already
     active, so that the `get_ipython()` call succeeds.  You can therefore use
     it in your startup files loaded after IPython initializes, but *not* in the
     IPython configuration file itself, which is executed before IPython is
     fully up and running.  Any file located in the `startup` subdirectory of
     your configuration profile will be OK in this sense.
     """)
     
     magic_deco.__doc__ = ds
     return magic_deco
 
 
-MAGIC_NO_VAR_EXPAND_ATTR = '_ipython_magic_no_var_expand'
+MAGIC_NO_VAR_EXPAND_ATTR = "_ipython_magic_no_var_expand"
+MAGIC_OUTPUT_CAN_BE_SILENCED = "_ipython_magic_output_can_be_silenced"
 
 
 def no_var_expand(magic_func):
     """Mark a magic function as not needing variable expansion
 
     By default, IPython interprets `{a}` or `$a` in the line passed to magics
     as variables that should be interpolated from the interactive namespace
     before passing the line to the magic function.
     This is not always desirable, e.g. when the magic executes Python code
     (%timeit, %time, etc.).
     Decorate magics with `@no_var_expand` to opt-out of variable expansion.
 
     .. versionadded:: 7.3
     """
     setattr(magic_func, MAGIC_NO_VAR_EXPAND_ATTR, True)
     return magic_func
 
 
+def output_can_be_silenced(magic_func):
+    """Mark a magic function so its output may be silenced.
+
+    The output is silenced if the Python code used as a parameter of
+    the magic ends in a semicolon, not counting a Python comment that can
+    follow it.
+    """
+    setattr(magic_func, MAGIC_OUTPUT_CAN_BE_SILENCED, True)
+    return magic_func
+
 # Create the actual decorators for public use
 
 # These three are used to decorate methods in class definitions
 line_magic = _method_magic_marker('line')
 cell_magic = _method_magic_marker('cell')
 line_cell_magic = _method_magic_marker('line_cell')
 
 # These three decorate standalone functions and perform the decoration
 # immediately.  They can only run where get_ipython() works
 register_line_magic = _function_magic_marker('line')
 register_cell_magic = _function_magic_marker('cell')
 register_line_cell_magic = _function_magic_marker('line_cell')
 
 #-----------------------------------------------------------------------------
 # Core Magic classes
 #-----------------------------------------------------------------------------
 
 class MagicsManager(Configurable):
     """Object that handles all magic-related functionality for IPython.
     """
     # Non-configurable class attributes
 
     # A two-level dict, first keyed by magic type, then by magic function, and
     # holding the actual callable object as value.  This is the dict used for
     # magic function dispatch
     magics = Dict()
     lazy_magics = Dict(
         help="""
     Mapping from magic names to modules to load.
 
     This can be used in IPython/IPykernel configuration to declare lazy magics
     that will only be imported/registered on first use.
 
     For example::
 
         c.MagicsManager.lazy_magics = {
           "my_magic": "slow.to.import",
           "my_other_magic": "also.slow",
         }
 
     On first invocation of `%my_magic`, `%%my_magic`, `%%my_other_magic` or
     `%%my_other_magic`, the corresponding module will be loaded as an ipython
     extensions as if you had previously done `%load_ext ipython`.
 
     Magics names should be without percent(s) as magics can be both cell
     and line magics.
 
     Lazy loading happen relatively late in execution process, and
     complex extensions that manipulate Python/IPython internal state or global state
     might not support lazy loading.
     """
     ).tag(
         config=True,
     )
 
     # A registry of the original objects that we've been given holding magics.
     registry = Dict()
 
     shell = Instance('IPython.core.interactiveshell.InteractiveShellABC', allow_none=True)
 
     auto_magic = Bool(True, help=
         "Automatically call line magics without requiring explicit % prefix"
     ).tag(config=True)
     @observe('auto_magic')
     def _auto_magic_changed(self, change):
         self.shell.automagic = change['new']
     
     _auto_status = [
         'Automagic is OFF, % prefix IS needed for line magics.',
         'Automagic is ON, % prefix IS NOT needed for line magics.']
 
     user_magics = Instance('IPython.core.magics.UserMagics', allow_none=True)
 
     def __init__(self, shell=None, config=None, user_magics=None, **traits):
 
         super(MagicsManager, self).__init__(shell=shell, config=config,
                                            user_magics=user_magics, **traits)
         self.magics = dict(line={}, cell={})
         # Let's add the user_magics to the registry for uniformity, so *all*
         # registered magic containers can be found there.
         self.registry[user_magics.__class__.__name__] = user_magics
 
     def auto_status(self):
         """Return descriptive string with automagic status."""
         return self._auto_status[self.auto_magic]
     
     def lsmagic(self):
         """Return a dict of currently available magic functions.
 
         The return dict has the keys 'line' and 'cell', corresponding to the
         two types of magics we support.  Each value is a list of names.
         """
         return self.magics
 
     def lsmagic_docs(self, brief=False, missing=''):
         """Return dict of documentation of magic functions.
 
         The return dict has the keys 'line' and 'cell', corresponding to the
         two types of magics we support. Each value is a dict keyed by magic
         name whose value is the function docstring. If a docstring is
         unavailable, the value of `missing` is used instead.
 
         If brief is True, only the first line of each docstring will be returned.
         """
         docs = {}
         for m_type in self.magics:
             m_docs = {}
             for m_name, m_func in self.magics[m_type].items():
                 if m_func.__doc__:
                     if brief:
                         m_docs[m_name] = m_func.__doc__.split('\n', 1)[0]
                     else:
                         m_docs[m_name] = m_func.__doc__.rstrip()
                 else:
                     m_docs[m_name] = missing
             docs[m_type] = m_docs
         return docs
 
     def register_lazy(self, name: str, fully_qualified_name: str):
         """
         Lazily register a magic via an extension.
 
 
         Parameters
         ----------
         name : str
             Name of the magic you wish to register.
         fully_qualified_name :
             Fully qualified name of the module/submodule that should be loaded
             as an extensions when the magic is first called.
             It is assumed that loading this extensions will register the given
             magic.
         """
 
         self.lazy_magics[name] = fully_qualified_name
 
     def register(self, *magic_objects):
         """Register one or more instances of Magics.
 
         Take one or more classes or instances of classes that subclass the main
         `core.Magic` class, and register them with IPython to use the magic
         functions they provide.  The registration process will then ensure that
         any methods that have decorated to provide line and/or cell magics will
         be recognized with the `%x`/`%%x` syntax as a line/cell magic
         respectively.
 
         If classes are given, they will be instantiated with the default
         constructor.  If your classes need a custom constructor, you should
         instanitate them first and pass the instance.
 
         The provided arguments can be an arbitrary mix of classes and instances.
 
         Parameters
         ----------
         *magic_objects : one or more classes or instances
         """
         # Start by validating them to ensure they have all had their magic
         # methods registered at the instance level
         for m in magic_objects:
             if not m.registered:
                 raise ValueError("Class of magics %r was constructed without "
                                  "the @register_magics class decorator")
             if isinstance(m, type):
                 # If we're given an uninstantiated class
                 m = m(shell=self.shell)
 
             # Now that we have an instance, we can register it and update the
             # table of callables
             self.registry[m.__class__.__name__] = m
             for mtype in magic_kinds:
                 self.magics[mtype].update(m.magics[mtype])
 
     def register_function(self, func, magic_kind='line', magic_name=None):
         """Expose a standalone function as magic function for IPython.
 
         This will create an IPython magic (line, cell or both) from a
         standalone function.  The functions should have the following
         signatures:
 
         * For line magics: `def f(line)`
         * For cell magics: `def f(line, cell)`
         * For a function that does both: `def f(line, cell=None)`
 
         In the latter case, the function will be called with `cell==None` when
         invoked as `%f`, and with cell as a string when invoked as `%%f`.
 
         Parameters
         ----------
         func : callable
             Function to be registered as a magic.
         magic_kind : str
             Kind of magic, one of 'line', 'cell' or 'line_cell'
         magic_name : optional str
             If given, the name the magic will have in the IPython namespace.  By
             default, the name of the function itself is used.
         """
 
         # Create the new method in the user_magics and register it in the
         # global table
         validate_type(magic_kind)
         magic_name = func.__name__ if magic_name is None else magic_name
         setattr(self.user_magics, magic_name, func)
         record_magic(self.magics, magic_kind, magic_name, func)
 
     def register_alias(self, alias_name, magic_name, magic_kind='line', magic_params=None):
         """Register an alias to a magic function.
 
         The alias is an instance of :class:`MagicAlias`, which holds the
         name and kind of the magic it should call. Binding is done at
         call time, so if the underlying magic function is changed the alias
         will call the new function.
 
         Parameters
         ----------
         alias_name : str
             The name of the magic to be registered.
         magic_name : str
             The name of an existing magic.
         magic_kind : str
             Kind of magic, one of 'line' or 'cell'
         """
 
         # `validate_type` is too permissive, as it allows 'line_cell'
         # which we do not handle.
         if magic_kind not in magic_kinds:
             raise ValueError('magic_kind must be one of %s, %s given' %
                              magic_kinds, magic_kind)
 
         alias = MagicAlias(self.shell, magic_name, magic_kind, magic_params)
         setattr(self.user_magics, alias_name, alias)
         record_magic(self.magics, magic_kind, alias_name, alias)
 
 # Key base class that provides the central functionality for magics.
 
 
 class Magics(Configurable):
     """Base class for implementing magic functions.
 
     Shell functions which can be reached as %function_name. All magic
     functions should accept a string, which they can parse for their own
     needs. This can make some functions easier to type, eg `%cd ../`
     vs. `%cd("../")`
 
     Classes providing magic functions need to subclass this class, and they
     MUST:
 
     - Use the method decorators `@line_magic` and `@cell_magic` to decorate
       individual methods as magic functions, AND
 
     - Use the class decorator `@magics_class` to ensure that the magic
       methods are properly registered at the instance level upon instance
       initialization.
 
     See :mod:`magic_functions` for examples of actual implementation classes.
     """
     # Dict holding all command-line options for each magic.
     options_table = None
     # Dict for the mapping of magic names to methods, set by class decorator
     magics = None
     # Flag to check that the class decorator was properly applied
     registered = False
     # Instance of IPython shell
     shell = None
 
     def __init__(self, shell=None, **kwargs):
         if not(self.__class__.registered):
             raise ValueError('Magics subclass without registration - '
                              'did you forget to apply @magics_class?')
         if shell is not None:
             if hasattr(shell, 'configurables'):
                 shell.configurables.append(self)
             if hasattr(shell, 'config'):
                 kwargs.setdefault('parent', shell)
 
         self.shell = shell
         self.options_table = {}
         # The method decorators are run when the instance doesn't exist yet, so
         # they can only record the names of the methods they are supposed to
         # grab.  Only now, that the instance exists, can we create the proper
         # mapping to bound methods.  So we read the info off the original names
         # table and replace each method name by the actual bound method.
         # But we mustn't clobber the *class* mapping, in case of multiple instances.
         class_magics = self.magics
         self.magics = {}
         for mtype in magic_kinds:
             tab = self.magics[mtype] = {}
             cls_tab = class_magics[mtype]
             for magic_name, meth_name in cls_tab.items():
                 if isinstance(meth_name, str):
                     # it's a method name, grab it
                     tab[magic_name] = getattr(self, meth_name)
                 else:
                     # it's the real thing
                     tab[magic_name] = meth_name
         # Configurable **needs** to be initiated at the end or the config
         # magics get screwed up.
         super(Magics, self).__init__(**kwargs)
 
     def arg_err(self,func):
         """Print docstring if incorrect arguments were passed"""
         print('Error in arguments:')
         print(oinspect.getdoc(func))
 
     def format_latex(self, strng):
         """Format a string for latex inclusion."""
 
         # Characters that need to be escaped for latex:
         escape_re = re.compile(r'(%|_|\$|#|&)',re.MULTILINE)
         # Magic command names as headers:
         cmd_name_re = re.compile(r'^(%s.*?):' % ESC_MAGIC,
                                  re.MULTILINE)
         # Magic commands
         cmd_re = re.compile(r'(?P<cmd>%s.+?\b)(?!\}\}:)' % ESC_MAGIC,
                             re.MULTILINE)
         # Paragraph continue
         par_re = re.compile(r'\\$',re.MULTILINE)
 
         # The "\n" symbol
         newline_re = re.compile(r'\\n')
 
         # Now build the string for output:
         #strng = cmd_name_re.sub(r'\n\\texttt{\\textsl{\\large \1}}:',strng)
         strng = cmd_name_re.sub(r'\n\\bigskip\n\\texttt{\\textbf{ \1}}:',
                                 strng)
         strng = cmd_re.sub(r'\\texttt{\g<cmd>}',strng)
         strng = par_re.sub(r'\\\\',strng)
         strng = escape_re.sub(r'\\\1',strng)
         strng = newline_re.sub(r'\\textbackslash{}n',strng)
         return strng
 
     def parse_options(self, arg_str, opt_str, *long_opts, **kw):
         """Parse options passed to an argument string.
 
         The interface is similar to that of :func:`getopt.getopt`, but it
         returns a :class:`~IPython.utils.struct.Struct` with the options as keys
         and the stripped argument string still as a string.
 
         arg_str is quoted as a true sys.argv vector by using shlex.split.
         This allows us to easily expand variables, glob files, quote
         arguments, etc.
 
         Parameters
         ----------
         arg_str : str
             The arguments to parse.
         opt_str : str
             The options specification.
         mode : str, default 'string'
             If given as 'list', the argument string is returned as a list (split
             on whitespace) instead of a string.
         list_all : bool, default False
             Put all option values in lists. Normally only options
             appearing more than once are put in a list.
         posix : bool, default True
             Whether to split the input line in POSIX mode or not, as per the
             conventions outlined in the :mod:`shlex` module from the standard
             library.
         """
 
         # inject default options at the beginning of the input line
         caller = sys._getframe(1).f_code.co_name
         arg_str = '%s %s' % (self.options_table.get(caller,''),arg_str)
 
         mode = kw.get('mode','string')
         if mode not in ['string','list']:
             raise ValueError('incorrect mode given: %s' % mode)
         # Get options
         list_all = kw.get('list_all',0)
         posix = kw.get('posix', os.name == 'posix')
         strict = kw.get('strict', True)
 
         preserve_non_opts = kw.get("preserve_non_opts", False)
         remainder_arg_str = arg_str
 
         # Check if we have more than one argument to warrant extra processing:
         odict = {}  # Dictionary with options
         args = arg_str.split()
         if len(args) >= 1:
             # If the list of inputs only has 0 or 1 thing in it, there's no
             # need to look for options
             argv = arg_split(arg_str, posix, strict)
             # Do regular option processing
             try:
                 opts,args = getopt(argv, opt_str, long_opts)
             except GetoptError as e:
                 raise UsageError(
                     '%s ( allowed: "%s" %s)' % (e.msg, opt_str, " ".join(long_opts))
                 ) from e
             for o, a in opts:
                 if mode == "string" and preserve_non_opts:
                     # remove option-parts from the original args-string and preserve remaining-part.
                     # This relies on the arg_split(...) and getopt(...)'s impl spec, that the parsed options are
                     # returned in the original order.
                     remainder_arg_str = remainder_arg_str.replace(o, "", 1).replace(
                         a, "", 1
                     )
                 if o.startswith("--"):
                     o = o[2:]
                 else:
                     o = o[1:]
                 try:
                     odict[o].append(a)
                 except AttributeError:
                     odict[o] = [odict[o],a]
                 except KeyError:
                     if list_all:
                         odict[o] = [a]
                     else:
                         odict[o] = a
 
         # Prepare opts,args for return
         opts = Struct(odict)
         if mode == 'string':
             if preserve_non_opts:
                 args = remainder_arg_str.lstrip()
             else:
                 args = " ".join(args)
 
         return opts,args
 
     def default_option(self, fn, optstr):
         """Make an entry in the options_table for fn, with value optstr"""
 
         if fn not in self.lsmagic():
             error("%s is not a magic function" % fn)
         self.options_table[fn] = optstr
 
 
 class MagicAlias(object):
     """An alias to another magic function.
 
     An alias is determined by its magic name and magic kind. Lookup
     is done at call time, so if the underlying magic changes the alias
     will call the new function.
 
     Use the :meth:`MagicsManager.register_alias` method or the
     `%alias_magic` magic function to create and register a new alias.
     """
     def __init__(self, shell, magic_name, magic_kind, magic_params=None):
         self.shell = shell
         self.magic_name = magic_name
         self.magic_params = magic_params
         self.magic_kind = magic_kind
 
         self.pretty_target = '%s%s' % (magic_escapes[self.magic_kind], self.magic_name)
         self.__doc__ = "Alias for `%s`." % self.pretty_target
 
         self._in_call = False
 
     def __call__(self, *args, **kwargs):
         """Call the magic alias."""
         fn = self.shell.find_magic(self.magic_name, self.magic_kind)
         if fn is None:
             raise UsageError("Magic `%s` not found." % self.pretty_target)
 
         # Protect against infinite recursion.
         if self._in_call:
             raise UsageError("Infinite recursion detected; "
                              "magic aliases cannot call themselves.")
         self._in_call = True
         try:
             if self.magic_params:
                 args_list = list(args)
                 args_list[0] = self.magic_params + " " + args[0]
                 args = tuple(args_list)
             return fn(*args, **kwargs)
         finally:
             self._in_call = False

IPython/core/magics/execution.py

@@ -1,1510 +1,1512 b''
 # -*- coding: utf-8 -*-
 """Implementation of execution-related magic functions."""
 
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 
 import ast
 import bdb
 import builtins as builtin_mod
 import cProfile as profile
 import gc
 import itertools
 import math
 import os
 import pstats
 import re
 import shlex
 import sys
 import time
 import timeit
 from ast import Module
 from io import StringIO
 from logging import error
 from pathlib import Path
 from pdb import Restart
 from warnings import warn
 
 from IPython.core import magic_arguments, oinspect, page
 from IPython.core.error import UsageError
 from IPython.core.macro import Macro
 from IPython.core.magic import (
     Magics,
     cell_magic,
     line_cell_magic,
     line_magic,
     magics_class,
     needs_local_scope,
     no_var_expand,
+    output_can_be_silenced,
     on_off,
 )
 from IPython.testing.skipdoctest import skip_doctest
 from IPython.utils.capture import capture_output
 from IPython.utils.contexts import preserve_keys
 from IPython.utils.ipstruct import Struct
 from IPython.utils.module_paths import find_mod
 from IPython.utils.path import get_py_filename, shellglob
 from IPython.utils.timing import clock, clock2
 
 #-----------------------------------------------------------------------------
 # Magic implementation classes
 #-----------------------------------------------------------------------------
 
 
 class TimeitResult(object):
     """
     Object returned by the timeit magic with info about the run.
 
     Contains the following attributes :
 
     loops: (int) number of loops done per measurement
     repeat: (int) number of times the measurement has been repeated
     best: (float) best execution time / number
     all_runs: (list of float) execution time of each run (in s)
     compile_time: (float) time of statement compilation (s)
 
     """
     def __init__(self, loops, repeat, best, worst, all_runs, compile_time, precision):
         self.loops = loops
         self.repeat = repeat
         self.best = best
         self.worst = worst
         self.all_runs = all_runs
         self.compile_time = compile_time
         self._precision = precision
         self.timings = [ dt / self.loops for dt in all_runs]
 
     @property
     def average(self):
         return math.fsum(self.timings) / len(self.timings)
 
     @property
     def stdev(self):
         mean = self.average
         return (math.fsum([(x - mean) ** 2 for x in self.timings]) / len(self.timings)) ** 0.5
 
     def __str__(self):
         pm = '+-'
         if hasattr(sys.stdout, 'encoding') and sys.stdout.encoding:
             try:
                 u'\xb1'.encode(sys.stdout.encoding)
                 pm = u'\xb1'
             except:
                 pass
         return "{mean} {pm} {std} per loop (mean {pm} std. dev. of {runs} run{run_plural}, {loops:,} loop{loop_plural} each)".format(
             pm=pm,
             runs=self.repeat,
             loops=self.loops,
             loop_plural="" if self.loops == 1 else "s",
             run_plural="" if self.repeat == 1 else "s",
             mean=_format_time(self.average, self._precision),
             std=_format_time(self.stdev, self._precision),
         )
 
     def _repr_pretty_(self, p , cycle):
         unic = self.__str__()
         p.text(u'<TimeitResult : '+unic+u'>')
 
 
 class TimeitTemplateFiller(ast.NodeTransformer):
     """Fill in the AST template for timing execution.
 
     This is quite closely tied to the template definition, which is in
     :meth:`ExecutionMagics.timeit`.
     """
     def __init__(self, ast_setup, ast_stmt):
         self.ast_setup = ast_setup
         self.ast_stmt = ast_stmt
 
     def visit_FunctionDef(self, node):
         "Fill in the setup statement"
         self.generic_visit(node)
         if node.name == "inner":
             node.body[:1] = self.ast_setup.body
 
         return node
 
     def visit_For(self, node):
         "Fill in the statement to be timed"
         if getattr(getattr(node.body[0], 'value', None), 'id', None) == 'stmt':
             node.body = self.ast_stmt.body
         return node
 
 
 class Timer(timeit.Timer):
     """Timer class that explicitly uses self.inner
     
     which is an undocumented implementation detail of CPython,
     not shared by PyPy.
     """
     # Timer.timeit copied from CPython 3.4.2
     def timeit(self, number=timeit.default_number):
         """Time 'number' executions of the main statement.
 
         To be precise, this executes the setup statement once, and
         then returns the time it takes to execute the main statement
         a number of times, as a float measured in seconds.  The
         argument is the number of times through the loop, defaulting
         to one million.  The main statement, the setup statement and
         the timer function to be used are passed to the constructor.
         """
         it = itertools.repeat(None, number)
         gcold = gc.isenabled()
         gc.disable()
         try:
             timing = self.inner(it, self.timer)
         finally:
             if gcold:
                 gc.enable()
         return timing
 
 
 @magics_class
 class ExecutionMagics(Magics):
     """Magics related to code execution, debugging, profiling, etc.
 
     """
 
     def __init__(self, shell):
         super(ExecutionMagics, self).__init__(shell)
         # Default execution function used to actually run user code.
         self.default_runner = None
 
     @skip_doctest
     @no_var_expand
     @line_cell_magic
     def prun(self, parameter_s='', cell=None):
 
         """Run a statement through the python code profiler.
 
         Usage, in line mode:
           %prun [options] statement
 
         Usage, in cell mode:
           %%prun [options] [statement]
           code...
           code...
 
         In cell mode, the additional code lines are appended to the (possibly
         empty) statement in the first line.  Cell mode allows you to easily
         profile multiline blocks without having to put them in a separate
         function.
 
         The given statement (which doesn't require quote marks) is run via the
         python profiler in a manner similar to the profile.run() function.
         Namespaces are internally managed to work correctly; profile.run
         cannot be used in IPython because it makes certain assumptions about
         namespaces which do not hold under IPython.
 
         Options:
 
         -l <limit>
           you can place restrictions on what or how much of the
           profile gets printed. The limit value can be:
 
              * A string: only information for function names containing this string
                is printed.
 
              * An integer: only these many lines are printed.
 
              * A float (between 0 and 1): this fraction of the report is printed
                (for example, use a limit of 0.4 to see the topmost 40% only).
 
           You can combine several limits with repeated use of the option. For
           example, ``-l __init__ -l 5`` will print only the topmost 5 lines of
           information about class constructors.
 
         -r
           return the pstats.Stats object generated by the profiling. This
           object has all the information about the profile in it, and you can
           later use it for further analysis or in other functions.
 
         -s <key>
           sort profile by given key. You can provide more than one key
           by using the option several times: '-s key1 -s key2 -s key3...'. The
           default sorting key is 'time'.
 
           The following is copied verbatim from the profile documentation
           referenced below:
 
           When more than one key is provided, additional keys are used as
           secondary criteria when the there is equality in all keys selected
           before them.
 
           Abbreviations can be used for any key names, as long as the
           abbreviation is unambiguous.  The following are the keys currently
           defined:
 
           ============  =====================
           Valid Arg     Meaning
           ============  =====================
           "calls"       call count
           "cumulative"  cumulative time
           "file"        file name
           "module"      file name
           "pcalls"      primitive call count
           "line"        line number
           "name"        function name
           "nfl"         name/file/line
           "stdname"     standard name
           "time"        internal time
           ============  =====================
 
           Note that all sorts on statistics are in descending order (placing
           most time consuming items first), where as name, file, and line number
           searches are in ascending order (i.e., alphabetical). The subtle
           distinction between "nfl" and "stdname" is that the standard name is a
           sort of the name as printed, which means that the embedded line
           numbers get compared in an odd way.  For example, lines 3, 20, and 40
           would (if the file names were the same) appear in the string order
           "20" "3" and "40".  In contrast, "nfl" does a numeric compare of the
           line numbers.  In fact, sort_stats("nfl") is the same as
           sort_stats("name", "file", "line").
 
         -T <filename>
           save profile results as shown on screen to a text
           file. The profile is still shown on screen.
 
         -D <filename>
           save (via dump_stats) profile statistics to given
           filename. This data is in a format understood by the pstats module, and
           is generated by a call to the dump_stats() method of profile
           objects. The profile is still shown on screen.
 
         -q
           suppress output to the pager.  Best used with -T and/or -D above.
 
         If you want to run complete programs under the profiler's control, use
         ``%run -p [prof_opts] filename.py [args to program]`` where prof_opts
         contains profiler specific options as described here.
 
         You can read the complete documentation for the profile module with::
 
           In [1]: import profile; profile.help()
 
         .. versionchanged:: 7.3
             User variables are no longer expanded,
             the magic line is always left unmodified.
 
         """
         opts, arg_str = self.parse_options(parameter_s, 'D:l:rs:T:q',
                                            list_all=True, posix=False)
         if cell is not None:
             arg_str += '\n' + cell
         arg_str = self.shell.transform_cell(arg_str)
         return self._run_with_profiler(arg_str, opts, self.shell.user_ns)
 
     def _run_with_profiler(self, code, opts, namespace):
         """
         Run `code` with profiler.  Used by ``%prun`` and ``%run -p``.
 
         Parameters
         ----------
         code : str
             Code to be executed.
         opts : Struct
             Options parsed by `self.parse_options`.
         namespace : dict
             A dictionary for Python namespace (e.g., `self.shell.user_ns`).
 
         """
 
         # Fill default values for unspecified options:
         opts.merge(Struct(D=[''], l=[], s=['time'], T=['']))
 
         prof = profile.Profile()
         try:
             prof = prof.runctx(code, namespace, namespace)
             sys_exit = ''
         except SystemExit:
             sys_exit = """*** SystemExit exception caught in code being profiled."""
 
         stats = pstats.Stats(prof).strip_dirs().sort_stats(*opts.s)
 
         lims = opts.l
         if lims:
             lims = []  # rebuild lims with ints/floats/strings
             for lim in opts.l:
                 try:
                     lims.append(int(lim))
                 except ValueError:
                     try:
                         lims.append(float(lim))
                     except ValueError:
                         lims.append(lim)
 
         # Trap output.
         stdout_trap = StringIO()
         stats_stream = stats.stream
         try:
             stats.stream = stdout_trap
             stats.print_stats(*lims)
         finally:
             stats.stream = stats_stream
 
         output = stdout_trap.getvalue()
         output = output.rstrip()
 
         if 'q' not in opts:
             page.page(output)
         print(sys_exit, end=' ')
 
         dump_file = opts.D[0]
         text_file = opts.T[0]
         if dump_file:
             prof.dump_stats(dump_file)
             print(
                 f"\n*** Profile stats marshalled to file {repr(dump_file)}.{sys_exit}"
             )
         if text_file:
             pfile = Path(text_file)
             pfile.touch(exist_ok=True)
             pfile.write_text(output, encoding="utf-8")
 
             print(
                 f"\n*** Profile printout saved to text file {repr(text_file)}.{sys_exit}"
             )
 
         if 'r' in opts:
             return stats
 
         return None
 
     @line_magic
     def pdb(self, parameter_s=''):
         """Control the automatic calling of the pdb interactive debugger.
 
         Call as '%pdb on', '%pdb 1', '%pdb off' or '%pdb 0'. If called without
         argument it works as a toggle.
 
         When an exception is triggered, IPython can optionally call the
         interactive pdb debugger after the traceback printout. %pdb toggles
         this feature on and off.
 
         The initial state of this feature is set in your configuration
         file (the option is ``InteractiveShell.pdb``).
 
         If you want to just activate the debugger AFTER an exception has fired,
         without having to type '%pdb on' and rerunning your code, you can use
         the %debug magic."""
 
         par = parameter_s.strip().lower()
 
         if par:
             try:
                 new_pdb = {'off':0,'0':0,'on':1,'1':1}[par]
             except KeyError:
                 print ('Incorrect argument. Use on/1, off/0, '
                        'or nothing for a toggle.')
                 return
         else:
             # toggle
             new_pdb = not self.shell.call_pdb
 
         # set on the shell
         self.shell.call_pdb = new_pdb
         print('Automatic pdb calling has been turned',on_off(new_pdb))
 
     @magic_arguments.magic_arguments()
     @magic_arguments.argument('--breakpoint', '-b', metavar='FILE:LINE',
         help="""
         Set break point at LINE in FILE.
         """
     )
     @magic_arguments.argument('statement', nargs='*',
         help="""
         Code to run in debugger.
         You can omit this in cell magic mode.
         """
     )
     @no_var_expand
     @line_cell_magic
     def debug(self, line='', cell=None):
         """Activate the interactive debugger.
 
         This magic command support two ways of activating debugger.
         One is to activate debugger before executing code.  This way, you
         can set a break point, to step through the code from the point.
         You can use this mode by giving statements to execute and optionally
         a breakpoint.
 
         The other one is to activate debugger in post-mortem mode.  You can
         activate this mode simply running %debug without any argument.
         If an exception has just occurred, this lets you inspect its stack
         frames interactively.  Note that this will always work only on the last
         traceback that occurred, so you must call this quickly after an
         exception that you wish to inspect has fired, because if another one
         occurs, it clobbers the previous one.
 
         If you want IPython to automatically do this on every exception, see
         the %pdb magic for more details.
 
         .. versionchanged:: 7.3
             When running code, user variables are no longer expanded,
             the magic line is always left unmodified.
 
         """
         args = magic_arguments.parse_argstring(self.debug, line)
 
         if not (args.breakpoint or args.statement or cell):
             self._debug_post_mortem()
         elif not (args.breakpoint or cell):
             # If there is no breakpoints, the line is just code to execute
             self._debug_exec(line, None)
         else:
             # Here we try to reconstruct the code from the output of
             # parse_argstring. This might not work if the code has spaces
             # For example this fails for `print("a b")`
             code = "\n".join(args.statement)
             if cell:
                 code += "\n" + cell
             self._debug_exec(code, args.breakpoint)
 
     def _debug_post_mortem(self):
         self.shell.debugger(force=True)
 
     def _debug_exec(self, code, breakpoint):
         if breakpoint:
             (filename, bp_line) = breakpoint.rsplit(':', 1)
             bp_line = int(bp_line)
         else:
             (filename, bp_line) = (None, None)
         self._run_with_debugger(code, self.shell.user_ns, filename, bp_line)
 
     @line_magic
     def tb(self, s):
         """Print the last traceback.
 
         Optionally, specify an exception reporting mode, tuning the
         verbosity of the traceback. By default the currently-active exception
         mode is used. See %xmode for changing exception reporting modes.
 
         Valid modes: Plain, Context, Verbose, and Minimal.
         """
         interactive_tb = self.shell.InteractiveTB
         if s:
             # Switch exception reporting mode for this one call.
             # Ensure it is switched back.
             def xmode_switch_err(name):
                 warn('Error changing %s exception modes.\n%s' %
                     (name,sys.exc_info()[1]))
 
             new_mode = s.strip().capitalize()
             original_mode = interactive_tb.mode
             try:
                 try:
                     interactive_tb.set_mode(mode=new_mode)
                 except Exception:
                     xmode_switch_err('user')
                 else:
                     self.shell.showtraceback()
             finally:
                 interactive_tb.set_mode(mode=original_mode)
         else:
             self.shell.showtraceback()
 
     @skip_doctest
     @line_magic
     def run(self, parameter_s='', runner=None,
                   file_finder=get_py_filename):
         """Run the named file inside IPython as a program.
 
         Usage::
 
           %run [-n -i -e -G]
                [( -t [-N<N>] | -d [-b<N>] | -p [profile options] )]
                ( -m mod | filename ) [args]
 
         The filename argument should be either a pure Python script (with
         extension ``.py``), or a file with custom IPython syntax (such as
         magics). If the latter, the file can be either a script with ``.ipy``
         extension, or a Jupyter notebook with ``.ipynb`` extension. When running
         a Jupyter notebook, the output from print statements and other
         displayed objects will appear in the terminal (even matplotlib figures
         will open, if a terminal-compliant backend is being used). Note that,
         at the system command line, the ``jupyter run`` command offers similar
         functionality for executing notebooks (albeit currently with some
         differences in supported options).
 
         Parameters after the filename are passed as command-line arguments to
         the program (put in sys.argv). Then, control returns to IPython's
         prompt.
 
         This is similar to running at a system prompt ``python file args``,
         but with the advantage of giving you IPython's tracebacks, and of
         loading all variables into your interactive namespace for further use
         (unless -p is used, see below).
 
         The file is executed in a namespace initially consisting only of
         ``__name__=='__main__'`` and sys.argv constructed as indicated. It thus
         sees its environment as if it were being run as a stand-alone program
         (except for sharing global objects such as previously imported
         modules). But after execution, the IPython interactive namespace gets
         updated with all variables defined in the program (except for __name__
         and sys.argv). This allows for very convenient loading of code for
         interactive work, while giving each program a 'clean sheet' to run in.
 
         Arguments are expanded using shell-like glob match.  Patterns
         '*', '?', '[seq]' and '[!seq]' can be used.  Additionally,
         tilde '~' will be expanded into user's home directory.  Unlike
         real shells, quotation does not suppress expansions.  Use
         *two* back slashes (e.g. ``\\\\*``) to suppress expansions.
         To completely disable these expansions, you can use -G flag.
 
         On Windows systems, the use of single quotes `'` when specifying
         a file is not supported. Use double quotes `"`.
 
         Options:
 
         -n
           __name__ is NOT set to '__main__', but to the running file's name
           without extension (as python does under import).  This allows running
           scripts and reloading the definitions in them without calling code
           protected by an ``if __name__ == "__main__"`` clause.
 
         -i
           run the file in IPython's namespace instead of an empty one. This
           is useful if you are experimenting with code written in a text editor
           which depends on variables defined interactively.
 
         -e
           ignore sys.exit() calls or SystemExit exceptions in the script
           being run.  This is particularly useful if IPython is being used to
           run unittests, which always exit with a sys.exit() call.  In such
           cases you are interested in the output of the test results, not in
           seeing a traceback of the unittest module.
 
         -t
           print timing information at the end of the run.  IPython will give
           you an estimated CPU time consumption for your script, which under
           Unix uses the resource module to avoid the wraparound problems of
           time.clock().  Under Unix, an estimate of time spent on system tasks
           is also given (for Windows platforms this is reported as 0.0).
 
         If -t is given, an additional ``-N<N>`` option can be given, where <N>
         must be an integer indicating how many times you want the script to
         run.  The final timing report will include total and per run results.
 
         For example (testing the script uniq_stable.py)::
 
             In [1]: run -t uniq_stable
 
             IPython CPU timings (estimated):
               User  :    0.19597 s.
               System:        0.0 s.
 
             In [2]: run -t -N5 uniq_stable
 
             IPython CPU timings (estimated):
             Total runs performed: 5
               Times :      Total       Per run
               User  :   0.910862 s,  0.1821724 s.
               System:        0.0 s,        0.0 s.
 
         -d
           run your program under the control of pdb, the Python debugger.
           This allows you to execute your program step by step, watch variables,
           etc.  Internally, what IPython does is similar to calling::
 
               pdb.run('execfile("YOURFILENAME")')
 
           with a breakpoint set on line 1 of your file.  You can change the line
           number for this automatic breakpoint to be <N> by using the -bN option
           (where N must be an integer). For example::
 
               %run -d -b40 myscript
 
           will set the first breakpoint at line 40 in myscript.py.  Note that
           the first breakpoint must be set on a line which actually does
           something (not a comment or docstring) for it to stop execution.
 
           Or you can specify a breakpoint in a different file::
 
               %run -d -b myotherfile.py:20 myscript
 
           When the pdb debugger starts, you will see a (Pdb) prompt.  You must
           first enter 'c' (without quotes) to start execution up to the first
           breakpoint.
 
           Entering 'help' gives information about the use of the debugger.  You
           can easily see pdb's full documentation with "import pdb;pdb.help()"
           at a prompt.
 
         -p
           run program under the control of the Python profiler module (which
           prints a detailed report of execution times, function calls, etc).
 
           You can pass other options after -p which affect the behavior of the
           profiler itself. See the docs for %prun for details.
 
           In this mode, the program's variables do NOT propagate back to the
           IPython interactive namespace (because they remain in the namespace
           where the profiler executes them).
 
           Internally this triggers a call to %prun, see its documentation for
           details on the options available specifically for profiling.
 
         There is one special usage for which the text above doesn't apply:
         if the filename ends with .ipy[nb], the file is run as ipython script,
         just as if the commands were written on IPython prompt.
 
         -m
           specify module name to load instead of script path. Similar to
           the -m option for the python interpreter. Use this option last if you
           want to combine with other %run options. Unlike the python interpreter
           only source modules are allowed no .pyc or .pyo files.
           For example::
 
               %run -m example
 
           will run the example module.
 
         -G
           disable shell-like glob expansion of arguments.
 
         """
 
         # Logic to handle issue #3664
         # Add '--' after '-m <module_name>' to ignore additional args passed to a module.
         if '-m' in parameter_s and '--' not in parameter_s:
             argv = shlex.split(parameter_s, posix=(os.name == 'posix'))
             for idx, arg in enumerate(argv):
                 if arg and arg.startswith('-') and arg != '-':
                     if arg == '-m':
                         argv.insert(idx + 2, '--')
                         break
                 else:
                     # Positional arg, break
                     break
             parameter_s = ' '.join(shlex.quote(arg) for arg in argv)
 
         # get arguments and set sys.argv for program to be run.
         opts, arg_lst = self.parse_options(parameter_s,
                                            'nidtN:b:pD:l:rs:T:em:G',
                                            mode='list', list_all=1)
         if "m" in opts:
             modulename = opts["m"][0]
             modpath = find_mod(modulename)
             if modpath is None:
                 msg = '%r is not a valid modulename on sys.path'%modulename
                 raise Exception(msg)
             arg_lst = [modpath] + arg_lst
         try:
             fpath = None # initialize to make sure fpath is in scope later
             fpath = arg_lst[0]
             filename = file_finder(fpath)
         except IndexError as e:
             msg = 'you must provide at least a filename.'
             raise Exception(msg) from e
         except IOError as e:
             try:
                 msg = str(e)
             except UnicodeError:
                 msg = e.message
             if os.name == 'nt' and re.match(r"^'.*'$",fpath):
                 warn('For Windows, use double quotes to wrap a filename: %run "mypath\\myfile.py"')
             raise Exception(msg) from e
         except TypeError:
             if fpath in sys.meta_path:
                 filename = ""
             else:
                 raise
 
         if filename.lower().endswith(('.ipy', '.ipynb')):
             with preserve_keys(self.shell.user_ns, '__file__'):
                 self.shell.user_ns['__file__'] = filename
                 self.shell.safe_execfile_ipy(filename, raise_exceptions=True)
             return
 
         # Control the response to exit() calls made by the script being run
         exit_ignore = 'e' in opts
 
         # Make sure that the running script gets a proper sys.argv as if it
         # were run from a system shell.
         save_argv = sys.argv # save it for later restoring
 
         if 'G' in opts:
             args = arg_lst[1:]
         else:
             # tilde and glob expansion
             args = shellglob(map(os.path.expanduser,  arg_lst[1:]))
 
         sys.argv = [filename] + args  # put in the proper filename
 
         if 'n' in opts:
             name = Path(filename).stem
         else:
             name = '__main__'
 
         if 'i' in opts:
             # Run in user's interactive namespace
             prog_ns = self.shell.user_ns
             __name__save = self.shell.user_ns['__name__']
             prog_ns['__name__'] = name
             main_mod = self.shell.user_module
 
             # Since '%run foo' emulates 'python foo.py' at the cmd line, we must
             # set the __file__ global in the script's namespace
             # TK: Is this necessary in interactive mode?
             prog_ns['__file__'] = filename
         else:
             # Run in a fresh, empty namespace
 
             # The shell MUST hold a reference to prog_ns so after %run
             # exits, the python deletion mechanism doesn't zero it out
             # (leaving dangling references). See interactiveshell for details
             main_mod = self.shell.new_main_mod(filename, name)
             prog_ns = main_mod.__dict__
 
         # pickle fix.  See interactiveshell for an explanation.  But we need to
         # make sure that, if we overwrite __main__, we replace it at the end
         main_mod_name = prog_ns['__name__']
 
         if main_mod_name == '__main__':
             restore_main = sys.modules['__main__']
         else:
             restore_main = False
 
         # This needs to be undone at the end to prevent holding references to
         # every single object ever created.
         sys.modules[main_mod_name] = main_mod
 
         if 'p' in opts or 'd' in opts:
             if 'm' in opts:
                 code = 'run_module(modulename, prog_ns)'
                 code_ns = {
                     'run_module': self.shell.safe_run_module,
                     'prog_ns': prog_ns,
                     'modulename': modulename,
                 }
             else:
                 if 'd' in opts:
                     # allow exceptions to raise in debug mode
                     code = 'execfile(filename, prog_ns, raise_exceptions=True)'
                 else:
                     code = 'execfile(filename, prog_ns)'
                 code_ns = {
                     'execfile': self.shell.safe_execfile,
                     'prog_ns': prog_ns,
                     'filename': get_py_filename(filename),
                 }
 
         try:
             stats = None
             if 'p' in opts:
                 stats = self._run_with_profiler(code, opts, code_ns)
             else:
                 if 'd' in opts:
                     bp_file, bp_line = parse_breakpoint(
                         opts.get('b', ['1'])[0], filename)
                     self._run_with_debugger(
                         code, code_ns, filename, bp_line, bp_file)
                 else:
                     if 'm' in opts:
                         def run():
                             self.shell.safe_run_module(modulename, prog_ns)
                     else:
                         if runner is None:
                             runner = self.default_runner
                         if runner is None:
                             runner = self.shell.safe_execfile
 
                         def run():
                             runner(filename, prog_ns, prog_ns,
                                     exit_ignore=exit_ignore)
 
                     if 't' in opts:
                         # timed execution
                         try:
                             nruns = int(opts['N'][0])
                             if nruns < 1:
                                 error('Number of runs must be >=1')
                                 return
                         except (KeyError):
                             nruns = 1
                         self._run_with_timing(run, nruns)
                     else:
                         # regular execution
                         run()
 
             if 'i' in opts:
                 self.shell.user_ns['__name__'] = __name__save
             else:
                 # update IPython interactive namespace
 
                 # Some forms of read errors on the file may mean the
                 # __name__ key was never set; using pop we don't have to
                 # worry about a possible KeyError.
                 prog_ns.pop('__name__', None)
 
                 with preserve_keys(self.shell.user_ns, '__file__'):
                     self.shell.user_ns.update(prog_ns)
         finally:
             # It's a bit of a mystery why, but __builtins__ can change from
             # being a module to becoming a dict missing some key data after
             # %run.  As best I can see, this is NOT something IPython is doing
             # at all, and similar problems have been reported before:
             # http://coding.derkeiler.com/Archive/Python/comp.lang.python/2004-10/0188.html
             # Since this seems to be done by the interpreter itself, the best
             # we can do is to at least restore __builtins__ for the user on
             # exit.
             self.shell.user_ns['__builtins__'] = builtin_mod
 
             # Ensure key global structures are restored
             sys.argv = save_argv
             if restore_main:
                 sys.modules['__main__'] = restore_main
                 if '__mp_main__' in sys.modules:
                     sys.modules['__mp_main__'] = restore_main
             else:
                 # Remove from sys.modules the reference to main_mod we'd
                 # added.  Otherwise it will trap references to objects
                 # contained therein.
                 del sys.modules[main_mod_name]
 
         return stats
 
     def _run_with_debugger(self, code, code_ns, filename=None,
                            bp_line=None, bp_file=None):
         """
         Run `code` in debugger with a break point.
 
         Parameters
         ----------
         code : str
             Code to execute.
         code_ns : dict
             A namespace in which `code` is executed.
         filename : str
             `code` is ran as if it is in `filename`.
         bp_line : int, optional
             Line number of the break point.
         bp_file : str, optional
             Path to the file in which break point is specified.
             `filename` is used if not given.
 
         Raises
         ------
         UsageError
             If the break point given by `bp_line` is not valid.
 
         """
         deb = self.shell.InteractiveTB.pdb
         if not deb:
             self.shell.InteractiveTB.pdb = self.shell.InteractiveTB.debugger_cls()
             deb = self.shell.InteractiveTB.pdb
 
         # deb.checkline() fails if deb.curframe exists but is None; it can
         # handle it not existing. https://github.com/ipython/ipython/issues/10028
         if hasattr(deb, 'curframe'):
             del deb.curframe
 
         # reset Breakpoint state, which is moronically kept
         # in a class
         bdb.Breakpoint.next = 1
         bdb.Breakpoint.bplist = {}
         bdb.Breakpoint.bpbynumber = [None]
         deb.clear_all_breaks()
         if bp_line is not None:
             # Set an initial breakpoint to stop execution
             maxtries = 10
             bp_file = bp_file or filename
             checkline = deb.checkline(bp_file, bp_line)
             if not checkline:
                 for bp in range(bp_line + 1, bp_line + maxtries + 1):
                     if deb.checkline(bp_file, bp):
                         break
                 else:
                     msg = ("\nI failed to find a valid line to set "
                            "a breakpoint\n"
                            "after trying up to line: %s.\n"
                            "Please set a valid breakpoint manually "
                            "with the -b option." % bp)
                     raise UsageError(msg)
             # if we find a good linenumber, set the breakpoint
             deb.do_break('%s:%s' % (bp_file, bp_line))
 
         if filename:
             # Mimic Pdb._runscript(...)
             deb._wait_for_mainpyfile = True
             deb.mainpyfile = deb.canonic(filename)
 
         # Start file run
         print("NOTE: Enter 'c' at the %s prompt to continue execution." % deb.prompt)
         try:
             if filename:
                 # save filename so it can be used by methods on the deb object
                 deb._exec_filename = filename
             while True:
                 try:
                     trace = sys.gettrace()
                     deb.run(code, code_ns)
                 except Restart:
                     print("Restarting")
                     if filename:
                         deb._wait_for_mainpyfile = True
                         deb.mainpyfile = deb.canonic(filename)
                     continue
                 else:
                     break
                 finally:
                     sys.settrace(trace)
             
 
         except:
             etype, value, tb = sys.exc_info()
             # Skip three frames in the traceback: the %run one,
             # one inside bdb.py, and the command-line typed by the
             # user (run by exec in pdb itself).
             self.shell.InteractiveTB(etype, value, tb, tb_offset=3)
 
     @staticmethod
     def _run_with_timing(run, nruns):
         """
         Run function `run` and print timing information.
 
         Parameters
         ----------
         run : callable
             Any callable object which takes no argument.
         nruns : int
             Number of times to execute `run`.
 
         """
         twall0 = time.perf_counter()
         if nruns == 1:
             t0 = clock2()
             run()
             t1 = clock2()
             t_usr = t1[0] - t0[0]
             t_sys = t1[1] - t0[1]
             print("\nIPython CPU timings (estimated):")
             print("  User   : %10.2f s." % t_usr)
             print("  System : %10.2f s." % t_sys)
         else:
             runs = range(nruns)
             t0 = clock2()
             for nr in runs:
                 run()
             t1 = clock2()
             t_usr = t1[0] - t0[0]
             t_sys = t1[1] - t0[1]
             print("\nIPython CPU timings (estimated):")
             print("Total runs performed:", nruns)
             print("  Times  : %10s   %10s" % ('Total', 'Per run'))
             print("  User   : %10.2f s, %10.2f s." % (t_usr, t_usr / nruns))
             print("  System : %10.2f s, %10.2f s." % (t_sys, t_sys / nruns))
         twall1 = time.perf_counter()
         print("Wall time: %10.2f s." % (twall1 - twall0))
 
     @skip_doctest
     @no_var_expand
     @line_cell_magic
     @needs_local_scope
     def timeit(self, line='', cell=None, local_ns=None):
         """Time execution of a Python statement or expression
 
         Usage, in line mode:
           %timeit [-n<N> -r<R> [-t|-c] -q -p<P> -o] statement
         or in cell mode:
           %%timeit [-n<N> -r<R> [-t|-c] -q -p<P> -o] setup_code
           code
           code...
 
         Time execution of a Python statement or expression using the timeit
         module.  This function can be used both as a line and cell magic:
 
         - In line mode you can time a single-line statement (though multiple
           ones can be chained with using semicolons).
 
         - In cell mode, the statement in the first line is used as setup code
           (executed but not timed) and the body of the cell is timed.  The cell
           body has access to any variables created in the setup code.
 
         Options:
         -n<N>: execute the given statement <N> times in a loop. If <N> is not
         provided, <N> is determined so as to get sufficient accuracy.
 
         -r<R>: number of repeats <R>, each consisting of <N> loops, and take the
         best result.
         Default: 7
 
         -t: use time.time to measure the time, which is the default on Unix.
         This function measures wall time.
 
         -c: use time.clock to measure the time, which is the default on
         Windows and measures wall time. On Unix, resource.getrusage is used
         instead and returns the CPU user time.
 
         -p<P>: use a precision of <P> digits to display the timing result.
         Default: 3
 
         -q: Quiet, do not print result.
 
         -o: return a TimeitResult that can be stored in a variable to inspect
             the result in more details.
 
         .. versionchanged:: 7.3
             User variables are no longer expanded,
             the magic line is always left unmodified.
 
         Examples
         --------
         ::
 
           In [1]: %timeit pass
           8.26 ns ± 0.12 ns per loop (mean ± std. dev. of 7 runs, 100000000 loops each)
 
           In [2]: u = None
 
           In [3]: %timeit u is None
           29.9 ns ± 0.643 ns per loop (mean ± std. dev. of 7 runs, 10000000 loops each)
 
           In [4]: %timeit -r 4 u == None
 
           In [5]: import time
 
           In [6]: %timeit -n1 time.sleep(2)
 
         The times reported by %timeit will be slightly higher than those
         reported by the timeit.py script when variables are accessed. This is
         due to the fact that %timeit executes the statement in the namespace
         of the shell, compared with timeit.py, which uses a single setup
         statement to import function or create variables. Generally, the bias
         does not matter as long as results from timeit.py are not mixed with
         those from %timeit."""
 
         opts, stmt = self.parse_options(
             line, "n:r:tcp:qo", posix=False, strict=False, preserve_non_opts=True
         )
         if stmt == "" and cell is None:
             return
         
         timefunc = timeit.default_timer
         number = int(getattr(opts, "n", 0))
         default_repeat = 7 if timeit.default_repeat < 7 else timeit.default_repeat
         repeat = int(getattr(opts, "r", default_repeat))
         precision = int(getattr(opts, "p", 3))
         quiet = 'q' in opts
         return_result = 'o' in opts
         if hasattr(opts, "t"):
             timefunc = time.time
         if hasattr(opts, "c"):
             timefunc = clock
 
         timer = Timer(timer=timefunc)
         # this code has tight coupling to the inner workings of timeit.Timer,
         # but is there a better way to achieve that the code stmt has access
         # to the shell namespace?
         transform  = self.shell.transform_cell
 
         if cell is None:
             # called as line magic
             ast_setup = self.shell.compile.ast_parse("pass")
             ast_stmt = self.shell.compile.ast_parse(transform(stmt))
         else:
             ast_setup = self.shell.compile.ast_parse(transform(stmt))
             ast_stmt = self.shell.compile.ast_parse(transform(cell))
 
         ast_setup = self.shell.transform_ast(ast_setup)
         ast_stmt = self.shell.transform_ast(ast_stmt)
 
         # Check that these compile to valid Python code *outside* the timer func
         # Invalid code may become valid when put inside the function & loop,
         # which messes up error messages.
         # https://github.com/ipython/ipython/issues/10636
         self.shell.compile(ast_setup, "<magic-timeit-setup>", "exec")
         self.shell.compile(ast_stmt, "<magic-timeit-stmt>", "exec")
 
         # This codestring is taken from timeit.template - we fill it in as an
         # AST, so that we can apply our AST transformations to the user code
         # without affecting the timing code.
         timeit_ast_template = ast.parse('def inner(_it, _timer):\n'
                                         '    setup\n'
                                         '    _t0 = _timer()\n'
                                         '    for _i in _it:\n'
                                         '        stmt\n'
                                         '    _t1 = _timer()\n'
                                         '    return _t1 - _t0\n')
 
         timeit_ast = TimeitTemplateFiller(ast_setup, ast_stmt).visit(timeit_ast_template)
         timeit_ast = ast.fix_missing_locations(timeit_ast)
 
         # Track compilation time so it can be reported if too long
         # Minimum time above which compilation time will be reported
         tc_min = 0.1
 
         t0 = clock()
         code = self.shell.compile(timeit_ast, "<magic-timeit>", "exec")
         tc = clock()-t0
 
         ns = {}
         glob = self.shell.user_ns
         # handles global vars with same name as local vars. We store them in conflict_globs.
         conflict_globs = {}
         if local_ns and cell is None:
             for var_name, var_val in glob.items():
                 if var_name in local_ns:
                     conflict_globs[var_name] = var_val
             glob.update(local_ns)
             
         exec(code, glob, ns)
         timer.inner = ns["inner"]
 
         # This is used to check if there is a huge difference between the
         # best and worst timings.
         # Issue: https://github.com/ipython/ipython/issues/6471
         if number == 0:
             # determine number so that 0.2 <= total time < 2.0
             for index in range(0, 10):
                 number = 10 ** index
                 time_number = timer.timeit(number)
                 if time_number >= 0.2:
                     break
 
         all_runs = timer.repeat(repeat, number)
         best = min(all_runs) / number
         worst = max(all_runs) / number
         timeit_result = TimeitResult(number, repeat, best, worst, all_runs, tc, precision)
 
         # Restore global vars from conflict_globs
         if conflict_globs:
            glob.update(conflict_globs)
                 
         if not quiet :
             # Check best timing is greater than zero to avoid a
             # ZeroDivisionError.
             # In cases where the slowest timing is lesser than a microsecond
             # we assume that it does not really matter if the fastest
             # timing is 4 times faster than the slowest timing or not.
             if worst > 4 * best and best > 0 and worst > 1e-6:
                 print("The slowest run took %0.2f times longer than the "
                       "fastest. This could mean that an intermediate result "
                       "is being cached." % (worst / best))
            
             print( timeit_result )
 
             if tc > tc_min:
                 print("Compiler time: %.2f s" % tc)
         if return_result:
             return timeit_result
 
     @skip_doctest
     @no_var_expand
     @needs_local_scope
     @line_cell_magic
+    @output_can_be_silenced
     def time(self,line='', cell=None, local_ns=None):
         """Time execution of a Python statement or expression.
 
         The CPU and wall clock times are printed, and the value of the
         expression (if any) is returned.  Note that under Win32, system time
         is always reported as 0, since it can not be measured.
 
         This function can be used both as a line and cell magic:
 
         - In line mode you can time a single-line statement (though multiple
           ones can be chained with using semicolons).
 
         - In cell mode, you can time the cell body (a directly
           following statement raises an error).
 
         This function provides very basic timing functionality.  Use the timeit
         magic for more control over the measurement.
 
         .. versionchanged:: 7.3
             User variables are no longer expanded,
             the magic line is always left unmodified.
 
         Examples
         --------
         ::
 
           In [1]: %time 2**128
           CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
           Wall time: 0.00
           Out[1]: 340282366920938463463374607431768211456L
 
           In [2]: n = 1000000
 
           In [3]: %time sum(range(n))
           CPU times: user 1.20 s, sys: 0.05 s, total: 1.25 s
           Wall time: 1.37
           Out[3]: 499999500000L
 
           In [4]: %time print 'hello world'
           hello world
           CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
           Wall time: 0.00
 
         .. note::
             The time needed by Python to compile the given expression will be
             reported if it is more than 0.1s.
 
             In the example below, the actual exponentiation is done by Python
             at compilation time, so while the expression can take a noticeable
             amount of time to compute, that time is purely due to the
             compilation::
 
                 In [5]: %time 3**9999;
                 CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
                 Wall time: 0.00 s
 
                 In [6]: %time 3**999999;
                 CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
                 Wall time: 0.00 s
                 Compiler : 0.78 s
         """
         # fail immediately if the given expression can't be compiled
         
         if line and cell:
             raise UsageError("Can't use statement directly after '%%time'!")
         
         if cell:
             expr = self.shell.transform_cell(cell)
         else:
             expr = self.shell.transform_cell(line)
 
         # Minimum time above which parse time will be reported
         tp_min = 0.1
 
         t0 = clock()
         expr_ast = self.shell.compile.ast_parse(expr)
         tp = clock()-t0
 
         # Apply AST transformations
         expr_ast = self.shell.transform_ast(expr_ast)
 
         # Minimum time above which compilation time will be reported
         tc_min = 0.1
 
         expr_val=None
         if len(expr_ast.body)==1 and isinstance(expr_ast.body[0], ast.Expr):
             mode = 'eval'
             source = '<timed eval>'
             expr_ast = ast.Expression(expr_ast.body[0].value)
         else:
             mode = 'exec'
             source = '<timed exec>'
             # multi-line %%time case
             if len(expr_ast.body) > 1 and isinstance(expr_ast.body[-1], ast.Expr):
                 expr_val= expr_ast.body[-1]
                 expr_ast = expr_ast.body[:-1]
                 expr_ast = Module(expr_ast, [])
                 expr_val = ast.Expression(expr_val.value)
 
         t0 = clock()
         code = self.shell.compile(expr_ast, source, mode)
         tc = clock()-t0
 
         # skew measurement as little as possible
         glob = self.shell.user_ns
         wtime = time.time
         # time execution
         wall_st = wtime()
         if mode=='eval':
             st = clock2()
             try:
                 out = eval(code, glob, local_ns)
             except:
                 self.shell.showtraceback()
                 return
             end = clock2()
         else:
             st = clock2()
             try:
                 exec(code, glob, local_ns)
                 out=None
                 # multi-line %%time case
                 if expr_val is not None:
                     code_2 = self.shell.compile(expr_val, source, 'eval')
                     out = eval(code_2, glob, local_ns)
             except:
                 self.shell.showtraceback()
                 return
             end = clock2()
 
         wall_end = wtime()
         # Compute actual times and report
         wall_time = wall_end - wall_st
         cpu_user = end[0] - st[0]
         cpu_sys = end[1] - st[1]
         cpu_tot = cpu_user + cpu_sys
         # On windows cpu_sys is always zero, so only total is displayed
         if sys.platform != "win32":
             print(
                 f"CPU times: user {_format_time(cpu_user)}, sys: {_format_time(cpu_sys)}, total: {_format_time(cpu_tot)}"
             )
         else:
             print(f"CPU times: total: {_format_time(cpu_tot)}")
         print(f"Wall time: {_format_time(wall_time)}")
         if tc > tc_min:
             print(f"Compiler : {_format_time(tc)}")
         if tp > tp_min:
             print(f"Parser   : {_format_time(tp)}")
         return out
 
     @skip_doctest
     @line_magic
     def macro(self, parameter_s=''):
         """Define a macro for future re-execution. It accepts ranges of history,
         filenames or string objects.
 
         Usage:\\
           %macro [options] name n1-n2 n3-n4 ... n5 .. n6 ...
 
         Options:
 
           -r: use 'raw' input.  By default, the 'processed' history is used,
           so that magics are loaded in their transformed version to valid
           Python.  If this option is given, the raw input as typed at the
           command line is used instead.
           
           -q: quiet macro definition.  By default, a tag line is printed 
           to indicate the macro has been created, and then the contents of 
           the macro are printed.  If this option is given, then no printout
           is produced once the macro is created.
 
         This will define a global variable called `name` which is a string
         made of joining the slices and lines you specify (n1,n2,... numbers
         above) from your input history into a single string. This variable
         acts like an automatic function which re-executes those lines as if
         you had typed them. You just type 'name' at the prompt and the code
         executes.
 
         The syntax for indicating input ranges is described in %history.
 
         Note: as a 'hidden' feature, you can also use traditional python slice
         notation, where N:M means numbers N through M-1.
 
         For example, if your history contains (print using %hist -n )::
 
           44: x=1
           45: y=3
           46: z=x+y
           47: print x
           48: a=5
           49: print 'x',x,'y',y
 
         you can create a macro with lines 44 through 47 (included) and line 49
         called my_macro with::
 
           In [55]: %macro my_macro 44-47 49
 
         Now, typing `my_macro` (without quotes) will re-execute all this code
         in one pass.
 
         You don't need to give the line-numbers in order, and any given line
         number can appear multiple times. You can assemble macros with any
         lines from your input history in any order.
 
         The macro is a simple object which holds its value in an attribute,
         but IPython's display system checks for macros and executes them as
         code instead of printing them when you type their name.
 
         You can view a macro's contents by explicitly printing it with::
 
           print macro_name
 
         """
         opts,args = self.parse_options(parameter_s,'rq',mode='list')
         if not args:   # List existing macros
             return sorted(k for k,v in self.shell.user_ns.items() if isinstance(v, Macro))
         if len(args) == 1:
             raise UsageError(
                 "%macro insufficient args; usage '%macro name n1-n2 n3-4...")
         name, codefrom = args[0], " ".join(args[1:])
 
         #print 'rng',ranges  # dbg
         try:
             lines = self.shell.find_user_code(codefrom, 'r' in opts)
         except (ValueError, TypeError) as e:
             print(e.args[0])
             return
         macro = Macro(lines)
         self.shell.define_macro(name, macro)
         if not ( 'q' in opts) : 
             print('Macro `%s` created. To execute, type its name (without quotes).' % name)
             print('=== Macro contents: ===')
             print(macro, end=' ')
 
     @magic_arguments.magic_arguments()
     @magic_arguments.argument('output', type=str, default='', nargs='?',
         help="""The name of the variable in which to store output.
         This is a utils.io.CapturedIO object with stdout/err attributes
         for the text of the captured output.
 
         CapturedOutput also has a show() method for displaying the output,
         and __call__ as well, so you can use that to quickly display the
         output.
 
         If unspecified, captured output is discarded.
         """
     )
     @magic_arguments.argument('--no-stderr', action="store_true",
         help="""Don't capture stderr."""
     )
     @magic_arguments.argument('--no-stdout', action="store_true",
         help="""Don't capture stdout."""
     )
     @magic_arguments.argument('--no-display', action="store_true",
         help="""Don't capture IPython's rich display."""
     )
     @cell_magic
     def capture(self, line, cell):
         """run the cell, capturing stdout, stderr, and IPython's rich display() calls."""
         args = magic_arguments.parse_argstring(self.capture, line)
         out = not args.no_stdout
         err = not args.no_stderr
         disp = not args.no_display
         with capture_output(out, err, disp) as io:
             self.shell.run_cell(cell)
         if args.output:
             self.shell.user_ns[args.output] = io
 
 def parse_breakpoint(text, current_file):
     '''Returns (file, line) for file:line and (current_file, line) for line'''
     colon = text.find(':')
     if colon == -1:
         return current_file, int(text)
     else:
         return text[:colon], int(text[colon+1:])
     
 def _format_time(timespan, precision=3):
     """Formats the timespan in a human readable form"""
 
     if timespan >= 60.0:
         # we have more than a minute, format that in a human readable form
         # Idea from http://snipplr.com/view/5713/
         parts = [("d", 60*60*24),("h", 60*60),("min", 60), ("s", 1)]
         time = []
         leftover = timespan
         for suffix, length in parts:
             value = int(leftover / length)
             if value > 0:
                 leftover = leftover % length
                 time.append(u'%s%s' % (str(value), suffix))
             if leftover < 1:
                 break
         return " ".join(time)
 
     
     # Unfortunately the unicode 'micro' symbol can cause problems in
     # certain terminals.  
     # See bug: https://bugs.launchpad.net/ipython/+bug/348466
     # Try to prevent crashes by being more secure than it needs to
     # E.g. eclipse is able to print a µ, but has no sys.stdout.encoding set.
     units = [u"s", u"ms",u'us',"ns"] # the save value   
     if hasattr(sys.stdout, 'encoding') and sys.stdout.encoding:
         try:
             u'\xb5'.encode(sys.stdout.encoding)
             units = [u"s", u"ms",u'\xb5s',"ns"]
         except:
             pass
     scaling = [1, 1e3, 1e6, 1e9]
         
     if timespan > 0.0:
         order = min(-int(math.floor(math.log10(timespan)) // 3), 3)
     else:
         order = 3
     return u"%.*g %s" % (precision, timespan * scaling[order], units[order])

IPython/core/magics/osm.py

@@ -1,855 +1,855 b''
 """Implementation of magic functions for interaction with the OS.
 
 Note: this module is named 'osm' instead of 'os' to avoid a collision with the
 builtin.
 """
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 import io
 import os
 import pathlib
 import re
 import sys
 from pprint import pformat
 
 from IPython.core import magic_arguments
 from IPython.core import oinspect
 from IPython.core import page
 from IPython.core.alias import AliasError, Alias
 from IPython.core.error import UsageError
 from IPython.core.magic import  (
     Magics, compress_dhist, magics_class, line_magic, cell_magic, line_cell_magic
 )
 from IPython.testing.skipdoctest import skip_doctest
 from IPython.utils.openpy import source_to_unicode
 from IPython.utils.process import abbrev_cwd
 from IPython.utils.terminal import set_term_title
 from traitlets import Bool
 from warnings import warn
 
 
 @magics_class
 class OSMagics(Magics):
     """Magics to interact with the underlying OS (shell-type functionality).
     """
 
     cd_force_quiet = Bool(False,
         help="Force %cd magic to be quiet even if -q is not passed."
     ).tag(config=True)
 
     def __init__(self, shell=None, **kwargs):
 
         # Now define isexec in a cross platform manner.
         self.is_posix = False
         self.execre = None
         if os.name == 'posix':
             self.is_posix = True
         else:
             try:
                 winext = os.environ['pathext'].replace(';','|').replace('.','')
             except KeyError:
                 winext = 'exe|com|bat|py'
             try:
                 self.execre = re.compile(r'(.*)\.(%s)$' % winext,re.IGNORECASE)
             except re.error:
                 warn("Seems like your pathext environmental "
                      "variable is malformed. Please check it to "
                      "enable a proper handle of file extensions "
                      "managed for your system")
                 winext = 'exe|com|bat|py'
                 self.execre = re.compile(r'(.*)\.(%s)$' % winext,re.IGNORECASE)
 
         # call up the chain
         super().__init__(shell=shell, **kwargs)
 
 
     def _isexec_POSIX(self, file):
         """
         Test for executable on a POSIX system
         """
         if os.access(file.path, os.X_OK):
             # will fail on maxOS if access is not X_OK
             return file.is_file()
         return False
 
 
     
     def _isexec_WIN(self, file):
         """
         Test for executable file on non POSIX system
         """
         return file.is_file() and self.execre.match(file.name) is not None
 
     def isexec(self, file):
         """
         Test for executable file on non POSIX system
         """
         if self.is_posix:
             return self._isexec_POSIX(file)
         else:
             return self._isexec_WIN(file)
 
 
     @skip_doctest
     @line_magic
     def alias(self, parameter_s=''):
         """Define an alias for a system command.
 
         '%alias alias_name cmd' defines 'alias_name' as an alias for 'cmd'
 
         Then, typing 'alias_name params' will execute the system command 'cmd
         params' (from your underlying operating system).
 
         Aliases have lower precedence than magic functions and Python normal
         variables, so if 'foo' is both a Python variable and an alias, the
         alias can not be executed until 'del foo' removes the Python variable.
 
         You can use the %l specifier in an alias definition to represent the
         whole line when the alias is called.  For example::
 
           In [2]: alias bracket echo "Input in brackets: <%l>"
           In [3]: bracket hello world
           Input in brackets: <hello world>
 
         You can also define aliases with parameters using %s specifiers (one
         per parameter)::
 
           In [1]: alias parts echo first %s second %s
           In [2]: %parts A B
           first A second B
           In [3]: %parts A
           Incorrect number of arguments: 2 expected.
           parts is an alias to: 'echo first %s second %s'
 
         Note that %l and %s are mutually exclusive.  You can only use one or
         the other in your aliases.
 
         Aliases expand Python variables just like system calls using ! or !!
         do: all expressions prefixed with '$' get expanded.  For details of
         the semantic rules, see PEP-215:
         https://peps.python.org/pep-0215/.  This is the library used by
         IPython for variable expansion.  If you want to access a true shell
         variable, an extra $ is necessary to prevent its expansion by
         IPython::
 
           In [6]: alias show echo
           In [7]: PATH='A Python string'
           In [8]: show $PATH
           A Python string
           In [9]: show $$PATH
           /usr/local/lf9560/bin:/usr/local/intel/compiler70/ia32/bin:...
 
         You can use the alias facility to access all of $PATH.  See the %rehashx
         function, which automatically creates aliases for the contents of your
         $PATH.
 
         If called with no parameters, %alias prints the current alias table
         for your system.  For posix systems, the default aliases are 'cat',
         'cp', 'mv', 'rm', 'rmdir', and 'mkdir', and other platform-specific
         aliases are added.  For windows-based systems, the default aliases are
         'copy', 'ddir', 'echo', 'ls', 'ldir', 'mkdir', 'ren', and 'rmdir'.
 
         You can see the definition of alias by adding a question mark in the
         end::
 
           In [1]: cat?
           Repr: <alias cat for 'cat'>"""
 
         par = parameter_s.strip()
         if not par:
             aliases = sorted(self.shell.alias_manager.aliases)
             # stored = self.shell.db.get('stored_aliases', {} )
             # for k, v in stored:
             #     atab.append(k, v[0])
 
             print("Total number of aliases:", len(aliases))
             sys.stdout.flush()
             return aliases
 
         # Now try to define a new one
         try:
             alias,cmd = par.split(None, 1)
         except TypeError:
             print(oinspect.getdoc(self.alias))
             return
         
         try:
             self.shell.alias_manager.define_alias(alias, cmd)
         except AliasError as e:
             print(e)
     # end magic_alias
 
     @line_magic
     def unalias(self, parameter_s=''):
         """Remove an alias"""
 
         aname = parameter_s.strip()
         try:
             self.shell.alias_manager.undefine_alias(aname)
         except ValueError as e:
             print(e)
             return
         
         stored = self.shell.db.get('stored_aliases', {} )
         if aname in stored:
             print("Removing %stored alias",aname)
             del stored[aname]
             self.shell.db['stored_aliases'] = stored
 
     @line_magic
     def rehashx(self, parameter_s=''):
         """Update the alias table with all executable files in $PATH.
 
         rehashx explicitly checks that every entry in $PATH is a file
         with execute access (os.X_OK).
 
         Under Windows, it checks executability as a match against a
         '|'-separated string of extensions, stored in the IPython config
         variable win_exec_ext.  This defaults to 'exe|com|bat'.
 
         This function also resets the root module cache of module completer,
         used on slow filesystems.
         """
         from IPython.core.alias import InvalidAliasError
 
         # for the benefit of module completer in ipy_completers.py
         del self.shell.db['rootmodules_cache']
 
         path = [os.path.abspath(os.path.expanduser(p)) for p in
             os.environ.get('PATH','').split(os.pathsep)]
 
         syscmdlist = []
         savedir = os.getcwd()
 
         # Now walk the paths looking for executables to alias.
         try:
             # write the whole loop for posix/Windows so we don't have an if in
             # the innermost part
             if self.is_posix:
                 for pdir in path:
                     try:
                         os.chdir(pdir)
                     except OSError:
                         continue
 
                     # for python 3.6+ rewrite to: with os.scandir(pdir) as dirlist:
                     dirlist = os.scandir(path=pdir)
                     for ff in dirlist:
                         if self.isexec(ff):
                             fname = ff.name
                             try:
                                 # Removes dots from the name since ipython
                                 # will assume names with dots to be python.
                                 if not self.shell.alias_manager.is_alias(fname):
                                     self.shell.alias_manager.define_alias(
                                         fname.replace('.',''), fname)
                             except InvalidAliasError:
                                 pass
                             else:
                                 syscmdlist.append(fname)
             else:
                 no_alias = Alias.blacklist
                 for pdir in path:
                     try:
                         os.chdir(pdir)
                     except OSError:
                         continue
 
                     # for python 3.6+ rewrite to: with os.scandir(pdir) as dirlist:
                     dirlist = os.scandir(pdir)
                     for ff in dirlist:
                         fname = ff.name
                         base, ext = os.path.splitext(fname)
                         if self.isexec(ff) and base.lower() not in no_alias:
                             if ext.lower() == '.exe':
                                 fname = base
                                 try:
                                     # Removes dots from the name since ipython
                                     # will assume names with dots to be python.
                                     self.shell.alias_manager.define_alias(
                                         base.lower().replace('.',''), fname)
                                 except InvalidAliasError:
                                     pass
                                 syscmdlist.append(fname)
 
             self.shell.db['syscmdlist'] = syscmdlist
         finally:
             os.chdir(savedir)
 
     @skip_doctest
     @line_magic
     def pwd(self, parameter_s=''):
         """Return the current working directory path.
 
         Examples
         --------
         ::
 
           In [9]: pwd
           Out[9]: '/home/tsuser/sprint/ipython'
         """
         try:
             return os.getcwd()
         except FileNotFoundError as e:
             raise UsageError("CWD no longer exists - please use %cd to change directory.") from e
 
     @skip_doctest
     @line_magic
     def cd(self, parameter_s=''):
         """Change the current working directory.
 
         This command automatically maintains an internal list of directories
         you visit during your IPython session, in the variable ``_dh``. The
         command :magic:`%dhist` shows this history nicely formatted. You can
         also do ``cd -<tab>`` to see directory history conveniently.
         Usage:
 
           - ``cd 'dir'``: changes to directory 'dir'.
           - ``cd -``: changes to the last visited directory.
           - ``cd -<n>``: changes to the n-th directory in the directory history.
           - ``cd --foo``: change to directory that matches 'foo' in history
           - ``cd -b <bookmark_name>``: jump to a bookmark set by %bookmark
           - Hitting a tab key after ``cd -b`` allows you to tab-complete
             bookmark names.
 
           .. note::
             ``cd <bookmark_name>`` is enough if there is no directory
             ``<bookmark_name>``, but a bookmark with the name exists.
 
         Options:
 
         -q               Be quiet. Do not print the working directory after the
                          cd command is executed. By default IPython's cd
                          command does print this directory, since the default
                          prompts do not display path information.
 
         .. note::
            Note that ``!cd`` doesn't work for this purpose because the shell
            where ``!command`` runs is immediately discarded after executing
            'command'.
 
         Examples
         --------
         ::
 
           In [10]: cd parent/child
           /home/tsuser/parent/child
         """
 
         try:
             oldcwd = os.getcwd()
         except FileNotFoundError:
             # Happens if the CWD has been deleted.
             oldcwd = None
 
         numcd = re.match(r'(-)(\d+)$',parameter_s)
         # jump in directory history by number
         if numcd:
             nn = int(numcd.group(2))
             try:
                 ps = self.shell.user_ns['_dh'][nn]
             except IndexError:
                 print('The requested directory does not exist in history.')
                 return
             else:
                 opts = {}
         elif parameter_s.startswith('--'):
             ps = None
             fallback = None
             pat = parameter_s[2:]
             dh = self.shell.user_ns['_dh']
             # first search only by basename (last component)
             for ent in reversed(dh):
                 if pat in os.path.basename(ent) and os.path.isdir(ent):
                     ps = ent
                     break
 
                 if fallback is None and pat in ent and os.path.isdir(ent):
                     fallback = ent
 
             # if we have no last part match, pick the first full path match
             if ps is None:
                 ps = fallback
 
             if ps is None:
                 print("No matching entry in directory history")
                 return
             else:
                 opts = {}
 
 
         else:
             opts, ps = self.parse_options(parameter_s, 'qb', mode='string')
         # jump to previous
         if ps == '-':
             try:
                 ps = self.shell.user_ns['_dh'][-2]
             except IndexError as e:
                 raise UsageError('%cd -: No previous directory to change to.') from e
         # jump to bookmark if needed
         else:
             if not os.path.isdir(ps) or 'b' in opts:
                 bkms = self.shell.db.get('bookmarks', {})
 
                 if ps in bkms:
                     target = bkms[ps]
                     print('(bookmark:%s) -> %s' % (ps, target))
                     ps = target
                 else:
                     if 'b' in opts:
                         raise UsageError("Bookmark '%s' not found.  "
                               "Use '%%bookmark -l' to see your bookmarks." % ps)
 
         # at this point ps should point to the target dir
         if ps:
             try:
                 os.chdir(os.path.expanduser(ps))
                 if hasattr(self.shell, 'term_title') and self.shell.term_title:
                     set_term_title(self.shell.term_title_format.format(cwd=abbrev_cwd()))
             except OSError:
                 print(sys.exc_info()[1])
             else:
                 cwd = pathlib.Path.cwd()
                 dhist = self.shell.user_ns['_dh']
                 if oldcwd != cwd:
                     dhist.append(cwd)
                     self.shell.db['dhist'] = compress_dhist(dhist)[-100:]
 
         else:
             os.chdir(self.shell.home_dir)
             if hasattr(self.shell, 'term_title') and self.shell.term_title:
                 set_term_title(self.shell.term_title_format.format(cwd="~"))
             cwd = pathlib.Path.cwd()
             dhist = self.shell.user_ns['_dh']
 
             if oldcwd != cwd:
                 dhist.append(cwd)
                 self.shell.db['dhist'] = compress_dhist(dhist)[-100:]
         if not 'q' in opts and not self.cd_force_quiet and self.shell.user_ns['_dh']:
             print(self.shell.user_ns['_dh'][-1])
 
     @line_magic
     def env(self, parameter_s=''):
         """Get, set, or list environment variables.
 
         Usage:\\
 
           :``%env``: lists all environment variables/values
           :``%env var``: get value for var
           :``%env var val``: set value for var
           :``%env var=val``: set value for var
           :``%env var=$val``: set value for var, using python expansion if possible
         """
         if parameter_s.strip():
             split = '=' if '=' in parameter_s else ' '
             bits = parameter_s.split(split)
             if len(bits) == 1:
                 key = parameter_s.strip()
                 if key in os.environ:
                     return os.environ[key]
                 else:
                     err = "Environment does not have key: {0}".format(key)
                     raise UsageError(err)
             if len(bits) > 1:
                 return self.set_env(parameter_s)
         env = dict(os.environ)
         # hide likely secrets when printing the whole environment
         for key in list(env):
             if any(s in key.lower() for s in ('key', 'token', 'secret')):
                 env[key] = '<hidden>'
 
         return env
 
     @line_magic
     def set_env(self, parameter_s):
         """Set environment variables.  Assumptions are that either "val" is a
         name in the user namespace, or val is something that evaluates to a
         string.
 
         Usage:\\
-          %set_env var val: set value for var
-          %set_env var=val: set value for var
-          %set_env var=$val: set value for var, using python expansion if possible
+          :``%set_env var val``: set value for var
+          :``%set_env var=val``: set value for var
+          :``%set_env var=$val``: set value for var, using python expansion if possible
         """
         split = '=' if '=' in parameter_s else ' '
         bits = parameter_s.split(split, 1)
         if not parameter_s.strip() or len(bits)<2:
             raise UsageError("usage is 'set_env var=val'")
         var = bits[0].strip()
         val = bits[1].strip()
         if re.match(r'.*\s.*', var):
             # an environment variable with whitespace is almost certainly
             # not what the user intended.  what's more likely is the wrong
             # split was chosen, ie for "set_env cmd_args A=B", we chose
             # '=' for the split and should have chosen ' '.  to get around
             # this, users should just assign directly to os.environ or use
             # standard magic {var} expansion.
             err = "refusing to set env var with whitespace: '{0}'"
             err = err.format(val)
             raise UsageError(err)
         os.environ[var] = val
         print('env: {0}={1}'.format(var,val))
 
     @line_magic
     def pushd(self, parameter_s=''):
         """Place the current dir on stack and change directory.
 
         Usage:\\
           %pushd ['dirname']
         """
 
         dir_s = self.shell.dir_stack
         tgt = os.path.expanduser(parameter_s)
         cwd = os.getcwd().replace(self.shell.home_dir,'~')
         if tgt:
             self.cd(parameter_s)
         dir_s.insert(0,cwd)
         return self.shell.run_line_magic('dirs', '')
 
     @line_magic
     def popd(self, parameter_s=''):
         """Change to directory popped off the top of the stack.
         """
         if not self.shell.dir_stack:
             raise UsageError("%popd on empty stack")
         top = self.shell.dir_stack.pop(0)
         self.cd(top)
         print("popd ->",top)
 
     @line_magic
     def dirs(self, parameter_s=''):
         """Return the current directory stack."""
 
         return self.shell.dir_stack
 
     @line_magic
     def dhist(self, parameter_s=''):
         """Print your history of visited directories.
 
         %dhist       -> print full history\\
         %dhist n     -> print last n entries only\\
         %dhist n1 n2 -> print entries between n1 and n2 (n2 not included)\\
 
         This history is automatically maintained by the %cd command, and
         always available as the global list variable _dh. You can use %cd -<n>
         to go to directory number <n>.
 
         Note that most of time, you should view directory history by entering
         cd -<TAB>.
 
         """
 
         dh = self.shell.user_ns['_dh']
         if parameter_s:
             try:
                 args = map(int,parameter_s.split())
             except:
                 self.arg_err(self.dhist)
                 return
             if len(args) == 1:
                 ini,fin = max(len(dh)-(args[0]),0),len(dh)
             elif len(args) == 2:
                 ini,fin = args
                 fin = min(fin, len(dh))
             else:
                 self.arg_err(self.dhist)
                 return
         else:
             ini,fin = 0,len(dh)
         print('Directory history (kept in _dh)')
         for i in range(ini, fin):
             print("%d: %s" % (i, dh[i]))
 
     @skip_doctest
     @line_magic
     def sc(self, parameter_s=''):
         """Shell capture - run shell command and capture output (DEPRECATED use !).
 
         DEPRECATED. Suboptimal, retained for backwards compatibility.
 
         You should use the form 'var = !command' instead. Example:
 
          "%sc -l myfiles = ls ~" should now be written as
 
          "myfiles = !ls ~"
 
         myfiles.s, myfiles.l and myfiles.n still apply as documented
         below.
 
         --
         %sc [options] varname=command
 
         IPython will run the given command using commands.getoutput(), and
         will then update the user's interactive namespace with a variable
         called varname, containing the value of the call.  Your command can
         contain shell wildcards, pipes, etc.
 
         The '=' sign in the syntax is mandatory, and the variable name you
         supply must follow Python's standard conventions for valid names.
 
         (A special format without variable name exists for internal use)
 
         Options:
 
           -l: list output.  Split the output on newlines into a list before
           assigning it to the given variable.  By default the output is stored
           as a single string.
 
           -v: verbose.  Print the contents of the variable.
 
         In most cases you should not need to split as a list, because the
         returned value is a special type of string which can automatically
         provide its contents either as a list (split on newlines) or as a
         space-separated string.  These are convenient, respectively, either
         for sequential processing or to be passed to a shell command.
 
         For example::
 
             # Capture into variable a
             In [1]: sc a=ls *py
 
             # a is a string with embedded newlines
             In [2]: a
             Out[2]: 'setup.py\\nwin32_manual_post_install.py'
 
             # which can be seen as a list:
             In [3]: a.l
             Out[3]: ['setup.py', 'win32_manual_post_install.py']
 
             # or as a whitespace-separated string:
             In [4]: a.s
             Out[4]: 'setup.py win32_manual_post_install.py'
 
             # a.s is useful to pass as a single command line:
             In [5]: !wc -l $a.s
               146 setup.py
               130 win32_manual_post_install.py
               276 total
 
             # while the list form is useful to loop over:
             In [6]: for f in a.l:
                ...:      !wc -l $f
                ...:
             146 setup.py
             130 win32_manual_post_install.py
 
         Similarly, the lists returned by the -l option are also special, in
         the sense that you can equally invoke the .s attribute on them to
         automatically get a whitespace-separated string from their contents::
 
             In [7]: sc -l b=ls *py
 
             In [8]: b
             Out[8]: ['setup.py', 'win32_manual_post_install.py']
 
             In [9]: b.s
             Out[9]: 'setup.py win32_manual_post_install.py'
 
         In summary, both the lists and strings used for output capture have
         the following special attributes::
 
             .l (or .list) : value as list.
             .n (or .nlstr): value as newline-separated string.
             .s (or .spstr): value as space-separated string.
         """
 
         opts,args = self.parse_options(parameter_s, 'lv')
         # Try to get a variable name and command to run
         try:
             # the variable name must be obtained from the parse_options
             # output, which uses shlex.split to strip options out.
             var,_ = args.split('=', 1)
             var = var.strip()
             # But the command has to be extracted from the original input
             # parameter_s, not on what parse_options returns, to avoid the
             # quote stripping which shlex.split performs on it.
             _,cmd = parameter_s.split('=', 1)
         except ValueError:
             var,cmd = '',''
         # If all looks ok, proceed
         split = 'l' in opts
         out = self.shell.getoutput(cmd, split=split)
         if 'v' in opts:
             print('%s ==\n%s' % (var, pformat(out)))
         if var:
             self.shell.user_ns.update({var:out})
         else:
             return out
 
     @line_cell_magic
     def sx(self, line='', cell=None):
         """Shell execute - run shell command and capture output (!! is short-hand).
 
         %sx command
 
         IPython will run the given command using commands.getoutput(), and
         return the result formatted as a list (split on '\\n').  Since the
         output is _returned_, it will be stored in ipython's regular output
         cache Out[N] and in the '_N' automatic variables.
 
         Notes:
 
         1) If an input line begins with '!!', then %sx is automatically
         invoked.  That is, while::
 
           !ls
 
         causes ipython to simply issue system('ls'), typing::
 
           !!ls
 
         is a shorthand equivalent to::
 
           %sx ls
 
         2) %sx differs from %sc in that %sx automatically splits into a list,
         like '%sc -l'.  The reason for this is to make it as easy as possible
         to process line-oriented shell output via further python commands.
         %sc is meant to provide much finer control, but requires more
         typing.
 
         3) Just like %sc -l, this is a list with special attributes:
         ::
 
           .l (or .list) : value as list.
           .n (or .nlstr): value as newline-separated string.
           .s (or .spstr): value as whitespace-separated string.
 
         This is very useful when trying to use such lists as arguments to
         system commands."""
         
         if cell is None:
             # line magic
             return self.shell.getoutput(line)
         else:
             opts,args = self.parse_options(line, '', 'out=')
             output = self.shell.getoutput(cell)
             out_name = opts.get('out', opts.get('o'))
             if out_name:
                 self.shell.user_ns[out_name] = output
             else:
                 return output
 
     system = line_cell_magic('system')(sx)
     bang = cell_magic('!')(sx)
 
     @line_magic
     def bookmark(self, parameter_s=''):
         """Manage IPython's bookmark system.
 
         %bookmark <name>       - set bookmark to current dir
         %bookmark <name> <dir> - set bookmark to <dir>
         %bookmark -l           - list all bookmarks
         %bookmark -d <name>    - remove bookmark
         %bookmark -r           - remove all bookmarks
 
         You can later on access a bookmarked folder with::
 
           %cd -b <name>
 
         or simply '%cd <name>' if there is no directory called <name> AND
         there is such a bookmark defined.
 
         Your bookmarks persist through IPython sessions, but they are
         associated with each profile."""
 
         opts,args = self.parse_options(parameter_s,'drl',mode='list')
         if len(args) > 2:
             raise UsageError("%bookmark: too many arguments")
 
         bkms = self.shell.db.get('bookmarks',{})
 
         if 'd' in opts:
             try:
                 todel = args[0]
             except IndexError as e:
                 raise UsageError(
                     "%bookmark -d: must provide a bookmark to delete") from e
             else:
                 try:
                     del bkms[todel]
                 except KeyError as e:
                     raise UsageError(
                         "%%bookmark -d: Can't delete bookmark '%s'" % todel) from e
 
         elif 'r' in opts:
             bkms = {}
         elif 'l' in opts:
             bks = sorted(bkms)
             if bks:
                 size = max(map(len, bks))
             else:
                 size = 0
             fmt = '%-'+str(size)+'s -> %s'
             print('Current bookmarks:')
             for bk in bks:
                 print(fmt % (bk, bkms[bk]))
         else:
             if not args:
                 raise UsageError("%bookmark: You must specify the bookmark name")
             elif len(args)==1:
                 bkms[args[0]] = os.getcwd()
             elif len(args)==2:
                 bkms[args[0]] = args[1]
         self.shell.db['bookmarks'] = bkms
 
     @line_magic
     def pycat(self, parameter_s=''):
         """Show a syntax-highlighted file through a pager.
 
         This magic is similar to the cat utility, but it will assume the file
         to be Python source and will show it with syntax highlighting.
 
         This magic command can either take a local filename, an url,
         an history range (see %history) or a macro as argument.
 
         If no parameter is given, prints out history of current session up to
         this point. ::
 
         %pycat myscript.py
         %pycat 7-27
         %pycat myMacro
         %pycat http://www.example.com/myscript.py
         """
         try:
             cont = self.shell.find_user_code(parameter_s, skip_encoding_cookie=False)
         except (ValueError, IOError):
             print("Error: no such file, variable, URL, history range or macro")
             return
 
         page.page(self.shell.pycolorize(source_to_unicode(cont)))
 
     @magic_arguments.magic_arguments()
     @magic_arguments.argument(
         '-a', '--append', action='store_true', default=False,
         help='Append contents of the cell to an existing file. '
              'The file will be created if it does not exist.'
     )
     @magic_arguments.argument(
         'filename', type=str,
         help='file to write'
     )
     @cell_magic
     def writefile(self, line, cell):
         """Write the contents of the cell to a file.
 
         The file will be overwritten unless the -a (--append) flag is specified.
         """
         args = magic_arguments.parse_argstring(self.writefile, line)
         if re.match(r'^(\'.*\')|(".*")$', args.filename):
             filename = os.path.expanduser(args.filename[1:-1])
         else:
             filename = os.path.expanduser(args.filename)
             
         if os.path.exists(filename):
             if args.append:
                 print("Appending to %s" % filename)
             else:
                 print("Overwriting %s" % filename)
         else:
             print("Writing %s" % filename)
         
         mode = 'a' if args.append else 'w'
         with io.open(filename, mode, encoding='utf-8') as f:
             f.write(cell)

IPython/core/magics/pylab.py

@@ -1,169 +1,169 b''
 """Implementation of magic functions for matplotlib/pylab support.
 """
 #-----------------------------------------------------------------------------
 #  Copyright (c) 2012 The IPython Development Team.
 #
 #  Distributed under the terms of the Modified BSD License.
 #
 #  The full license is in the file COPYING.txt, distributed with this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 # Our own packages
 from traitlets.config.application import Application
 from IPython.core import magic_arguments
 from IPython.core.magic import Magics, magics_class, line_magic
 from IPython.testing.skipdoctest import skip_doctest
 from warnings import warn
 from IPython.core.pylabtools import backends
 
 #-----------------------------------------------------------------------------
 # Magic implementation classes
 #-----------------------------------------------------------------------------
 
 magic_gui_arg = magic_arguments.argument(
         'gui', nargs='?',
         help="""Name of the matplotlib backend to use %s.
         If given, the corresponding matplotlib backend is used,
         otherwise it will be matplotlib's default
         (which you can set in your matplotlib config file).
         """ % str(tuple(sorted(backends.keys())))
 )
 
 
 @magics_class
 class PylabMagics(Magics):
     """Magics related to matplotlib's pylab support"""
 
     @skip_doctest
     @line_magic
     @magic_arguments.magic_arguments()
     @magic_arguments.argument('-l', '--list', action='store_true',
                               help='Show available matplotlib backends')
     @magic_gui_arg
     def matplotlib(self, line=''):
         """Set up matplotlib to work interactively.
 
         This function lets you activate matplotlib interactive support
         at any point during an IPython session. It does not import anything
         into the interactive namespace.
 
         If you are using the inline matplotlib backend in the IPython Notebook
         you can set which figure formats are enabled using the following::
 
-            In [1]: from IPython.display import set_matplotlib_formats
+            In [1]: from matplotlib_inline.backend_inline import set_matplotlib_formats
 
             In [2]: set_matplotlib_formats('pdf', 'svg')
 
         The default for inline figures sets `bbox_inches` to 'tight'. This can
         cause discrepancies between the displayed image and the identical
         image created using `savefig`. This behavior can be disabled using the
         `%config` magic::
 
             In [3]: %config InlineBackend.print_figure_kwargs = {'bbox_inches':None}
 
-        In addition, see the docstring of
-        `IPython.display.set_matplotlib_formats` and
-        `IPython.display.set_matplotlib_close` for more information on
+        In addition, see the docstrings of
+        `matplotlib_inline.backend_inline.set_matplotlib_formats` and
+        `matplotlib_inline.backend_inline.set_matplotlib_close` for more information on
         changing additional behaviors of the inline backend.
 
         Examples
         --------
         To enable the inline backend for usage with the IPython Notebook::
 
             In [1]: %matplotlib inline
 
         In this case, where the matplotlib default is TkAgg::
 
             In [2]: %matplotlib
             Using matplotlib backend: TkAgg
 
         But you can explicitly request a different GUI backend::
 
             In [3]: %matplotlib qt
 
         You can list the available backends using the -l/--list option::
 
            In [4]: %matplotlib --list
            Available matplotlib backends: ['osx', 'qt4', 'qt5', 'gtk3', 'gtk4', 'notebook', 'wx', 'qt', 'nbagg',
            'gtk', 'tk', 'inline']
         """
         args = magic_arguments.parse_argstring(self.matplotlib, line)
         if args.list:
             backends_list = list(backends.keys())
             print("Available matplotlib backends: %s" % backends_list)
         else:
             gui, backend = self.shell.enable_matplotlib(args.gui.lower() if isinstance(args.gui, str) else args.gui)
             self._show_matplotlib_backend(args.gui, backend)
 
     @skip_doctest
     @line_magic
     @magic_arguments.magic_arguments()
     @magic_arguments.argument(
         '--no-import-all', action='store_true', default=None,
         help="""Prevent IPython from performing ``import *`` into the interactive namespace.
 
         You can govern the default behavior of this flag with the
         InteractiveShellApp.pylab_import_all configurable.
         """
     )
     @magic_gui_arg
     def pylab(self, line=''):
         """Load numpy and matplotlib to work interactively.
 
         This function lets you activate pylab (matplotlib, numpy and
         interactive support) at any point during an IPython session.
 
         %pylab makes the following imports::
 
             import numpy
             import matplotlib
             from matplotlib import pylab, mlab, pyplot
             np = numpy
             plt = pyplot
 
             from IPython.display import display
             from IPython.core.pylabtools import figsize, getfigs
 
             from pylab import *
             from numpy import *
 
         If you pass `--no-import-all`, the last two `*` imports will be excluded.
 
         See the %matplotlib magic for more details about activating matplotlib
         without affecting the interactive namespace.
         """
         args = magic_arguments.parse_argstring(self.pylab, line)
         if args.no_import_all is None:
             # get default from Application
             if Application.initialized():
                 app = Application.instance()
                 try:
                     import_all = app.pylab_import_all
                 except AttributeError:
                     import_all = True
             else:
                 # nothing specified, no app - default True
                 import_all = True
         else:
             # invert no-import flag
             import_all = not args.no_import_all
 
         gui, backend, clobbered = self.shell.enable_pylab(args.gui, import_all=import_all)
         self._show_matplotlib_backend(args.gui, backend)
         print(
             "%pylab is deprecated, use %matplotlib inline and import the required libraries."
         )
         print("Populating the interactive namespace from numpy and matplotlib")
         if clobbered:
             warn("pylab import has clobbered these variables: %s"  % clobbered +
             "\n`%matplotlib` prevents importing * from pylab and numpy"
             )
 
     def _show_matplotlib_backend(self, gui, backend):
         """show matplotlib message backend message"""
         if not gui or gui == 'auto':
             print("Using matplotlib backend: %s" % backend)

IPython/core/magics/script.py

@@ -1,362 +1,362 b''
 """Magic functions for running cells in various scripts."""
 
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 import asyncio
 import atexit
 import errno
 import os
 import signal
 import sys
 import time
 from subprocess import CalledProcessError
 from threading import Thread
 
 from traitlets import Any, Dict, List, default
 
 from IPython.core import magic_arguments
 from IPython.core.async_helpers import _AsyncIOProxy
 from IPython.core.magic import Magics, cell_magic, line_magic, magics_class
 from IPython.utils.process import arg_split
 
 #-----------------------------------------------------------------------------
 # Magic implementation classes
 #-----------------------------------------------------------------------------
 
 def script_args(f):
     """single decorator for adding script args"""
     args = [
         magic_arguments.argument(
             '--out', type=str,
             help="""The variable in which to store stdout from the script.
             If the script is backgrounded, this will be the stdout *pipe*,
             instead of the stderr text itself and will not be auto closed.
             """
         ),
         magic_arguments.argument(
             '--err', type=str,
             help="""The variable in which to store stderr from the script.
             If the script is backgrounded, this will be the stderr *pipe*,
             instead of the stderr text itself and will not be autoclosed.
             """
         ),
         magic_arguments.argument(
             '--bg', action="store_true",
             help="""Whether to run the script in the background.
             If given, the only way to see the output of the command is
             with --out/err.
             """
         ),
         magic_arguments.argument(
             '--proc', type=str,
             help="""The variable in which to store Popen instance.
             This is used only when --bg option is given.
             """
         ),
         magic_arguments.argument(
             '--no-raise-error', action="store_false", dest='raise_error',
             help="""Whether you should raise an error message in addition to
             a stream on stderr if you get a nonzero exit code.
             """,
         ),
     ]
     for arg in args:
         f = arg(f)
     return f
 
 
 @magics_class
 class ScriptMagics(Magics):
     """Magics for talking to scripts
     
     This defines a base `%%script` cell magic for running a cell
     with a program in a subprocess, and registers a few top-level
     magics that call %%script with common interpreters.
     """
 
     event_loop = Any(
         help="""
         The event loop on which to run subprocesses
 
         Not the main event loop,
         because we want to be able to make blocking calls
         and have certain requirements we don't want to impose on the main loop.
         """
     )
 
     script_magics = List(
         help="""Extra script cell magics to define
         
         This generates simple wrappers of `%%script foo` as `%%foo`.
         
         If you want to add script magics that aren't on your path,
         specify them in script_paths
         """,
     ).tag(config=True)
     @default('script_magics')
     def _script_magics_default(self):
         """default to a common list of programs"""
         
         defaults = [
             'sh',
             'bash',
             'perl',
             'ruby',
             'python',
             'python2',
             'python3',
             'pypy',
         ]
         if os.name == 'nt':
             defaults.extend([
                 'cmd',
             ])
         
         return defaults
     
     script_paths = Dict(
         help="""Dict mapping short 'ruby' names to full paths, such as '/opt/secret/bin/ruby'
         
         Only necessary for items in script_magics where the default path will not
         find the right interpreter.
         """
     ).tag(config=True)
     
     def __init__(self, shell=None):
         super(ScriptMagics, self).__init__(shell=shell)
         self._generate_script_magics()
         self.bg_processes = []
         atexit.register(self.kill_bg_processes)
 
     def __del__(self):
         self.kill_bg_processes()
     
     def _generate_script_magics(self):
         cell_magics = self.magics['cell']
         for name in self.script_magics:
             cell_magics[name] = self._make_script_magic(name)
     
     def _make_script_magic(self, name):
         """make a named magic, that calls %%script with a particular program"""
         # expand to explicit path if necessary:
         script = self.script_paths.get(name, name)
         
         @magic_arguments.magic_arguments()
         @script_args
         def named_script_magic(line, cell):
             # if line, add it as cl-flags
             if line:
                 line = "%s %s" % (script, line)
             else:
                 line = script
             return self.shebang(line, cell)
         
         # write a basic docstring:
         named_script_magic.__doc__ = \
         """%%{name} script magic
         
         Run cells with {script} in a subprocess.
         
         This is a shortcut for `%%script {script}`
         """.format(**locals())
         
         return named_script_magic
     
     @magic_arguments.magic_arguments()
     @script_args
     @cell_magic("script")
     def shebang(self, line, cell):
         """Run a cell via a shell command
 
         The `%%script` line is like the #! line of script,
         specifying a program (bash, perl, ruby, etc.) with which to run.
 
         The rest of the cell is run by that program.
 
         Examples
         --------
         ::
 
             In [1]: %%script bash
                ...: for i in 1 2 3; do
                ...:   echo $i
                ...: done
             1
             2
             3
         """
 
         # Create the event loop in which to run script magics
         # this operates on a background thread
         if self.event_loop is None:
             if sys.platform == "win32":
                 # don't override the current policy,
                 # just create an event loop
                 event_loop = asyncio.WindowsProactorEventLoopPolicy().new_event_loop()
             else:
                 event_loop = asyncio.new_event_loop()
             self.event_loop = event_loop
 
             # start the loop in a background thread
             asyncio_thread = Thread(target=event_loop.run_forever, daemon=True)
             asyncio_thread.start()
         else:
             event_loop = self.event_loop
 
         def in_thread(coro):
             """Call a coroutine on the asyncio thread"""
             return asyncio.run_coroutine_threadsafe(coro, event_loop).result()
 
         async def _handle_stream(stream, stream_arg, file_object):
             while True:
-                line = (await stream.readline()).decode("utf8")
+                line = (await stream.readline()).decode("utf8", errors="replace")
                 if not line:
                     break
                 if stream_arg:
                     self.shell.user_ns[stream_arg] = line
                 else:
                     file_object.write(line)
                     file_object.flush()
 
         async def _stream_communicate(process, cell):
             process.stdin.write(cell)
             process.stdin.close()
             stdout_task = asyncio.create_task(
                 _handle_stream(process.stdout, args.out, sys.stdout)
             )
             stderr_task = asyncio.create_task(
                 _handle_stream(process.stderr, args.err, sys.stderr)
             )
             await asyncio.wait([stdout_task, stderr_task])
             await process.wait()
 
         argv = arg_split(line, posix=not sys.platform.startswith("win"))
         args, cmd = self.shebang.parser.parse_known_args(argv)
 
         try:
             p = in_thread(
                 asyncio.create_subprocess_exec(
                     *cmd,
                     stdout=asyncio.subprocess.PIPE,
                     stderr=asyncio.subprocess.PIPE,
                     stdin=asyncio.subprocess.PIPE,
                 )
             )
         except OSError as e:
             if e.errno == errno.ENOENT:
                 print("Couldn't find program: %r" % cmd[0])
                 return
             else:
                 raise
 
         if not cell.endswith('\n'):
             cell += '\n'
         cell = cell.encode('utf8', 'replace')
         if args.bg:
             self.bg_processes.append(p)
             self._gc_bg_processes()
             to_close = []
             if args.out:
                 self.shell.user_ns[args.out] = _AsyncIOProxy(p.stdout, event_loop)
             else:
                 to_close.append(p.stdout)
             if args.err:
                 self.shell.user_ns[args.err] = _AsyncIOProxy(p.stderr, event_loop)
             else:
                 to_close.append(p.stderr)
             event_loop.call_soon_threadsafe(
                 lambda: asyncio.Task(self._run_script(p, cell, to_close))
             )
             if args.proc:
                 proc_proxy = _AsyncIOProxy(p, event_loop)
                 proc_proxy.stdout = _AsyncIOProxy(p.stdout, event_loop)
                 proc_proxy.stderr = _AsyncIOProxy(p.stderr, event_loop)
                 self.shell.user_ns[args.proc] = proc_proxy
             return
 
         try:
             in_thread(_stream_communicate(p, cell))
         except KeyboardInterrupt:
             try:
                 p.send_signal(signal.SIGINT)
                 in_thread(asyncio.wait_for(p.wait(), timeout=0.1))
                 if p.returncode is not None:
                     print("Process is interrupted.")
                     return
                 p.terminate()
                 in_thread(asyncio.wait_for(p.wait(), timeout=0.1))
                 if p.returncode is not None:
                     print("Process is terminated.")
                     return
                 p.kill()
                 print("Process is killed.")
             except OSError:
                 pass
             except Exception as e:
                 print("Error while terminating subprocess (pid=%i): %s" % (p.pid, e))
             return
 
         if args.raise_error and p.returncode != 0:
             # If we get here and p.returncode is still None, we must have
             # killed it but not yet seen its return code. We don't wait for it,
             # in case it's stuck in uninterruptible sleep. -9 = SIGKILL
             rc = p.returncode or -9
             raise CalledProcessError(rc, cell)
 
     shebang.__skip_doctest__ = os.name != "posix"
 
     async def _run_script(self, p, cell, to_close):
         """callback for running the script in the background"""
 
         p.stdin.write(cell)
         await p.stdin.drain()
         p.stdin.close()
         await p.stdin.wait_closed()
         await p.wait()
         # asyncio read pipes have no close
         # but we should drain the data anyway
         for s in to_close:
             await s.read()
         self._gc_bg_processes()
 
     @line_magic("killbgscripts")
     def killbgscripts(self, _nouse_=''):
         """Kill all BG processes started by %%script and its family."""
         self.kill_bg_processes()
         print("All background processes were killed.")
 
     def kill_bg_processes(self):
         """Kill all BG processes which are still running."""
         if not self.bg_processes:
             return
         for p in self.bg_processes:
             if p.returncode is None:
                 try:
                     p.send_signal(signal.SIGINT)
                 except:
                     pass
         time.sleep(0.1)
         self._gc_bg_processes()
         if not self.bg_processes:
             return
         for p in self.bg_processes:
             if p.returncode is None:
                 try:
                     p.terminate()
                 except:
                     pass
         time.sleep(0.1)
         self._gc_bg_processes()
         if not self.bg_processes:
             return
         for p in self.bg_processes:
             if p.returncode is None:
                 try:
                     p.kill()
                 except:
                     pass
         self._gc_bg_processes()
 
     def _gc_bg_processes(self):
         self.bg_processes = [p for p in self.bg_processes if p.returncode is None]

IPython/core/release.py

@@ -1,54 +1,54 b''
 # -*- coding: utf-8 -*-
 """Release data for the IPython project."""
 
 #-----------------------------------------------------------------------------
 #  Copyright (c) 2008, IPython Development Team.
 #  Copyright (c) 2001, Fernando Perez <[e-mail]>
 #  Copyright (c) 2001, Janko Hauser <[e-mail]>
 #  Copyright (c) 2001, Nathaniel Gray <[e-mail]>
 #
 #  Distributed under the terms of the Modified BSD License.
 #
 #  The full license is in the file COPYING.txt, distributed with this software.
 #-----------------------------------------------------------------------------
 
 # IPython version information.  An empty _version_extra corresponds to a full
 # release.  'dev' as a _version_extra string means this is a development
 # version
 _version_major = 8
-_version_minor = 9
+_version_minor = 10
 _version_patch = 0
 _version_extra = ".dev"
 # _version_extra = "rc1"
 # _version_extra = ""  # Uncomment this for full releases
 
 # Construct full version string from these.
 _ver = [_version_major, _version_minor, _version_patch]
 
 __version__ = '.'.join(map(str, _ver))
 if _version_extra:
     __version__ = __version__  + _version_extra
 
 version = __version__  # backwards compatibility name
 version_info = (_version_major, _version_minor, _version_patch, _version_extra)
 
 # Change this when incrementing the kernel protocol version
 kernel_protocol_version_info = (5, 0)
 kernel_protocol_version = "%i.%i" % kernel_protocol_version_info
 
 license = "BSD-3-Clause"
 
 authors = {'Fernando' : ('Fernando Perez','[e-mail]'),
            'Janko'    : ('Janko Hauser','[e-mail]'),
            'Nathan'   : ('Nathaniel Gray','[e-mail]'),
            'Ville'    : ('Ville Vainio','[e-mail]'),
            'Brian'    : ('Brian E Granger', '[e-mail]'),
            'Min'      : ('Min Ragan-Kelley', '[e-mail]'),
            'Thomas'   : ('Thomas A. Kluyver', '[e-mail]'),
            'Jorgen'   : ('Jorgen Stenarson', '[e-mail]'),
            'Matthias' : ('Matthias Bussonnier', '[e-mail]'),
            }
 
 author = 'The IPython Development Team'
 
 author_email = '[e-mail]'

IPython/core/shellapp.py

@@ -1,451 +1,451 b''
 # encoding: utf-8
 """
 A mixin for :class:`~IPython.core.application.Application` classes that
 launch InteractiveShell instances, load extensions, etc.
 """
 
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 import glob
 from itertools import chain
 import os
 import sys
 
 from traitlets.config.application import boolean_flag
 from traitlets.config.configurable import Configurable
 from traitlets.config.loader import Config
 from IPython.core.application import SYSTEM_CONFIG_DIRS, ENV_CONFIG_DIRS
 from IPython.core import pylabtools
 from IPython.utils.contexts import preserve_keys
 from IPython.utils.path import filefind
 from traitlets import (
     Unicode, Instance, List, Bool, CaselessStrEnum, observe,
     DottedObjectName,
 )
 from IPython.terminal import pt_inputhooks
 
 #-----------------------------------------------------------------------------
 # Aliases and Flags
 #-----------------------------------------------------------------------------
 
 gui_keys = tuple(sorted(pt_inputhooks.backends) + sorted(pt_inputhooks.aliases))
 
 backend_keys = sorted(pylabtools.backends.keys())
 backend_keys.insert(0, 'auto')
 
 shell_flags = {}
 
 addflag = lambda *args: shell_flags.update(boolean_flag(*args))
 addflag('autoindent', 'InteractiveShell.autoindent',
         'Turn on autoindenting.', 'Turn off autoindenting.'
 )
 addflag('automagic', 'InteractiveShell.automagic',
         """Turn on the auto calling of magic commands. Type %%magic at the
         IPython  prompt  for  more information.""",
         'Turn off the auto calling of magic commands.'
 )
 addflag('pdb', 'InteractiveShell.pdb',
     "Enable auto calling the pdb debugger after every exception.",
     "Disable auto calling the pdb debugger after every exception."
 )
 addflag('pprint', 'PlainTextFormatter.pprint',
     "Enable auto pretty printing of results.",
     "Disable auto pretty printing of results."
 )
 addflag('color-info', 'InteractiveShell.color_info',
     """IPython can display information about objects via a set of functions,
     and optionally can use colors for this, syntax highlighting
     source code and various other elements. This is on by default, but can cause
     problems with some pagers. If you see such problems, you can disable the
     colours.""",
     "Disable using colors for info related things."
 )
 addflag('ignore-cwd', 'InteractiveShellApp.ignore_cwd',
         "Exclude the current working directory from sys.path",
         "Include the current working directory in sys.path",
 )
 nosep_config = Config()
 nosep_config.InteractiveShell.separate_in = ''
 nosep_config.InteractiveShell.separate_out = ''
 nosep_config.InteractiveShell.separate_out2 = ''
 
 shell_flags['nosep']=(nosep_config, "Eliminate all spacing between prompts.")
 shell_flags['pylab'] = (
     {'InteractiveShellApp' : {'pylab' : 'auto'}},
     """Pre-load matplotlib and numpy for interactive use with
     the default matplotlib backend."""
 )
 shell_flags['matplotlib'] = (
     {'InteractiveShellApp' : {'matplotlib' : 'auto'}},
     """Configure matplotlib for interactive use with
     the default matplotlib backend."""
 )
 
 # it's possible we don't want short aliases for *all* of these:
 shell_aliases = dict(
     autocall='InteractiveShell.autocall',
     colors='InteractiveShell.colors',
     logfile='InteractiveShell.logfile',
     logappend='InteractiveShell.logappend',
     c='InteractiveShellApp.code_to_run',
     m='InteractiveShellApp.module_to_run',
     ext="InteractiveShellApp.extra_extensions",
     gui='InteractiveShellApp.gui',
     pylab='InteractiveShellApp.pylab',
     matplotlib='InteractiveShellApp.matplotlib',
 )
 shell_aliases['cache-size'] = 'InteractiveShell.cache_size'
 
 #-----------------------------------------------------------------------------
 # Main classes and functions
 #-----------------------------------------------------------------------------
 
 class InteractiveShellApp(Configurable):
     """A Mixin for applications that start InteractiveShell instances.
 
     Provides configurables for loading extensions and executing files
     as part of configuring a Shell environment.
 
     The following methods should be called by the :meth:`initialize` method
     of the subclass:
 
       - :meth:`init_path`
       - :meth:`init_shell` (to be implemented by the subclass)
       - :meth:`init_gui_pylab`
       - :meth:`init_extensions`
       - :meth:`init_code`
     """
     extensions = List(Unicode(),
         help="A list of dotted module names of IPython extensions to load."
     ).tag(config=True)
 
     extra_extensions = List(
         DottedObjectName(),
         help="""
         Dotted module name(s) of one or more IPython extensions to load.
 
         For specifying extra extensions to load on the command-line.
 
         .. versionadded:: 7.10
         """,
     ).tag(config=True)
 
     reraise_ipython_extension_failures = Bool(False,
         help="Reraise exceptions encountered loading IPython extensions?",
     ).tag(config=True)
 
     # Extensions that are always loaded (not configurable)
     default_extensions = List(Unicode(), [u'storemagic']).tag(config=False)
 
     hide_initial_ns = Bool(True,
         help="""Should variables loaded at startup (by startup files, exec_lines, etc.)
         be hidden from tools like %who?"""
     ).tag(config=True)
 
     exec_files = List(Unicode(),
         help="""List of files to run at IPython startup."""
     ).tag(config=True)
     exec_PYTHONSTARTUP = Bool(True,
         help="""Run the file referenced by the PYTHONSTARTUP environment
         variable at IPython startup."""
     ).tag(config=True)
     file_to_run = Unicode('',
         help="""A file to be run""").tag(config=True)
 
     exec_lines = List(Unicode(),
         help="""lines of code to run at IPython startup."""
     ).tag(config=True)
     code_to_run = Unicode('',
         help="Execute the given command string."
     ).tag(config=True)
     module_to_run = Unicode('',
         help="Run the module as a script."
     ).tag(config=True)
     gui = CaselessStrEnum(gui_keys, allow_none=True,
         help="Enable GUI event loop integration with any of {0}.".format(gui_keys)
     ).tag(config=True)
     matplotlib = CaselessStrEnum(backend_keys, allow_none=True,
         help="""Configure matplotlib for interactive use with
         the default matplotlib backend."""
     ).tag(config=True)
     pylab = CaselessStrEnum(backend_keys, allow_none=True,
         help="""Pre-load matplotlib and numpy for interactive use,
         selecting a particular matplotlib backend and loop integration.
         """
     ).tag(config=True)
     pylab_import_all = Bool(True,
         help="""If true, IPython will populate the user namespace with numpy, pylab, etc.
         and an ``import *`` is done from numpy and pylab, when using pylab mode.
 
         When False, pylab mode should not import any names into the user namespace.
         """
     ).tag(config=True)
     ignore_cwd = Bool(
         False,
         help="""If True, IPython will not add the current working directory to sys.path.
         When False, the current working directory is added to sys.path, allowing imports
         of modules defined in the current directory."""
     ).tag(config=True)
     shell = Instance('IPython.core.interactiveshell.InteractiveShellABC',
                      allow_none=True)
     # whether interact-loop should start
     interact = Bool(True)
 
     user_ns = Instance(dict, args=None, allow_none=True)
     @observe('user_ns')
     def _user_ns_changed(self, change):
         if self.shell is not None:
             self.shell.user_ns = change['new']
             self.shell.init_user_ns()
 
     def init_path(self):
         """Add current working directory, '', to sys.path
 
         Unlike Python's default, we insert before the first `site-packages`
         or `dist-packages` directory,
         so that it is after the standard library.
 
         .. versionchanged:: 7.2
             Try to insert after the standard library, instead of first.
         .. versionchanged:: 8.0
             Allow optionally not including the current directory in sys.path
         """
         if '' in sys.path or self.ignore_cwd:
             return
         for idx, path in enumerate(sys.path):
             parent, last_part = os.path.split(path)
             if last_part in {'site-packages', 'dist-packages'}:
                 break
         else:
             # no site-packages or dist-packages found (?!)
             # back to original behavior of inserting at the front
             idx = 0
         sys.path.insert(idx, '')
 
     def init_shell(self):
         raise NotImplementedError("Override in subclasses")
 
     def init_gui_pylab(self):
         """Enable GUI event loop integration, taking pylab into account."""
         enable = False
         shell = self.shell
         if self.pylab:
             enable = lambda key: shell.enable_pylab(key, import_all=self.pylab_import_all)
             key = self.pylab
         elif self.matplotlib:
             enable = shell.enable_matplotlib
             key = self.matplotlib
         elif self.gui:
             enable = shell.enable_gui
             key = self.gui
 
         if not enable:
             return
 
         try:
             r = enable(key)
         except ImportError:
             self.log.warning("Eventloop or matplotlib integration failed. Is matplotlib installed?")
             self.shell.showtraceback()
             return
         except Exception:
             self.log.warning("GUI event loop or pylab initialization failed")
             self.shell.showtraceback()
             return
 
         if isinstance(r, tuple):
             gui, backend = r[:2]
             self.log.info("Enabling GUI event loop integration, "
                       "eventloop=%s, matplotlib=%s", gui, backend)
             if key == "auto":
                 print("Using matplotlib backend: %s" % backend)
         else:
             gui = r
             self.log.info("Enabling GUI event loop integration, "
                       "eventloop=%s", gui)
 
     def init_extensions(self):
         """Load all IPython extensions in IPythonApp.extensions.
 
         This uses the :meth:`ExtensionManager.load_extensions` to load all
         the extensions listed in ``self.extensions``.
         """
         try:
             self.log.debug("Loading IPython extensions...")
             extensions = (
                 self.default_extensions + self.extensions + self.extra_extensions
             )
             for ext in extensions:
                 try:
-                    self.log.info("Loading IPython extension: %s" % ext)
+                    self.log.info("Loading IPython extension: %s", ext)
                     self.shell.extension_manager.load_extension(ext)
                 except:
                     if self.reraise_ipython_extension_failures:
                         raise
                     msg = ("Error in loading extension: {ext}\n"
                            "Check your config files in {location}".format(
                                ext=ext,
                                location=self.profile_dir.location
                            ))
                     self.log.warning(msg, exc_info=True)
         except:
             if self.reraise_ipython_extension_failures:
                 raise
             self.log.warning("Unknown error in loading extensions:", exc_info=True)
 
     def init_code(self):
         """run the pre-flight code, specified via exec_lines"""
         self._run_startup_files()
         self._run_exec_lines()
         self._run_exec_files()
 
         # Hide variables defined here from %who etc.
         if self.hide_initial_ns:
             self.shell.user_ns_hidden.update(self.shell.user_ns)
 
         # command-line execution (ipython -i script.py, ipython -m module)
         # should *not* be excluded from %whos
         self._run_cmd_line_code()
         self._run_module()
 
         # flush output, so itwon't be attached to the first cell
         sys.stdout.flush()
         sys.stderr.flush()
         self.shell._sys_modules_keys = set(sys.modules.keys())
 
     def _run_exec_lines(self):
         """Run lines of code in IPythonApp.exec_lines in the user's namespace."""
         if not self.exec_lines:
             return
         try:
             self.log.debug("Running code from IPythonApp.exec_lines...")
             for line in self.exec_lines:
                 try:
                     self.log.info("Running code in user namespace: %s" %
                                   line)
                     self.shell.run_cell(line, store_history=False)
                 except:
                     self.log.warning("Error in executing line in user "
                                   "namespace: %s" % line)
                     self.shell.showtraceback()
         except:
             self.log.warning("Unknown error in handling IPythonApp.exec_lines:")
             self.shell.showtraceback()
 
     def _exec_file(self, fname, shell_futures=False):
         try:
             full_filename = filefind(fname, [u'.', self.ipython_dir])
         except IOError:
             self.log.warning("File not found: %r"%fname)
             return
         # Make sure that the running script gets a proper sys.argv as if it
         # were run from a system shell.
         save_argv = sys.argv
         sys.argv = [full_filename] + self.extra_args[1:]
         try:
             if os.path.isfile(full_filename):
                 self.log.info("Running file in user namespace: %s" %
                               full_filename)
                 # Ensure that __file__ is always defined to match Python
                 # behavior.
                 with preserve_keys(self.shell.user_ns, '__file__'):
                     self.shell.user_ns['__file__'] = fname
                     if full_filename.endswith('.ipy') or full_filename.endswith('.ipynb'):
                         self.shell.safe_execfile_ipy(full_filename,
                                                      shell_futures=shell_futures)
                     else:
                         # default to python, even without extension
                         self.shell.safe_execfile(full_filename,
                                                  self.shell.user_ns,
                                                  shell_futures=shell_futures,
                                                  raise_exceptions=True)
         finally:
             sys.argv = save_argv
 
     def _run_startup_files(self):
         """Run files from profile startup directory"""
         startup_dirs = [self.profile_dir.startup_dir] + [
             os.path.join(p, 'startup') for p in chain(ENV_CONFIG_DIRS, SYSTEM_CONFIG_DIRS)
         ]
         startup_files = []
 
         if self.exec_PYTHONSTARTUP and os.environ.get('PYTHONSTARTUP', False) and \
                 not (self.file_to_run or self.code_to_run or self.module_to_run):
             python_startup = os.environ['PYTHONSTARTUP']
             self.log.debug("Running PYTHONSTARTUP file %s...", python_startup)
             try:
                 self._exec_file(python_startup)
             except:
                 self.log.warning("Unknown error in handling PYTHONSTARTUP file %s:", python_startup)
                 self.shell.showtraceback()
         for startup_dir in startup_dirs[::-1]:
             startup_files += glob.glob(os.path.join(startup_dir, '*.py'))
             startup_files += glob.glob(os.path.join(startup_dir, '*.ipy'))
         if not startup_files:
             return
 
         self.log.debug("Running startup files from %s...", startup_dir)
         try:
             for fname in sorted(startup_files):
                 self._exec_file(fname)
         except:
             self.log.warning("Unknown error in handling startup files:")
             self.shell.showtraceback()
 
     def _run_exec_files(self):
         """Run files from IPythonApp.exec_files"""
         if not self.exec_files:
             return
 
         self.log.debug("Running files in IPythonApp.exec_files...")
         try:
             for fname in self.exec_files:
                 self._exec_file(fname)
         except:
             self.log.warning("Unknown error in handling IPythonApp.exec_files:")
             self.shell.showtraceback()
 
     def _run_cmd_line_code(self):
         """Run code or file specified at the command-line"""
         if self.code_to_run:
             line = self.code_to_run
             try:
                 self.log.info("Running code given at command line (c=): %s" %
                               line)
                 self.shell.run_cell(line, store_history=False)
             except:
                 self.log.warning("Error in executing line in user namespace: %s" %
                               line)
                 self.shell.showtraceback()
                 if not self.interact:
                     self.exit(1)
 
         # Like Python itself, ignore the second if the first of these is present
         elif self.file_to_run:
             fname = self.file_to_run
             if os.path.isdir(fname):
                 fname = os.path.join(fname, "__main__.py")
             if not os.path.exists(fname):
                 self.log.warning("File '%s' doesn't exist", fname)
                 if not self.interact:
                     self.exit(2)
             try:
                 self._exec_file(fname, shell_futures=True)
             except:
                 self.shell.showtraceback(tb_offset=4)
                 if not self.interact:
                     self.exit(1)
 
     def _run_module(self):
         """Run module specified at the command-line."""
         if self.module_to_run:
             # Make sure that the module gets a proper sys.argv as if it were
             # run using `python -m`.
             save_argv = sys.argv
             sys.argv = [sys.executable] + self.extra_args
             try:
                 self.shell.safe_run_module(self.module_to_run,
                                            self.shell.user_ns)
             finally:
                 sys.argv = save_argv

IPython/core/tests/test_magic.py

@@ -1,1454 +1,1513 b''
 # -*- coding: utf-8 -*-
 """Tests for various magic functions."""
 
 import gc
 import io
 import os
 import re
 import shlex
 import sys
 import warnings
 from importlib import invalidate_caches
 from io import StringIO
 from pathlib import Path
 from textwrap import dedent
 from unittest import TestCase, mock
 
 import pytest
 
 from IPython import get_ipython
 from IPython.core import magic
 from IPython.core.error import UsageError
 from IPython.core.magic import (
     Magics,
     cell_magic,
     line_magic,
     magics_class,
     register_cell_magic,
     register_line_magic,
 )
 from IPython.core.magics import code, execution, logging, osm, script
 from IPython.testing import decorators as dec
 from IPython.testing import tools as tt
 from IPython.utils.io import capture_output
 from IPython.utils.process import find_cmd
 from IPython.utils.tempdir import TemporaryDirectory, TemporaryWorkingDirectory
 from IPython.utils.syspathcontext import prepended_to_syspath
 
 from .test_debugger import PdbTestInput
 
 from tempfile import NamedTemporaryFile
 
 @magic.magics_class
 class DummyMagics(magic.Magics): pass
 
 def test_extract_code_ranges():
     instr = "1 3 5-6 7-9 10:15 17: :10 10- -13 :"
     expected = [
         (0, 1),
         (2, 3),
         (4, 6),
         (6, 9),
         (9, 14),
         (16, None),
         (None, 9),
         (9, None),
         (None, 13),
         (None, None),
     ]
     actual = list(code.extract_code_ranges(instr))
     assert actual == expected
 
 def test_extract_symbols():
     source = """import foo\na = 10\ndef b():\n    return 42\n\n\nclass A: pass\n\n\n"""
     symbols_args = ["a", "b", "A", "A,b", "A,a", "z"]
     expected = [([], ['a']),
                 (["def b():\n    return 42\n"], []),
                 (["class A: pass\n"], []),
                 (["class A: pass\n", "def b():\n    return 42\n"], []),
                 (["class A: pass\n"], ['a']),
                 ([], ['z'])]
     for symbols, exp in zip(symbols_args, expected):
         assert code.extract_symbols(source, symbols) == exp
 
 
 def test_extract_symbols_raises_exception_with_non_python_code():
     source = ("=begin A Ruby program :)=end\n"
               "def hello\n"
               "puts 'Hello world'\n"
               "end")
     with pytest.raises(SyntaxError):
         code.extract_symbols(source, "hello")
 
 
 def test_magic_not_found():
     # magic not found raises UsageError
     with pytest.raises(UsageError):
         _ip.run_line_magic("doesntexist", "")
 
     # ensure result isn't success when a magic isn't found
     result = _ip.run_cell('%doesntexist')
     assert isinstance(result.error_in_exec, UsageError)
 
 
 def test_cell_magic_not_found():
     # magic not found raises UsageError
     with pytest.raises(UsageError):
         _ip.run_cell_magic('doesntexist', 'line', 'cell')
 
     # ensure result isn't success when a magic isn't found
     result = _ip.run_cell('%%doesntexist')
     assert isinstance(result.error_in_exec, UsageError)
 
 
 def test_magic_error_status():
     def fail(shell):
         1/0
     _ip.register_magic_function(fail)
     result = _ip.run_cell('%fail')
     assert isinstance(result.error_in_exec, ZeroDivisionError)
 
 
 def test_config():
     """ test that config magic does not raise
     can happen if Configurable init is moved too early into
     Magics.__init__ as then a Config object will be registered as a
     magic.
     """
     ## should not raise.
     _ip.run_line_magic("config", "")
 
 
 def test_config_available_configs():
     """ test that config magic prints available configs in unique and
     sorted order. """
     with capture_output() as captured:
         _ip.run_line_magic("config", "")
 
     stdout = captured.stdout
     config_classes = stdout.strip().split('\n')[1:]
     assert config_classes == sorted(set(config_classes))
 
 def test_config_print_class():
     """ test that config with a classname prints the class's options. """
     with capture_output() as captured:
         _ip.run_line_magic("config", "TerminalInteractiveShell")
 
     stdout = captured.stdout
     assert re.match(
         "TerminalInteractiveShell.* options", stdout.splitlines()[0]
     ), f"{stdout}\n\n1st line of stdout not like 'TerminalInteractiveShell.* options'"
 
 
 def test_rehashx():
     # clear up everything
     _ip.alias_manager.clear_aliases()
     del _ip.db['syscmdlist']
 
     _ip.run_line_magic("rehashx", "")
     # Practically ALL ipython development systems will have more than 10 aliases
 
     assert len(_ip.alias_manager.aliases) > 10
     for name, cmd in _ip.alias_manager.aliases:
         # we must strip dots from alias names
         assert "." not in name
 
     # rehashx must fill up syscmdlist
     scoms = _ip.db['syscmdlist']
     assert len(scoms) > 10
 
 
 def test_magic_parse_options():
     """Test that we don't mangle paths when parsing magic options."""
     ip = get_ipython()
     path = 'c:\\x'
     m = DummyMagics(ip)
     opts = m.parse_options('-f %s' % path,'f:')[0]
     # argv splitting is os-dependent
     if os.name == 'posix':
         expected = 'c:x'
     else:
         expected = path
     assert opts["f"] == expected
 
 
 def test_magic_parse_long_options():
     """Magic.parse_options can handle --foo=bar long options"""
     ip = get_ipython()
     m = DummyMagics(ip)
     opts, _ = m.parse_options("--foo --bar=bubble", "a", "foo", "bar=")
     assert "foo" in opts
     assert "bar" in opts
     assert opts["bar"] == "bubble"
 
 
 def doctest_hist_f():
     """Test %hist -f with temporary filename.
 
     In [9]: import tempfile
 
     In [10]: tfile = tempfile.mktemp('.py','tmp-ipython-')
 
     In [11]: %hist -nl -f $tfile 3
 
     In [13]: import os; os.unlink(tfile)
     """
 
 
 def doctest_hist_op():
     """Test %hist -op
 
     In [1]: class b(float):
        ...:     pass
        ...:
 
     In [2]: class s(object):
        ...:     def __str__(self):
        ...:         return 's'
        ...:
 
     In [3]:
 
     In [4]: class r(b):
        ...:     def __repr__(self):
        ...:         return 'r'
        ...:
 
     In [5]: class sr(s,r): pass
        ...:
 
     In [6]:
 
     In [7]: bb=b()
 
     In [8]: ss=s()
 
     In [9]: rr=r()
 
     In [10]: ssrr=sr()
 
     In [11]: 4.5
     Out[11]: 4.5
 
     In [12]: str(ss)
     Out[12]: 's'
 
     In [13]:
 
     In [14]: %hist -op
     >>> class b:
     ...     pass
     ...
     >>> class s(b):
     ...     def __str__(self):
     ...         return 's'
     ...
     >>>
     >>> class r(b):
     ...     def __repr__(self):
     ...         return 'r'
     ...
     >>> class sr(s,r): pass
     >>>
     >>> bb=b()
     >>> ss=s()
     >>> rr=r()
     >>> ssrr=sr()
     >>> 4.5
     4.5
     >>> str(ss)
     's'
     >>>
     """
 
 def test_hist_pof():
     ip = get_ipython()
     ip.run_cell("1+2", store_history=True)
     #raise Exception(ip.history_manager.session_number)
     #raise Exception(list(ip.history_manager._get_range_session()))
     with TemporaryDirectory() as td:
         tf = os.path.join(td, 'hist.py')
         ip.run_line_magic('history', '-pof %s' % tf)
         assert os.path.isfile(tf)
 
 
 def test_macro():
     ip = get_ipython()
     ip.history_manager.reset()   # Clear any existing history.
     cmds = ["a=1", "def b():\n  return a**2", "print(a,b())"]
     for i, cmd in enumerate(cmds, start=1):
         ip.history_manager.store_inputs(i, cmd)
     ip.run_line_magic("macro", "test 1-3")
     assert ip.user_ns["test"].value == "\n".join(cmds) + "\n"
 
     # List macros
     assert "test" in ip.run_line_magic("macro", "")
 
 
 def test_macro_run():
     """Test that we can run a multi-line macro successfully."""
     ip = get_ipython()
     ip.history_manager.reset()
     cmds = ["a=10", "a+=1", "print(a)", "%macro test 2-3"]
     for cmd in cmds:
         ip.run_cell(cmd, store_history=True)
     assert ip.user_ns["test"].value == "a+=1\nprint(a)\n"
     with tt.AssertPrints("12"):
         ip.run_cell("test")
     with tt.AssertPrints("13"):
         ip.run_cell("test")
 
 
 def test_magic_magic():
     """Test %magic"""
     ip = get_ipython()
     with capture_output() as captured:
         ip.run_line_magic("magic", "")
 
     stdout = captured.stdout
     assert "%magic" in stdout
     assert "IPython" in stdout
     assert "Available" in stdout
 
 
 @dec.skipif_not_numpy
 def test_numpy_reset_array_undec():
     "Test '%reset array' functionality"
     _ip.ex("import numpy as np")
     _ip.ex("a = np.empty(2)")
     assert "a" in _ip.user_ns
     _ip.run_line_magic("reset", "-f array")
     assert "a" not in _ip.user_ns
 
 
 def test_reset_out():
     "Test '%reset out' magic"
     _ip.run_cell("parrot = 'dead'", store_history=True)
     # test '%reset -f out', make an Out prompt
     _ip.run_cell("parrot", store_history=True)
     assert "dead" in [_ip.user_ns[x] for x in ("_", "__", "___")]
     _ip.run_line_magic("reset", "-f out")
     assert "dead" not in [_ip.user_ns[x] for x in ("_", "__", "___")]
     assert len(_ip.user_ns["Out"]) == 0
 
 
 def test_reset_in():
     "Test '%reset in' magic"
     # test '%reset -f in'
     _ip.run_cell("parrot", store_history=True)
     assert "parrot" in [_ip.user_ns[x] for x in ("_i", "_ii", "_iii")]
     _ip.run_line_magic("reset", "-f in")
     assert "parrot" not in [_ip.user_ns[x] for x in ("_i", "_ii", "_iii")]
     assert len(set(_ip.user_ns["In"])) == 1
 
 
 def test_reset_dhist():
     "Test '%reset dhist' magic"
     _ip.run_cell("tmp = [d for d in _dh]")  # copy before clearing
     _ip.run_line_magic("cd", os.path.dirname(pytest.__file__))
     _ip.run_line_magic("cd", "-")
     assert len(_ip.user_ns["_dh"]) > 0
     _ip.run_line_magic("reset", "-f dhist")
     assert len(_ip.user_ns["_dh"]) == 0
     _ip.run_cell("_dh = [d for d in tmp]")  # restore
 
 
 def test_reset_in_length():
     "Test that '%reset in' preserves In[] length"
     _ip.run_cell("print 'foo'")
     _ip.run_cell("reset -f in")
     assert len(_ip.user_ns["In"]) == _ip.displayhook.prompt_count + 1
 
 
 class TestResetErrors(TestCase):
 
     def test_reset_redefine(self):
 
         @magics_class
         class KernelMagics(Magics):
               @line_magic
               def less(self, shell): pass
 
         _ip.register_magics(KernelMagics)
 
         with self.assertLogs() as cm:
             # hack, we want to just capture logs, but assertLogs fails if not
             # logs get produce.
             # so log one things we ignore.
             import logging as log_mod
             log = log_mod.getLogger()
             log.info('Nothing')
             # end hack.
             _ip.run_cell("reset -f")
 
         assert len(cm.output) == 1
         for out in cm.output:
             assert "Invalid alias" not in out
 
 def test_tb_syntaxerror():
     """test %tb after a SyntaxError"""
     ip = get_ipython()
     ip.run_cell("for")
 
     # trap and validate stdout
     save_stdout = sys.stdout
     try:
         sys.stdout = StringIO()
         ip.run_cell("%tb")
         out = sys.stdout.getvalue()
     finally:
         sys.stdout = save_stdout
     # trim output, and only check the last line
     last_line = out.rstrip().splitlines()[-1].strip()
     assert last_line == "SyntaxError: invalid syntax"
 
 
 def test_time():
     ip = get_ipython()
 
     with tt.AssertPrints("Wall time: "):
         ip.run_cell("%time None")
 
     ip.run_cell("def f(kmjy):\n"
                 "    %time print (2*kmjy)")
 
     with tt.AssertPrints("Wall time: "):
         with tt.AssertPrints("hihi", suppress=False):
             ip.run_cell("f('hi')")
 
+
+# ';' at the end of %time prevents instruction value to be printed.
+# This tests fix for #13837.
+def test_time_no_output_with_semicolon():
+    ip = get_ipython()
+
+    # Test %time cases
+    with tt.AssertPrints(" 123456"):
+        with tt.AssertPrints("Wall time: ", suppress=False):
+            with tt.AssertPrints("CPU times: ", suppress=False):
+                ip.run_cell("%time 123000+456")
+
+    with tt.AssertNotPrints(" 123456"):
+        with tt.AssertPrints("Wall time: ", suppress=False):
+            with tt.AssertPrints("CPU times: ", suppress=False):
+                ip.run_cell("%time 123000+456;")
+
+    with tt.AssertPrints(" 123456"):
+        with tt.AssertPrints("Wall time: ", suppress=False):
+            with tt.AssertPrints("CPU times: ", suppress=False):
+                ip.run_cell("%time 123000+456 # Comment")
+
+    with tt.AssertNotPrints(" 123456"):
+        with tt.AssertPrints("Wall time: ", suppress=False):
+            with tt.AssertPrints("CPU times: ", suppress=False):
+                ip.run_cell("%time 123000+456; # Comment")
+
+    with tt.AssertPrints(" 123456"):
+        with tt.AssertPrints("Wall time: ", suppress=False):
+            with tt.AssertPrints("CPU times: ", suppress=False):
+                ip.run_cell("%time 123000+456 # ;Comment")
+
+    # Test %%time cases
+    with tt.AssertPrints("123456"):
+        with tt.AssertPrints("Wall time: ", suppress=False):
+            with tt.AssertPrints("CPU times: ", suppress=False):
+                ip.run_cell("%%time\n123000+456\n\n\n")
+
+    with tt.AssertNotPrints("123456"):
+        with tt.AssertPrints("Wall time: ", suppress=False):
+            with tt.AssertPrints("CPU times: ", suppress=False):
+                ip.run_cell("%%time\n123000+456;\n\n\n")
+
+    with tt.AssertPrints("123456"):
+        with tt.AssertPrints("Wall time: ", suppress=False):
+            with tt.AssertPrints("CPU times: ", suppress=False):
+                ip.run_cell("%%time\n123000+456  # Comment\n\n\n")
+
+    with tt.AssertNotPrints("123456"):
+        with tt.AssertPrints("Wall time: ", suppress=False):
+            with tt.AssertPrints("CPU times: ", suppress=False):
+                ip.run_cell("%%time\n123000+456;  # Comment\n\n\n")
+
+    with tt.AssertPrints("123456"):
+        with tt.AssertPrints("Wall time: ", suppress=False):
+            with tt.AssertPrints("CPU times: ", suppress=False):
+                ip.run_cell("%%time\n123000+456  # ;Comment\n\n\n")
+
+
 def test_time_last_not_expression():
     ip.run_cell("%%time\n"
                 "var_1 = 1\n"
                 "var_2 = 2\n")
     assert ip.user_ns['var_1'] == 1
     del ip.user_ns['var_1']
     assert ip.user_ns['var_2'] == 2
     del ip.user_ns['var_2']
 
 
 @dec.skip_win32
 def test_time2():
     ip = get_ipython()
 
     with tt.AssertPrints("CPU times: user "):
         ip.run_cell("%time None")
 
 def test_time3():
     """Erroneous magic function calls, issue gh-3334"""
     ip = get_ipython()
     ip.user_ns.pop('run', None)
 
     with tt.AssertNotPrints("not found", channel='stderr'):
         ip.run_cell("%%time\n"
                     "run = 0\n"
                     "run += 1")
 
 def test_multiline_time():
     """Make sure last statement from time return a value."""
     ip = get_ipython()
     ip.user_ns.pop('run', None)
 
     ip.run_cell(
         dedent(
             """\
         %%time
         a = "ho"
         b = "hey"
         a+b
         """
         )
     )
     assert ip.user_ns_hidden["_"] == "hohey"
 
 
 def test_time_local_ns():
     """
     Test that local_ns is actually global_ns when running a cell magic
     """
     ip = get_ipython()
     ip.run_cell("%%time\n" "myvar = 1")
     assert ip.user_ns["myvar"] == 1
     del ip.user_ns["myvar"]
 
 
 def test_doctest_mode():
     "Toggle doctest_mode twice, it should be a no-op and run without error"
     _ip.run_line_magic("doctest_mode", "")
     _ip.run_line_magic("doctest_mode", "")
 
 
 def test_parse_options():
     """Tests for basic options parsing in magics."""
     # These are only the most minimal of tests, more should be added later.  At
     # the very least we check that basic text/unicode calls work OK.
     m = DummyMagics(_ip)
     assert m.parse_options("foo", "")[1] == "foo"
     assert m.parse_options("foo", "")[1] == "foo"
 
 
 def test_parse_options_preserve_non_option_string():
     """Test to assert preservation of non-option part of magic-block, while parsing magic options."""
     m = DummyMagics(_ip)
     opts, stmt = m.parse_options(
         " -n1  -r 13 _ = 314 + foo", "n:r:", preserve_non_opts=True
     )
     assert opts == {"n": "1", "r": "13"}
     assert stmt == "_ = 314 + foo"
 
 
 def test_run_magic_preserve_code_block():
     """Test to assert preservation of non-option part of magic-block, while running magic."""
     _ip.user_ns["spaces"] = []
     _ip.run_line_magic(
         "timeit", "-n1 -r1 spaces.append([s.count(' ') for s in ['document']])"
     )
     assert _ip.user_ns["spaces"] == [[0]]
 
 
 def test_dirops():
     """Test various directory handling operations."""
     # curpath = lambda :os.path.splitdrive(os.getcwd())[1].replace('\\','/')
     curpath = os.getcwd
     startdir = os.getcwd()
     ipdir = os.path.realpath(_ip.ipython_dir)
     try:
         _ip.run_line_magic("cd", '"%s"' % ipdir)
         assert curpath() == ipdir
         _ip.run_line_magic("cd", "-")
         assert curpath() == startdir
         _ip.run_line_magic("pushd", '"%s"' % ipdir)
         assert curpath() == ipdir
         _ip.run_line_magic("popd", "")
         assert curpath() == startdir
     finally:
         os.chdir(startdir)
 
 
 def test_cd_force_quiet():
     """Test OSMagics.cd_force_quiet option"""
     _ip.config.OSMagics.cd_force_quiet = True
     osmagics = osm.OSMagics(shell=_ip)
 
     startdir = os.getcwd()
     ipdir = os.path.realpath(_ip.ipython_dir)
 
     try:
         with tt.AssertNotPrints(ipdir):
             osmagics.cd('"%s"' % ipdir)
         with tt.AssertNotPrints(startdir):
             osmagics.cd('-')
     finally:
         os.chdir(startdir)
 
 
 def test_xmode():
     # Calling xmode three times should be a no-op
     xmode = _ip.InteractiveTB.mode
     for i in range(4):
         _ip.run_line_magic("xmode", "")
     assert _ip.InteractiveTB.mode == xmode
 
 def test_reset_hard():
     monitor = []
     class A(object):
         def __del__(self):
             monitor.append(1)
         def __repr__(self):
             return "<A instance>"
 
     _ip.user_ns["a"] = A()
     _ip.run_cell("a")
 
     assert monitor == []
     _ip.run_line_magic("reset", "-f")
     assert monitor == [1]
 
 class TestXdel(tt.TempFileMixin):
     def test_xdel(self):
         """Test that references from %run are cleared by xdel."""
         src = ("class A(object):\n"
                "    monitor = []\n"
                "    def __del__(self):\n"
                "        self.monitor.append(1)\n"
                "a = A()\n")
         self.mktmp(src)
         # %run creates some hidden references...
         _ip.run_line_magic("run", "%s" % self.fname)
         # ... as does the displayhook.
         _ip.run_cell("a")
 
         monitor = _ip.user_ns["A"].monitor
         assert monitor == []
 
         _ip.run_line_magic("xdel", "a")
 
         # Check that a's __del__ method has been called.
         gc.collect(0)
         assert monitor == [1]
 
 def doctest_who():
     """doctest for %who
 
     In [1]: %reset -sf
 
     In [2]: alpha = 123
 
     In [3]: beta = 'beta'
 
     In [4]: %who int
     alpha
 
     In [5]: %who str
     beta
 
     In [6]: %whos
     Variable   Type    Data/Info
     ----------------------------
     alpha      int     123
     beta       str     beta
 
     In [7]: %who_ls
     Out[7]: ['alpha', 'beta']
     """
 
 def test_whos():
     """Check that whos is protected against objects where repr() fails."""
     class A(object):
         def __repr__(self):
             raise Exception()
     _ip.user_ns['a'] = A()
     _ip.run_line_magic("whos", "")
 
 def doctest_precision():
     """doctest for %precision
 
     In [1]: f = get_ipython().display_formatter.formatters['text/plain']
 
     In [2]: %precision 5
     Out[2]: '%.5f'
 
     In [3]: f.float_format
     Out[3]: '%.5f'
 
     In [4]: %precision %e
     Out[4]: '%e'
 
     In [5]: f(3.1415927)
     Out[5]: '3.141593e+00'
     """
 
 def test_debug_magic():
     """Test debugging a small code with %debug
 
     In [1]: with PdbTestInput(['c']):
        ...:     %debug print("a b") #doctest: +ELLIPSIS
        ...:
     ...
     ipdb> c
     a b
     In [2]:
     """
 
 def test_psearch():
     with tt.AssertPrints("dict.fromkeys"):
         _ip.run_cell("dict.fr*?")
     with tt.AssertPrints("π.is_integer"):
         _ip.run_cell("π = 3.14;\nπ.is_integ*?")
 
 def test_timeit_shlex():
     """test shlex issues with timeit (#1109)"""
     _ip.ex("def f(*a,**kw): pass")
     _ip.run_line_magic("timeit", '-n1 "this is a bug".count(" ")')
     _ip.run_line_magic("timeit", '-r1 -n1 f(" ", 1)')
     _ip.run_line_magic("timeit", '-r1 -n1 f(" ", 1, " ", 2, " ")')
     _ip.run_line_magic("timeit", '-r1 -n1 ("a " + "b")')
     _ip.run_line_magic("timeit", '-r1 -n1 f("a " + "b")')
     _ip.run_line_magic("timeit", '-r1 -n1 f("a " + "b ")')
 
 
 def test_timeit_special_syntax():
     "Test %%timeit with IPython special syntax"
     @register_line_magic
     def lmagic(line):
         ip = get_ipython()
         ip.user_ns['lmagic_out'] = line
 
     # line mode test
     _ip.run_line_magic("timeit", "-n1 -r1 %lmagic my line")
     assert _ip.user_ns["lmagic_out"] == "my line"
     # cell mode test
     _ip.run_cell_magic("timeit", "-n1 -r1", "%lmagic my line2")
     assert _ip.user_ns["lmagic_out"] == "my line2"
 
 
 def test_timeit_return():
     """
     test whether timeit -o return object
     """
 
     res = _ip.run_line_magic('timeit','-n10 -r10 -o 1')
     assert(res is not None)
 
 def test_timeit_quiet():
     """
     test quiet option of timeit magic
     """
     with tt.AssertNotPrints("loops"):
         _ip.run_cell("%timeit -n1 -r1 -q 1")
 
 def test_timeit_return_quiet():
     with tt.AssertNotPrints("loops"):
         res = _ip.run_line_magic('timeit', '-n1 -r1 -q -o 1')
     assert (res is not None)
 
 def test_timeit_invalid_return():
     with pytest.raises(SyntaxError):
         _ip.run_line_magic('timeit', 'return')
 
 @dec.skipif(execution.profile is None)
 def test_prun_special_syntax():
     "Test %%prun with IPython special syntax"
     @register_line_magic
     def lmagic(line):
         ip = get_ipython()
         ip.user_ns['lmagic_out'] = line
 
     # line mode test
     _ip.run_line_magic("prun", "-q %lmagic my line")
     assert _ip.user_ns["lmagic_out"] == "my line"
     # cell mode test
     _ip.run_cell_magic("prun", "-q", "%lmagic my line2")
     assert _ip.user_ns["lmagic_out"] == "my line2"
 
 
 @dec.skipif(execution.profile is None)
 def test_prun_quotes():
     "Test that prun does not clobber string escapes (GH #1302)"
     _ip.magic(r"prun -q x = '\t'")
     assert _ip.user_ns["x"] == "\t"
 
 
 def test_extension():
     # Debugging information for failures of this test
     print('sys.path:')
     for p in sys.path:
         print(' ', p)
     print('CWD', os.getcwd())
 
     pytest.raises(ImportError, _ip.magic, "load_ext daft_extension")
     daft_path = os.path.join(os.path.dirname(__file__), "daft_extension")
     sys.path.insert(0, daft_path)
     try:
         _ip.user_ns.pop('arq', None)
         invalidate_caches()   # Clear import caches
         _ip.run_line_magic("load_ext", "daft_extension")
         assert _ip.user_ns["arq"] == 185
         _ip.run_line_magic("unload_ext", "daft_extension")
         assert 'arq' not in _ip.user_ns
     finally:
         sys.path.remove(daft_path)
 
 
 def test_notebook_export_json():
     pytest.importorskip("nbformat")
     _ip = get_ipython()
     _ip.history_manager.reset()   # Clear any existing history.
     cmds = ["a=1", "def b():\n  return a**2", "print('noël, été', b())"]
     for i, cmd in enumerate(cmds, start=1):
         _ip.history_manager.store_inputs(i, cmd)
     with TemporaryDirectory() as td:
         outfile = os.path.join(td, "nb.ipynb")
         _ip.run_line_magic("notebook", "%s" % outfile)
 
 
 class TestEnv(TestCase):
 
     def test_env(self):
         env = _ip.run_line_magic("env", "")
         self.assertTrue(isinstance(env, dict))
 
     def test_env_secret(self):
         env = _ip.run_line_magic("env", "")
         hidden = "<hidden>"
         with mock.patch.dict(
             os.environ,
             {
                 "API_KEY": "abc123",
                 "SECRET_THING": "ssshhh",
                 "JUPYTER_TOKEN": "",
                 "VAR": "abc"
             }
         ):
             env = _ip.run_line_magic("env", "")
         assert env["API_KEY"] == hidden
         assert env["SECRET_THING"] == hidden
         assert env["JUPYTER_TOKEN"] == hidden
         assert env["VAR"] == "abc"
 
     def test_env_get_set_simple(self):
         env = _ip.run_line_magic("env", "var val1")
         self.assertEqual(env, None)
         self.assertEqual(os.environ["var"], "val1")
         self.assertEqual(_ip.run_line_magic("env", "var"), "val1")
         env = _ip.run_line_magic("env", "var=val2")
         self.assertEqual(env, None)
         self.assertEqual(os.environ['var'], 'val2')
 
     def test_env_get_set_complex(self):
         env = _ip.run_line_magic("env", "var 'val1 '' 'val2")
         self.assertEqual(env, None)
         self.assertEqual(os.environ['var'], "'val1 '' 'val2")
         self.assertEqual(_ip.run_line_magic("env", "var"), "'val1 '' 'val2")
         env = _ip.run_line_magic("env", 'var=val2 val3="val4')
         self.assertEqual(env, None)
         self.assertEqual(os.environ['var'], 'val2 val3="val4')
 
     def test_env_set_bad_input(self):
         self.assertRaises(UsageError, lambda: _ip.run_line_magic("set_env", "var"))
 
     def test_env_set_whitespace(self):
         self.assertRaises(UsageError, lambda: _ip.run_line_magic("env", "var A=B"))
 
 
 class CellMagicTestCase(TestCase):
 
     def check_ident(self, magic):
         # Manually called, we get the result
         out = _ip.run_cell_magic(magic, "a", "b")
         assert out == ("a", "b")
         # Via run_cell, it goes into the user's namespace via displayhook
         _ip.run_cell("%%" + magic + " c\nd\n")
         assert _ip.user_ns["_"] == ("c", "d\n")
 
     def test_cell_magic_func_deco(self):
         "Cell magic using simple decorator"
         @register_cell_magic
         def cellm(line, cell):
             return line, cell
 
         self.check_ident('cellm')
 
     def test_cell_magic_reg(self):
         "Cell magic manually registered"
         def cellm(line, cell):
             return line, cell
 
         _ip.register_magic_function(cellm, 'cell', 'cellm2')
         self.check_ident('cellm2')
 
     def test_cell_magic_class(self):
         "Cell magics declared via a class"
         @magics_class
         class MyMagics(Magics):
 
             @cell_magic
             def cellm3(self, line, cell):
                 return line, cell
 
         _ip.register_magics(MyMagics)
         self.check_ident('cellm3')
 
     def test_cell_magic_class2(self):
         "Cell magics declared via a class, #2"
         @magics_class
         class MyMagics2(Magics):
 
             @cell_magic('cellm4')
             def cellm33(self, line, cell):
                 return line, cell
 
         _ip.register_magics(MyMagics2)
         self.check_ident('cellm4')
         # Check that nothing is registered as 'cellm33'
         c33 = _ip.find_cell_magic('cellm33')
         assert c33 == None
 
 def test_file():
     """Basic %%writefile"""
     ip = get_ipython()
     with TemporaryDirectory() as td:
         fname = os.path.join(td, "file1")
         ip.run_cell_magic(
             "writefile",
             fname,
             "\n".join(
                 [
                     "line1",
                     "line2",
                 ]
             ),
         )
         s = Path(fname).read_text(encoding="utf-8")
         assert "line1\n" in s
         assert "line2" in s
 
 
 @dec.skip_win32
 def test_file_single_quote():
     """Basic %%writefile with embedded single quotes"""
     ip = get_ipython()
     with TemporaryDirectory() as td:
         fname = os.path.join(td, "'file1'")
         ip.run_cell_magic(
             "writefile",
             fname,
             "\n".join(
                 [
                     "line1",
                     "line2",
                 ]
             ),
         )
         s = Path(fname).read_text(encoding="utf-8")
         assert "line1\n" in s
         assert "line2" in s
 
 
 @dec.skip_win32
 def test_file_double_quote():
     """Basic %%writefile with embedded double quotes"""
     ip = get_ipython()
     with TemporaryDirectory() as td:
         fname = os.path.join(td, '"file1"')
         ip.run_cell_magic(
             "writefile",
             fname,
             "\n".join(
                 [
                     "line1",
                     "line2",
                 ]
             ),
         )
         s = Path(fname).read_text(encoding="utf-8")
         assert "line1\n" in s
         assert "line2" in s
 
 
 def test_file_var_expand():
     """%%writefile $filename"""
     ip = get_ipython()
     with TemporaryDirectory() as td:
         fname = os.path.join(td, "file1")
         ip.user_ns["filename"] = fname
         ip.run_cell_magic(
             "writefile",
             "$filename",
             "\n".join(
                 [
                     "line1",
                     "line2",
                 ]
             ),
         )
         s = Path(fname).read_text(encoding="utf-8")
         assert "line1\n" in s
         assert "line2" in s
 
 
 def test_file_unicode():
     """%%writefile with unicode cell"""
     ip = get_ipython()
     with TemporaryDirectory() as td:
         fname = os.path.join(td, 'file1')
         ip.run_cell_magic("writefile", fname, u'\n'.join([
             u'liné1',
             u'liné2',
         ]))
         with io.open(fname, encoding='utf-8') as f:
             s = f.read()
         assert "liné1\n" in s
         assert "liné2" in s
 
 
 def test_file_amend():
     """%%writefile -a amends files"""
     ip = get_ipython()
     with TemporaryDirectory() as td:
         fname = os.path.join(td, "file2")
         ip.run_cell_magic(
             "writefile",
             fname,
             "\n".join(
                 [
                     "line1",
                     "line2",
                 ]
             ),
         )
         ip.run_cell_magic(
             "writefile",
             "-a %s" % fname,
             "\n".join(
                 [
                     "line3",
                     "line4",
                 ]
             ),
         )
         s = Path(fname).read_text(encoding="utf-8")
         assert "line1\n" in s
         assert "line3\n" in s
 
 
 def test_file_spaces():
     """%%file with spaces in filename"""
     ip = get_ipython()
     with TemporaryWorkingDirectory() as td:
         fname = "file name"
         ip.run_cell_magic(
             "file",
             '"%s"' % fname,
             "\n".join(
                 [
                     "line1",
                     "line2",
                 ]
             ),
         )
         s = Path(fname).read_text(encoding="utf-8")
         assert "line1\n" in s
         assert "line2" in s
 
 
 def test_script_config():
     ip = get_ipython()
     ip.config.ScriptMagics.script_magics = ['whoda']
     sm = script.ScriptMagics(shell=ip)
     assert "whoda" in sm.magics["cell"]
 
 
 def test_script_out():
     ip = get_ipython()
     ip.run_cell_magic("script", f"--out output {sys.executable}", "print('hi')")
     assert ip.user_ns["output"].strip() == "hi"
 
 
 def test_script_err():
     ip = get_ipython()
     ip.run_cell_magic(
         "script",
         f"--err error {sys.executable}",
         "import sys; print('hello', file=sys.stderr)",
     )
     assert ip.user_ns["error"].strip() == "hello"
 
 
 def test_script_out_err():
 
     ip = get_ipython()
     ip.run_cell_magic(
         "script",
         f"--out output --err error {sys.executable}",
         "\n".join(
             [
                 "import sys",
                 "print('hi')",
                 "print('hello', file=sys.stderr)",
             ]
         ),
     )
     assert ip.user_ns["output"].strip() == "hi"
     assert ip.user_ns["error"].strip() == "hello"
 
 
 async def test_script_bg_out():
     ip = get_ipython()
     ip.run_cell_magic("script", f"--bg --out output {sys.executable}", "print('hi')")
     assert (await ip.user_ns["output"].read()).strip() == b"hi"
     assert ip.user_ns["output"].at_eof()
 
 
 async def test_script_bg_err():
     ip = get_ipython()
     ip.run_cell_magic(
         "script",
         f"--bg --err error {sys.executable}",
         "import sys; print('hello', file=sys.stderr)",
     )
     assert (await ip.user_ns["error"].read()).strip() == b"hello"
     assert ip.user_ns["error"].at_eof()
 
 
 async def test_script_bg_out_err():
     ip = get_ipython()
     ip.run_cell_magic(
         "script",
         f"--bg --out output --err error {sys.executable}",
         "\n".join(
             [
                 "import sys",
                 "print('hi')",
                 "print('hello', file=sys.stderr)",
             ]
         ),
     )
     assert (await ip.user_ns["output"].read()).strip() == b"hi"
     assert (await ip.user_ns["error"].read()).strip() == b"hello"
     assert ip.user_ns["output"].at_eof()
     assert ip.user_ns["error"].at_eof()
 
 
 async def test_script_bg_proc():
     ip = get_ipython()
     ip.run_cell_magic(
         "script",
         f"--bg --out output --proc p {sys.executable}",
         "\n".join(
             [
                 "import sys",
                 "print('hi')",
                 "print('hello', file=sys.stderr)",
             ]
         ),
     )
     p = ip.user_ns["p"]
     await p.wait()
     assert p.returncode == 0
     assert (await p.stdout.read()).strip() == b"hi"
     # not captured, so empty
     assert (await p.stderr.read()) == b""
     assert p.stdout.at_eof()
     assert p.stderr.at_eof()
 
 
 def test_script_defaults():
     ip = get_ipython()
     for cmd in ['sh', 'bash', 'perl', 'ruby']:
         try:
             find_cmd(cmd)
         except Exception:
             pass
         else:
             assert cmd in ip.magics_manager.magics["cell"]
 
 
 @magics_class
 class FooFoo(Magics):
     """class with both %foo and %%foo magics"""
     @line_magic('foo')
     def line_foo(self, line):
         "I am line foo"
         pass
 
     @cell_magic("foo")
     def cell_foo(self, line, cell):
         "I am cell foo, not line foo"
         pass
 
 def test_line_cell_info():
     """%%foo and %foo magics are distinguishable to inspect"""
     ip = get_ipython()
     ip.magics_manager.register(FooFoo)
     oinfo = ip.object_inspect("foo")
     assert oinfo["found"] is True
     assert oinfo["ismagic"] is True
 
     oinfo = ip.object_inspect("%%foo")
     assert oinfo["found"] is True
     assert oinfo["ismagic"] is True
     assert oinfo["docstring"] == FooFoo.cell_foo.__doc__
 
     oinfo = ip.object_inspect("%foo")
     assert oinfo["found"] is True
     assert oinfo["ismagic"] is True
     assert oinfo["docstring"] == FooFoo.line_foo.__doc__
 
 
 def test_multiple_magics():
     ip = get_ipython()
     foo1 = FooFoo(ip)
     foo2 = FooFoo(ip)
     mm = ip.magics_manager
     mm.register(foo1)
     assert mm.magics["line"]["foo"].__self__ is foo1
     mm.register(foo2)
     assert mm.magics["line"]["foo"].__self__ is foo2
 
 
 def test_alias_magic():
     """Test %alias_magic."""
     ip = get_ipython()
     mm = ip.magics_manager
 
     # Basic operation: both cell and line magics are created, if possible.
     ip.run_line_magic("alias_magic", "timeit_alias timeit")
     assert "timeit_alias" in mm.magics["line"]
     assert "timeit_alias" in mm.magics["cell"]
 
     # --cell is specified, line magic not created.
     ip.run_line_magic("alias_magic", "--cell timeit_cell_alias timeit")
     assert "timeit_cell_alias" not in mm.magics["line"]
     assert "timeit_cell_alias" in mm.magics["cell"]
 
     # Test that line alias is created successfully.
     ip.run_line_magic("alias_magic", "--line env_alias env")
     assert ip.run_line_magic("env", "") == ip.run_line_magic("env_alias", "")
 
     # Test that line alias with parameters passed in is created successfully.
     ip.run_line_magic(
         "alias_magic", "--line history_alias history --params " + shlex.quote("3")
     )
     assert "history_alias" in mm.magics["line"]
 
 
 def test_save():
     """Test %save."""
     ip = get_ipython()
     ip.history_manager.reset()   # Clear any existing history.
     cmds = ["a=1", "def b():\n  return a**2", "print(a, b())"]
     for i, cmd in enumerate(cmds, start=1):
         ip.history_manager.store_inputs(i, cmd)
     with TemporaryDirectory() as tmpdir:
         file = os.path.join(tmpdir, "testsave.py")
         ip.run_line_magic("save", "%s 1-10" % file)
         content = Path(file).read_text(encoding="utf-8")
         assert content.count(cmds[0]) == 1
         assert "coding: utf-8" in content
         ip.run_line_magic("save", "-a %s 1-10" % file)
         content = Path(file).read_text(encoding="utf-8")
         assert content.count(cmds[0]) == 2
         assert "coding: utf-8" in content
 
 
 def test_save_with_no_args():
     ip = get_ipython()
     ip.history_manager.reset()  # Clear any existing history.
     cmds = ["a=1", "def b():\n    return a**2", "print(a, b())", "%save"]
     for i, cmd in enumerate(cmds, start=1):
         ip.history_manager.store_inputs(i, cmd)
 
     with TemporaryDirectory() as tmpdir:
         path = os.path.join(tmpdir, "testsave.py")
         ip.run_line_magic("save", path)
         content = Path(path).read_text(encoding="utf-8")
         expected_content = dedent(
             """\
             # coding: utf-8
             a=1
             def b():
                 return a**2
             print(a, b())
             """
         )
         assert content == expected_content
 
 
 def test_store():
     """Test %store."""
     ip = get_ipython()
     ip.run_line_magic('load_ext', 'storemagic')
 
     # make sure the storage is empty
     ip.run_line_magic("store", "-z")
     ip.user_ns["var"] = 42
     ip.run_line_magic("store", "var")
     ip.user_ns["var"] = 39
     ip.run_line_magic("store", "-r")
     assert ip.user_ns["var"] == 42
 
     ip.run_line_magic("store", "-d var")
     ip.user_ns["var"] = 39
     ip.run_line_magic("store", "-r")
     assert ip.user_ns["var"] == 39
 
 
 def _run_edit_test(arg_s, exp_filename=None,
                         exp_lineno=-1,
                         exp_contents=None,
                         exp_is_temp=None):
     ip = get_ipython()
     M = code.CodeMagics(ip)
     last_call = ['','']
     opts,args = M.parse_options(arg_s,'prxn:')
     filename, lineno, is_temp = M._find_edit_target(ip, args, opts, last_call)
 
     if exp_filename is not None:
         assert exp_filename == filename
     if exp_contents is not None:
         with io.open(filename, 'r', encoding='utf-8') as f:
             contents = f.read()
         assert exp_contents == contents
     if exp_lineno != -1:
         assert exp_lineno == lineno
     if exp_is_temp is not None:
         assert exp_is_temp == is_temp
 
 
 def test_edit_interactive():
     """%edit on interactively defined objects"""
     ip = get_ipython()
     n = ip.execution_count
     ip.run_cell("def foo(): return 1", store_history=True)
 
     with pytest.raises(code.InteractivelyDefined) as e:
         _run_edit_test("foo")
     assert e.value.index == n
 
 
 def test_edit_cell():
     """%edit [cell id]"""
     ip = get_ipython()
 
     ip.run_cell("def foo(): return 1", store_history=True)
 
     # test
     _run_edit_test("1", exp_contents=ip.user_ns['In'][1], exp_is_temp=True)
 
 def test_edit_fname():
     """%edit file"""
     # test
     _run_edit_test("test file.py", exp_filename="test file.py")
 
 def test_bookmark():
     ip = get_ipython()
     ip.run_line_magic('bookmark', 'bmname')
     with tt.AssertPrints('bmname'):
         ip.run_line_magic('bookmark', '-l')
     ip.run_line_magic('bookmark', '-d bmname')
 
 def test_ls_magic():
     ip = get_ipython()
     json_formatter = ip.display_formatter.formatters['application/json']
     json_formatter.enabled = True
     lsmagic = ip.run_line_magic("lsmagic", "")
     with warnings.catch_warnings(record=True) as w:
         j = json_formatter(lsmagic)
     assert sorted(j) == ["cell", "line"]
     assert w == []  # no warnings
 
 
 def test_strip_initial_indent():
     def sii(s):
         lines = s.splitlines()
         return '\n'.join(code.strip_initial_indent(lines))
 
     assert sii("  a = 1\nb = 2") == "a = 1\nb = 2"
     assert sii("  a\n    b\nc") == "a\n  b\nc"
     assert sii("a\n  b") == "a\n  b"
 
 def test_logging_magic_quiet_from_arg():
     _ip.config.LoggingMagics.quiet = False
     lm = logging.LoggingMagics(shell=_ip)
     with TemporaryDirectory() as td:
         try:
             with tt.AssertNotPrints(re.compile("Activating.*")):
                 lm.logstart('-q {}'.format(
                         os.path.join(td, "quiet_from_arg.log")))
         finally:
             _ip.logger.logstop()
 
 def test_logging_magic_quiet_from_config():
     _ip.config.LoggingMagics.quiet = True
     lm = logging.LoggingMagics(shell=_ip)
     with TemporaryDirectory() as td:
         try:
             with tt.AssertNotPrints(re.compile("Activating.*")):
                 lm.logstart(os.path.join(td, "quiet_from_config.log"))
         finally:
             _ip.logger.logstop()
 
 
 def test_logging_magic_not_quiet():
     _ip.config.LoggingMagics.quiet = False
     lm = logging.LoggingMagics(shell=_ip)
     with TemporaryDirectory() as td:
         try:
             with tt.AssertPrints(re.compile("Activating.*")):
                 lm.logstart(os.path.join(td, "not_quiet.log"))
         finally:
             _ip.logger.logstop()
 
 
 def test_time_no_var_expand():
     _ip.user_ns["a"] = 5
     _ip.user_ns["b"] = []
     _ip.run_line_magic("time", 'b.append("{a}")')
     assert _ip.user_ns["b"] == ["{a}"]
 
 
 # this is slow, put at the end for local testing.
 def test_timeit_arguments():
     "Test valid timeit arguments, should not cause SyntaxError (GH #1269)"
     _ip.run_line_magic("timeit", "-n1 -r1 a=('#')")
 
 
 MINIMAL_LAZY_MAGIC = """
 from IPython.core.magic import (
     Magics,
     magics_class,
     line_magic,
     cell_magic,
 )
 
 
 @magics_class
 class LazyMagics(Magics):
     @line_magic
     def lazy_line(self, line):
         print("Lazy Line")
 
     @cell_magic
     def lazy_cell(self, line, cell):
         print("Lazy Cell")
 
 
 def load_ipython_extension(ipython):
     ipython.register_magics(LazyMagics)
 """
 
 
 def test_lazy_magics():
     with pytest.raises(UsageError):
         ip.run_line_magic("lazy_line", "")
 
     startdir = os.getcwd()
 
     with TemporaryDirectory() as tmpdir:
         with prepended_to_syspath(tmpdir):
             ptempdir = Path(tmpdir)
             tf = ptempdir / "lazy_magic_module.py"
             tf.write_text(MINIMAL_LAZY_MAGIC)
             ip.magics_manager.register_lazy("lazy_line", Path(tf.name).name[:-3])
             with tt.AssertPrints("Lazy Line"):
                 ip.run_line_magic("lazy_line", "")
 
 
 TEST_MODULE = """
 print('Loaded my_tmp')
 if __name__ == "__main__":
     print('I just ran a script')
 """
 
 def test_run_module_from_import_hook():
     "Test that a module can be loaded via an import hook"
     with TemporaryDirectory() as tmpdir:
         fullpath = os.path.join(tmpdir, "my_tmp.py")
         Path(fullpath).write_text(TEST_MODULE, encoding="utf-8")
 
         import importlib.abc
         import importlib.util
 
         class MyTempImporter(importlib.abc.MetaPathFinder, importlib.abc.SourceLoader):
             def find_spec(self, fullname, path, target=None):
                 if fullname == "my_tmp":
                     return importlib.util.spec_from_loader(fullname, self)
 
             def get_filename(self, fullname):
                 assert fullname == "my_tmp"
                 return fullpath
 
             def get_data(self, path):
                 assert Path(path).samefile(fullpath)
                 return Path(fullpath).read_text(encoding="utf-8")
 
         sys.meta_path.insert(0, MyTempImporter())
 
         with capture_output() as captured:
             _ip.run_line_magic("run", "-m my_tmp")
             _ip.run_cell("import my_tmp")
 
         output = "Loaded my_tmp\nI just ran a script\nLoaded my_tmp\n"
         assert output == captured.stdout
 
         sys.meta_path.pop(0)

IPython/terminal/interactiveshell.py

@@ -1,773 +1,808 b''
 """IPython terminal interface using prompt_toolkit"""
 
 import asyncio
 import os
 import sys
 from warnings import warn
+from typing import Union as UnionType
 
 from IPython.core.async_helpers import get_asyncio_loop
 from IPython.core.interactiveshell import InteractiveShell, InteractiveShellABC
 from IPython.utils.py3compat import input
 from IPython.utils.terminal import toggle_set_term_title, set_term_title, restore_term_title
 from IPython.utils.process import abbrev_cwd
 from traitlets import (
     Bool,
     Unicode,
     Dict,
     Integer,
     observe,
     Instance,
     Type,
     default,
     Enum,
     Union,
     Any,
     validate,
     Float,
 )
 
 from prompt_toolkit.auto_suggest import AutoSuggestFromHistory
 from prompt_toolkit.enums import DEFAULT_BUFFER, EditingMode
 from prompt_toolkit.filters import (HasFocus, Condition, IsDone)
 from prompt_toolkit.formatted_text import PygmentsTokens
 from prompt_toolkit.history import History
 from prompt_toolkit.layout.processors import ConditionalProcessor, HighlightMatchingBracketProcessor
 from prompt_toolkit.output import ColorDepth
 from prompt_toolkit.patch_stdout import patch_stdout
 from prompt_toolkit.shortcuts import PromptSession, CompleteStyle, print_formatted_text
 from prompt_toolkit.styles import DynamicStyle, merge_styles
 from prompt_toolkit.styles.pygments import style_from_pygments_cls, style_from_pygments_dict
 from prompt_toolkit import __version__ as ptk_version
 
 from pygments.styles import get_style_by_name
 from pygments.style import Style
 from pygments.token import Token
 
 from .debugger import TerminalPdb, Pdb
 from .magics import TerminalMagics
 from .pt_inputhooks import get_inputhook_name_and_func
 from .prompts import Prompts, ClassicPrompts, RichPromptDisplayHook
 from .ptutils import IPythonPTCompleter, IPythonPTLexer
 from .shortcuts import create_ipython_shortcuts
+from .shortcuts.auto_suggest import (
+    NavigableAutoSuggestFromHistory,
+    AppendAutoSuggestionInAnyLine,
+)
 
 PTK3 = ptk_version.startswith('3.')
 
 
 class _NoStyle(Style): pass
 
 
 
 _style_overrides_light_bg = {
             Token.Prompt: '#ansibrightblue',
             Token.PromptNum: '#ansiblue bold',
             Token.OutPrompt: '#ansibrightred',
             Token.OutPromptNum: '#ansired bold',
 }
 
 _style_overrides_linux = {
             Token.Prompt: '#ansibrightgreen',
             Token.PromptNum: '#ansigreen bold',
             Token.OutPrompt: '#ansibrightred',
             Token.OutPromptNum: '#ansired bold',
 }
 
 def get_default_editor():
     try:
         return os.environ['EDITOR']
     except KeyError:
         pass
     except UnicodeError:
         warn("$EDITOR environment variable is not pure ASCII. Using platform "
              "default editor.")
 
     if os.name == 'posix':
         return 'vi'  # the only one guaranteed to be there!
     else:
         return 'notepad' # same in Windows!
 
 # conservatively check for tty
 # overridden streams can result in things like:
 # - sys.stdin = None
 # - no isatty method
 for _name in ('stdin', 'stdout', 'stderr'):
     _stream = getattr(sys, _name)
     try:
         if not _stream or not hasattr(_stream, "isatty") or not _stream.isatty():
             _is_tty = False
             break
     except ValueError:
         # stream is closed
         _is_tty = False
         break
 else:
     _is_tty = True
 
 
 _use_simple_prompt = ('IPY_TEST_SIMPLE_PROMPT' in os.environ) or (not _is_tty)
 
 def black_reformat_handler(text_before_cursor):
     """
     We do not need to protect against error,
     this is taken care at a higher level where any reformat error is ignored.
     Indeed we may call reformatting on incomplete code.
     """
     import black
 
     formatted_text = black.format_str(text_before_cursor, mode=black.FileMode())
     if not text_before_cursor.endswith("\n") and formatted_text.endswith("\n"):
         formatted_text = formatted_text[:-1]
     return formatted_text
 
 
 def yapf_reformat_handler(text_before_cursor):
     from yapf.yapflib import file_resources
     from yapf.yapflib import yapf_api
 
     style_config = file_resources.GetDefaultStyleForDir(os.getcwd())
     formatted_text, was_formatted = yapf_api.FormatCode(
         text_before_cursor, style_config=style_config
     )
     if was_formatted:
         if not text_before_cursor.endswith("\n") and formatted_text.endswith("\n"):
             formatted_text = formatted_text[:-1]
         return formatted_text
     else:
         return text_before_cursor
 
 
 class PtkHistoryAdapter(History):
     """
     Prompt toolkit has it's own way of handling history, Where it assumes it can
     Push/pull from history.
 
     """
 
     def __init__(self, shell):
         super().__init__()
         self.shell = shell
         self._refresh()
 
     def append_string(self, string):
         # we rely on sql for that.
         self._loaded = False
         self._refresh()
 
     def _refresh(self):
         if not self._loaded:
             self._loaded_strings = list(self.load_history_strings())
 
     def load_history_strings(self):
         last_cell = ""
         res = []
         for __, ___, cell in self.shell.history_manager.get_tail(
             self.shell.history_load_length, include_latest=True
         ):
             # Ignore blank lines and consecutive duplicates
             cell = cell.rstrip()
             if cell and (cell != last_cell):
                 res.append(cell)
                 last_cell = cell
         yield from res[::-1]
 
     def store_string(self, string: str) -> None:
         pass
 
 class TerminalInteractiveShell(InteractiveShell):
     mime_renderers = Dict().tag(config=True)
 
     space_for_menu = Integer(6, help='Number of line at the bottom of the screen '
                                      'to reserve for the tab completion menu, '
                                      'search history, ...etc, the height of '
                                      'these menus will at most this value. '
                                      'Increase it is you prefer long and skinny '
                                      'menus, decrease for short and wide.'
                             ).tag(config=True)
 
-    pt_app = None
+    pt_app: UnionType[PromptSession, None] = None
+    auto_suggest: UnionType[
+        AutoSuggestFromHistory, NavigableAutoSuggestFromHistory, None
+    ] = None
     debugger_history = None
 
     debugger_history_file = Unicode(
         "~/.pdbhistory", help="File in which to store and read history"
     ).tag(config=True)
 
     simple_prompt = Bool(_use_simple_prompt,
         help="""Use `raw_input` for the REPL, without completion and prompt colors.
 
             Useful when controlling IPython as a subprocess, and piping STDIN/OUT/ERR. Known usage are:
             IPython own testing machinery, and emacs inferior-shell integration through elpy.
 
             This mode default to `True` if the `IPY_TEST_SIMPLE_PROMPT`
             environment variable is set, or the current terminal is not a tty."""
             ).tag(config=True)
 
     @property
     def debugger_cls(self):
         return Pdb if self.simple_prompt else TerminalPdb
 
     confirm_exit = Bool(True,
         help="""
         Set to confirm when you try to exit IPython with an EOF (Control-D
         in Unix, Control-Z/Enter in Windows). By typing 'exit' or 'quit',
         you can force a direct exit without any confirmation.""",
     ).tag(config=True)
 
     editing_mode = Unicode('emacs',
         help="Shortcut style to use at the prompt. 'vi' or 'emacs'.",
     ).tag(config=True)
 
     emacs_bindings_in_vi_insert_mode = Bool(
         True,
         help="Add shortcuts from 'emacs' insert mode to 'vi' insert mode.",
     ).tag(config=True)
 
     modal_cursor = Bool(
         True,
         help="""
        Cursor shape changes depending on vi mode: beam in vi insert mode,
        block in nav mode, underscore in replace mode.""",
     ).tag(config=True)
 
     ttimeoutlen = Float(
         0.01,
         help="""The time in milliseconds that is waited for a key code
        to complete.""",
     ).tag(config=True)
 
     timeoutlen = Float(
         0.5,
         help="""The time in milliseconds that is waited for a mapped key
        sequence to complete.""",
     ).tag(config=True)
 
     autoformatter = Unicode(
         None,
         help="Autoformatter to reformat Terminal code. Can be `'black'`, `'yapf'` or `None`",
         allow_none=True
     ).tag(config=True)
 
     auto_match = Bool(
         False,
         help="""
         Automatically add/delete closing bracket or quote when opening bracket or quote is entered/deleted.
         Brackets: (), [], {}
         Quotes: '', \"\"
         """,
     ).tag(config=True)
 
     mouse_support = Bool(False,
         help="Enable mouse support in the prompt\n(Note: prevents selecting text with the mouse)"
     ).tag(config=True)
 
     # We don't load the list of styles for the help string, because loading
     # Pygments plugins takes time and can cause unexpected errors.
     highlighting_style = Union([Unicode('legacy'), Type(klass=Style)],
         help="""The name or class of a Pygments style to use for syntax
         highlighting. To see available styles, run `pygmentize -L styles`."""
     ).tag(config=True)
 
     @validate('editing_mode')
     def _validate_editing_mode(self, proposal):
         if proposal['value'].lower() == 'vim':
             proposal['value']= 'vi'
         elif proposal['value'].lower() == 'default':
             proposal['value']= 'emacs'
 
         if hasattr(EditingMode, proposal['value'].upper()):
             return proposal['value'].lower()
 
         return self.editing_mode
 
 
     @observe('editing_mode')
     def _editing_mode(self, change):
         if self.pt_app:
             self.pt_app.editing_mode = getattr(EditingMode, change.new.upper())
 
     def _set_formatter(self, formatter):
         if formatter is None:
             self.reformat_handler = lambda x:x
         elif formatter == 'black':
             self.reformat_handler = black_reformat_handler
         elif formatter == "yapf":
             self.reformat_handler = yapf_reformat_handler
         else:
             raise ValueError
 
     @observe("autoformatter")
     def _autoformatter_changed(self, change):
         formatter = change.new
         self._set_formatter(formatter)
 
     @observe('highlighting_style')
     @observe('colors')
     def _highlighting_style_changed(self, change):
         self.refresh_style()
 
     def refresh_style(self):
         self._style = self._make_style_from_name_or_cls(self.highlighting_style)
 
 
     highlighting_style_overrides = Dict(
         help="Override highlighting format for specific tokens"
     ).tag(config=True)
 
     true_color = Bool(False,
         help="""Use 24bit colors instead of 256 colors in prompt highlighting.
         If your terminal supports true color, the following command should
         print ``TRUECOLOR`` in orange::
 
             printf \"\\x1b[38;2;255;100;0mTRUECOLOR\\x1b[0m\\n\"
         """,
     ).tag(config=True)
 
     editor = Unicode(get_default_editor(),
         help="Set the editor used by IPython (default to $EDITOR/vi/notepad)."
     ).tag(config=True)
 
     prompts_class = Type(Prompts, help='Class used to generate Prompt token for prompt_toolkit').tag(config=True)
 
     prompts = Instance(Prompts)
 
     @default('prompts')
     def _prompts_default(self):
         return self.prompts_class(self)
 
 #    @observe('prompts')
 #    def _(self, change):
 #        self._update_layout()
 
     @default('displayhook_class')
     def _displayhook_class_default(self):
         return RichPromptDisplayHook
 
     term_title = Bool(True,
         help="Automatically set the terminal title"
     ).tag(config=True)
 
     term_title_format = Unicode("IPython: {cwd}",
         help="Customize the terminal title format.  This is a python format string. " +
              "Available substitutions are: {cwd}."
     ).tag(config=True)
 
     display_completions = Enum(('column', 'multicolumn','readlinelike'),
         help= ( "Options for displaying tab completions, 'column', 'multicolumn', and "
                 "'readlinelike'. These options are for `prompt_toolkit`, see "
                 "`prompt_toolkit` documentation for more information."
                 ),
         default_value='multicolumn').tag(config=True)
 
     highlight_matching_brackets = Bool(True,
         help="Highlight matching brackets.",
     ).tag(config=True)
 
     extra_open_editor_shortcuts = Bool(False,
         help="Enable vi (v) or Emacs (C-X C-E) shortcuts to open an external editor. "
              "This is in addition to the F2 binding, which is always enabled."
     ).tag(config=True)
 
     handle_return = Any(None,
         help="Provide an alternative handler to be called when the user presses "
              "Return. This is an advanced option intended for debugging, which "
              "may be changed or removed in later releases."
     ).tag(config=True)
 
     enable_history_search = Bool(True,
         help="Allows to enable/disable the prompt toolkit history search"
     ).tag(config=True)
 
     autosuggestions_provider = Unicode(
-        "AutoSuggestFromHistory",
+        "NavigableAutoSuggestFromHistory",
         help="Specifies from which source automatic suggestions are provided. "
-        "Can be set to `'AutoSuggestFromHistory`' or `None` to disable"
-        "automatic suggestions. Default is `'AutoSuggestFromHistory`'.",
+        "Can be set to ``'NavigableAutoSuggestFromHistory'`` (:kbd:`up` and "
+        ":kbd:`down` swap suggestions), ``'AutoSuggestFromHistory'``, "
+        " or ``None`` to disable automatic suggestions. "
+        "Default is `'NavigableAutoSuggestFromHistory`'.",
         allow_none=True,
     ).tag(config=True)
 
     def _set_autosuggestions(self, provider):
+        # disconnect old handler
+        if self.auto_suggest and isinstance(
+            self.auto_suggest, NavigableAutoSuggestFromHistory
+        ):
+            self.auto_suggest.disconnect()
         if provider is None:
             self.auto_suggest = None
         elif provider == "AutoSuggestFromHistory":
             self.auto_suggest = AutoSuggestFromHistory()
+        elif provider == "NavigableAutoSuggestFromHistory":
+            self.auto_suggest = NavigableAutoSuggestFromHistory()
         else:
             raise ValueError("No valid provider.")
         if self.pt_app:
             self.pt_app.auto_suggest = self.auto_suggest
 
     @observe("autosuggestions_provider")
     def _autosuggestions_provider_changed(self, change):
         provider = change.new
         self._set_autosuggestions(provider)
 
     prompt_includes_vi_mode = Bool(True,
         help="Display the current vi mode (when using vi editing mode)."
     ).tag(config=True)
 
     @observe('term_title')
     def init_term_title(self, change=None):
         # Enable or disable the terminal title.
         if self.term_title and _is_tty:
             toggle_set_term_title(True)
             set_term_title(self.term_title_format.format(cwd=abbrev_cwd()))
         else:
             toggle_set_term_title(False)
 
     def restore_term_title(self):
         if self.term_title and _is_tty:
             restore_term_title()
 
     def init_display_formatter(self):
         super(TerminalInteractiveShell, self).init_display_formatter()
         # terminal only supports plain text
         self.display_formatter.active_types = ["text/plain"]
 
     def init_prompt_toolkit_cli(self):
         if self.simple_prompt:
             # Fall back to plain non-interactive output for tests.
             # This is very limited.
             def prompt():
                 prompt_text = "".join(x[1] for x in self.prompts.in_prompt_tokens())
                 lines = [input(prompt_text)]
                 prompt_continuation = "".join(x[1] for x in self.prompts.continuation_prompt_tokens())
                 while self.check_complete('\n'.join(lines))[0] == 'incomplete':
                     lines.append( input(prompt_continuation) )
                 return '\n'.join(lines)
             self.prompt_for_code = prompt
             return
 
         # Set up keyboard shortcuts
         key_bindings = create_ipython_shortcuts(self)
 
 
         # Pre-populate history from IPython's history database
         history = PtkHistoryAdapter(self)
 
         self._style = self._make_style_from_name_or_cls(self.highlighting_style)
         self.style = DynamicStyle(lambda: self._style)
 
         editing_mode = getattr(EditingMode, self.editing_mode.upper())
 
         self.pt_loop = asyncio.new_event_loop()
         self.pt_app = PromptSession(
             auto_suggest=self.auto_suggest,
             editing_mode=editing_mode,
             key_bindings=key_bindings,
             history=history,
             completer=IPythonPTCompleter(shell=self),
             enable_history_search=self.enable_history_search,
             style=self.style,
             include_default_pygments_style=False,
             mouse_support=self.mouse_support,
             enable_open_in_editor=self.extra_open_editor_shortcuts,
             color_depth=self.color_depth,
             tempfile_suffix=".py",
             **self._extra_prompt_options()
         )
+        if isinstance(self.auto_suggest, NavigableAutoSuggestFromHistory):
+            self.auto_suggest.connect(self.pt_app)
 
     def _make_style_from_name_or_cls(self, name_or_cls):
         """
         Small wrapper that make an IPython compatible style from a style name
 
         We need that to add style for prompt ... etc.
         """
         style_overrides = {}
         if name_or_cls == 'legacy':
             legacy = self.colors.lower()
             if legacy == 'linux':
                 style_cls = get_style_by_name('monokai')
                 style_overrides = _style_overrides_linux
             elif legacy == 'lightbg':
                 style_overrides = _style_overrides_light_bg
                 style_cls = get_style_by_name('pastie')
             elif legacy == 'neutral':
                 # The default theme needs to be visible on both a dark background
                 # and a light background, because we can't tell what the terminal
                 # looks like. These tweaks to the default theme help with that.
                 style_cls = get_style_by_name('default')
                 style_overrides.update({
                     Token.Number: '#ansigreen',
                     Token.Operator: 'noinherit',
                     Token.String: '#ansiyellow',
                     Token.Name.Function: '#ansiblue',
                     Token.Name.Class: 'bold #ansiblue',
                     Token.Name.Namespace: 'bold #ansiblue',
                     Token.Name.Variable.Magic: '#ansiblue',
                     Token.Prompt: '#ansigreen',
                     Token.PromptNum: '#ansibrightgreen bold',
                     Token.OutPrompt: '#ansired',
                     Token.OutPromptNum: '#ansibrightred bold',
                 })
 
                 # Hack: Due to limited color support on the Windows console
                 # the prompt colors will be wrong without this
                 if os.name == 'nt':
                     style_overrides.update({
                         Token.Prompt: '#ansidarkgreen',
                         Token.PromptNum: '#ansigreen bold',
                         Token.OutPrompt: '#ansidarkred',
                         Token.OutPromptNum: '#ansired bold',
                     })
             elif legacy =='nocolor':
                 style_cls=_NoStyle
                 style_overrides = {}
             else :
                 raise ValueError('Got unknown colors: ', legacy)
         else :
             if isinstance(name_or_cls, str):
                 style_cls = get_style_by_name(name_or_cls)
             else:
                 style_cls = name_or_cls
             style_overrides = {
                 Token.Prompt: '#ansigreen',
                 Token.PromptNum: '#ansibrightgreen bold',
                 Token.OutPrompt: '#ansired',
                 Token.OutPromptNum: '#ansibrightred bold',
             }
         style_overrides.update(self.highlighting_style_overrides)
         style = merge_styles([
             style_from_pygments_cls(style_cls),
             style_from_pygments_dict(style_overrides),
         ])
 
         return style
 
     @property
     def pt_complete_style(self):
         return {
             'multicolumn': CompleteStyle.MULTI_COLUMN,
             'column': CompleteStyle.COLUMN,
             'readlinelike': CompleteStyle.READLINE_LIKE,
         }[self.display_completions]
 
     @property
     def color_depth(self):
         return (ColorDepth.TRUE_COLOR if self.true_color else None)
 
     def _extra_prompt_options(self):
         """
         Return the current layout option for the current Terminal InteractiveShell
         """
         def get_message():
             return PygmentsTokens(self.prompts.in_prompt_tokens())
 
         if self.editing_mode == 'emacs':
             # with emacs mode the prompt is (usually) static, so we call only
             # the function once. With VI mode it can toggle between [ins] and
             # [nor] so we can't precompute.
             # here I'm going to favor the default keybinding which almost
             # everybody uses to decrease CPU usage.
             # if we have issues with users with custom Prompts we can see how to
             # work around this.
             get_message = get_message()
 
         options = {
-                'complete_in_thread': False,
-                'lexer':IPythonPTLexer(),
-                'reserve_space_for_menu':self.space_for_menu,
-                'message': get_message,
-                'prompt_continuation': (
-                    lambda width, lineno, is_soft_wrap:
-                        PygmentsTokens(self.prompts.continuation_prompt_tokens(width))),
-                'multiline': True,
-                'complete_style': self.pt_complete_style,
-
+            "complete_in_thread": False,
+            "lexer": IPythonPTLexer(),
+            "reserve_space_for_menu": self.space_for_menu,
+            "message": get_message,
+            "prompt_continuation": (
+                lambda width, lineno, is_soft_wrap: PygmentsTokens(
+                    self.prompts.continuation_prompt_tokens(width)
+                )
+            ),
+            "multiline": True,
+            "complete_style": self.pt_complete_style,
+            "input_processors": [
                 # Highlight matching brackets, but only when this setting is
                 # enabled, and only when the DEFAULT_BUFFER has the focus.
-                'input_processors': [ConditionalProcessor(
-                        processor=HighlightMatchingBracketProcessor(chars='[](){}'),
-                        filter=HasFocus(DEFAULT_BUFFER) & ~IsDone() &
-                            Condition(lambda: self.highlight_matching_brackets))],
-                }
+                ConditionalProcessor(
+                    processor=HighlightMatchingBracketProcessor(chars="[](){}"),
+                    filter=HasFocus(DEFAULT_BUFFER)
+                    & ~IsDone()
+                    & Condition(lambda: self.highlight_matching_brackets),
+                ),
+                # Show auto-suggestion in lines other than the last line.
+                ConditionalProcessor(
+                    processor=AppendAutoSuggestionInAnyLine(),
+                    filter=HasFocus(DEFAULT_BUFFER)
+                    & ~IsDone()
+                    & Condition(
+                        lambda: isinstance(
+                            self.auto_suggest, NavigableAutoSuggestFromHistory
+                        )
+                    ),
+                ),
+            ],
+        }
         if not PTK3:
             options['inputhook'] = self.inputhook
 
         return options
 
     def prompt_for_code(self):
         if self.rl_next_input:
             default = self.rl_next_input
             self.rl_next_input = None
         else:
             default = ''
 
         # In order to make sure that asyncio code written in the
         # interactive shell doesn't interfere with the prompt, we run the
         # prompt in a different event loop.
         # If we don't do this, people could spawn coroutine with a
         # while/true inside which will freeze the prompt.
 
         policy = asyncio.get_event_loop_policy()
         old_loop = get_asyncio_loop()
 
         # FIXME: prompt_toolkit is using the deprecated `asyncio.get_event_loop`
         # to get the current event loop.
         # This will probably be replaced by an attribute or input argument,
         # at which point we can stop calling the soon-to-be-deprecated `set_event_loop` here.
         if old_loop is not self.pt_loop:
             policy.set_event_loop(self.pt_loop)
         try:
             with patch_stdout(raw=True):
                 text = self.pt_app.prompt(
                     default=default,
                     **self._extra_prompt_options())
         finally:
             # Restore the original event loop.
             if old_loop is not None and old_loop is not self.pt_loop:
                 policy.set_event_loop(old_loop)
 
         return text
 
     def enable_win_unicode_console(self):
         # Since IPython 7.10 doesn't support python < 3.6 and PEP 528, Python uses the unicode APIs for the Windows
         # console by default, so WUC shouldn't be needed.
         warn("`enable_win_unicode_console` is deprecated since IPython 7.10, does not do anything and will be removed in the future",
              DeprecationWarning,
              stacklevel=2)
 
     def init_io(self):
         if sys.platform not in {'win32', 'cli'}:
             return
 
         import colorama
         colorama.init()
 
     def init_magics(self):
         super(TerminalInteractiveShell, self).init_magics()
         self.register_magics(TerminalMagics)
 
     def init_alias(self):
         # The parent class defines aliases that can be safely used with any
         # frontend.
         super(TerminalInteractiveShell, self).init_alias()
 
         # Now define aliases that only make sense on the terminal, because they
         # need direct access to the console in a way that we can't emulate in
         # GUI or web frontend
         if os.name == 'posix':
             for cmd in ('clear', 'more', 'less', 'man'):
                 self.alias_manager.soft_define_alias(cmd, cmd)
 
 
-    def __init__(self, *args, **kwargs):
+    def __init__(self, *args, **kwargs) -> None:
         super(TerminalInteractiveShell, self).__init__(*args, **kwargs)
         self._set_autosuggestions(self.autosuggestions_provider)
         self.init_prompt_toolkit_cli()
         self.init_term_title()
         self.keep_running = True
         self._set_formatter(self.autoformatter)
 
 
     def ask_exit(self):
         self.keep_running = False
 
     rl_next_input = None
 
     def interact(self):
         self.keep_running = True
         while self.keep_running:
             print(self.separate_in, end='')
 
             try:
                 code = self.prompt_for_code()
             except EOFError:
                 if (not self.confirm_exit) \
                         or self.ask_yes_no('Do you really want to exit ([y]/n)?','y','n'):
                     self.ask_exit()
 
             else:
                 if code:
                     self.run_cell(code, store_history=True)
 
     def mainloop(self):
         # An extra layer of protection in case someone mashing Ctrl-C breaks
         # out of our internal code.
         while True:
             try:
                 self.interact()
                 break
             except KeyboardInterrupt as e:
                 print("\n%s escaped interact()\n" % type(e).__name__)
             finally:
                 # An interrupt during the eventloop will mess up the
                 # internal state of the prompt_toolkit library.
                 # Stopping the eventloop fixes this, see
                 # https://github.com/ipython/ipython/pull/9867
                 if hasattr(self, '_eventloop'):
                     self._eventloop.stop()
 
                 self.restore_term_title()
 
         # try to call some at-exit operation optimistically as some things can't
         # be done during interpreter shutdown. this is technically inaccurate as
         # this make mainlool not re-callable, but that should be a rare if not
         # in existent use case.
 
         self._atexit_once()
 
 
     _inputhook = None
     def inputhook(self, context):
         if self._inputhook is not None:
             self._inputhook(context)
 
     active_eventloop = None
     def enable_gui(self, gui=None):
         if gui and (gui not in {"inline", "webagg"}):
             self.active_eventloop, self._inputhook = get_inputhook_name_and_func(gui)
         else:
             self.active_eventloop = self._inputhook = None
 
         # For prompt_toolkit 3.0. We have to create an asyncio event loop with
         # this inputhook.
         if PTK3:
             import asyncio
             from prompt_toolkit.eventloop import new_eventloop_with_inputhook
 
             if gui == 'asyncio':
                 # When we integrate the asyncio event loop, run the UI in the
                 # same event loop as the rest of the code. don't use an actual
                 # input hook. (Asyncio is not made for nesting event loops.)
                 self.pt_loop = get_asyncio_loop()
 
             elif self._inputhook:
                 # If an inputhook was set, create a new asyncio event loop with
                 # this inputhook for the prompt.
                 self.pt_loop = new_eventloop_with_inputhook(self._inputhook)
             else:
                 # When there's no inputhook, run the prompt in a separate
                 # asyncio event loop.
                 self.pt_loop = asyncio.new_event_loop()
 
     # Run !system commands directly, not through pipes, so terminal programs
     # work correctly.
     system = InteractiveShell.system_raw
 
     def auto_rewrite_input(self, cmd):
         """Overridden from the parent class to use fancy rewriting prompt"""
         if not self.show_rewritten_input:
             return
 
         tokens = self.prompts.rewrite_prompt_tokens()
         if self.pt_app:
             print_formatted_text(PygmentsTokens(tokens), end='',
                                  style=self.pt_app.app.style)
             print(cmd)
         else:
             prompt = ''.join(s for t, s in tokens)
             print(prompt, cmd, sep='')
 
     _prompts_before = None
     def switch_doctest_mode(self, mode):
         """Switch prompts to classic for %doctest_mode"""
         if mode:
             self._prompts_before = self.prompts
             self.prompts = ClassicPrompts(self)
         elif self._prompts_before:
             self.prompts = self._prompts_before
             self._prompts_before = None
 #        self._update_layout()
 
 
 InteractiveShellABC.register(TerminalInteractiveShell)
 
 if __name__ == '__main__':
     TerminalInteractiveShell.instance().interact()

IPython/terminal/ipapp.py

@@ -1,343 +1,343 b''
 #!/usr/bin/env python
 # encoding: utf-8
 """
 The :class:`~traitlets.config.application.Application` object for the command
 line :command:`ipython` program.
 """
 
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 
 import logging
 import os
 import sys
 import warnings
 
 from traitlets.config.loader import Config
 from traitlets.config.application import boolean_flag, catch_config_error
 from IPython.core import release
 from IPython.core import usage
 from IPython.core.completer import IPCompleter
 from IPython.core.crashhandler import CrashHandler
 from IPython.core.formatters import PlainTextFormatter
 from IPython.core.history import HistoryManager
 from IPython.core.application import (
     ProfileDir, BaseIPythonApplication, base_flags, base_aliases
 )
 from IPython.core.magic import MagicsManager
 from IPython.core.magics import (
     ScriptMagics, LoggingMagics
 )
 from IPython.core.shellapp import (
     InteractiveShellApp, shell_flags, shell_aliases
 )
 from IPython.extensions.storemagic import StoreMagics
 from .interactiveshell import TerminalInteractiveShell
 from IPython.paths import get_ipython_dir
 from traitlets import (
     Bool, List, default, observe, Type
 )
 
 #-----------------------------------------------------------------------------
 # Globals, utilities and helpers
 #-----------------------------------------------------------------------------
 
 _examples = """
 ipython --matplotlib       # enable matplotlib integration
 ipython --matplotlib=qt    # enable matplotlib integration with qt4 backend
 
 ipython --log-level=DEBUG  # set logging to DEBUG
 ipython --profile=foo      # start with profile foo
 
 ipython profile create foo # create profile foo w/ default config files
 ipython help profile       # show the help for the profile subcmd
 
 ipython locate             # print the path to the IPython directory
 ipython locate profile foo # print the path to the directory for profile `foo`
 """
 
 #-----------------------------------------------------------------------------
 # Crash handler for this application
 #-----------------------------------------------------------------------------
 
 class IPAppCrashHandler(CrashHandler):
     """sys.excepthook for IPython itself, leaves a detailed report on disk."""
 
     def __init__(self, app):
         contact_name = release.author
         contact_email = release.author_email
         bug_tracker = 'https://github.com/ipython/ipython/issues'
         super(IPAppCrashHandler,self).__init__(
             app, contact_name, contact_email, bug_tracker
         )
 
     def make_report(self,traceback):
         """Return a string containing a crash report."""
 
         sec_sep = self.section_sep
         # Start with parent report
         report = [super(IPAppCrashHandler, self).make_report(traceback)]
         # Add interactive-specific info we may have
         rpt_add = report.append
         try:
             rpt_add(sec_sep+"History of session input:")
             for line in self.app.shell.user_ns['_ih']:
                 rpt_add(line)
             rpt_add('\n*** Last line of input (may not be in above history):\n')
             rpt_add(self.app.shell._last_input_line+'\n')
         except:
             pass
 
         return ''.join(report)
 
 #-----------------------------------------------------------------------------
 # Aliases and Flags
 #-----------------------------------------------------------------------------
 flags = dict(base_flags)
 flags.update(shell_flags)
 frontend_flags = {}
 addflag = lambda *args: frontend_flags.update(boolean_flag(*args))
 addflag('autoedit-syntax', 'TerminalInteractiveShell.autoedit_syntax',
         'Turn on auto editing of files with syntax errors.',
         'Turn off auto editing of files with syntax errors.'
 )
 addflag('simple-prompt', 'TerminalInteractiveShell.simple_prompt',
         "Force simple minimal prompt using `raw_input`",
         "Use a rich interactive prompt with prompt_toolkit",
 )
 
 addflag('banner', 'TerminalIPythonApp.display_banner',
         "Display a banner upon starting IPython.",
         "Don't display a banner upon starting IPython."
 )
 addflag('confirm-exit', 'TerminalInteractiveShell.confirm_exit',
     """Set to confirm when you try to exit IPython with an EOF (Control-D
     in Unix, Control-Z/Enter in Windows). By typing 'exit' or 'quit',
     you can force a direct exit without any confirmation.""",
     "Don't prompt the user when exiting."
 )
 addflag('term-title', 'TerminalInteractiveShell.term_title',
     "Enable auto setting the terminal title.",
     "Disable auto setting the terminal title."
 )
 classic_config = Config()
 classic_config.InteractiveShell.cache_size = 0
 classic_config.PlainTextFormatter.pprint = False
 classic_config.TerminalInteractiveShell.prompts_class='IPython.terminal.prompts.ClassicPrompts'
 classic_config.InteractiveShell.separate_in = ''
 classic_config.InteractiveShell.separate_out = ''
 classic_config.InteractiveShell.separate_out2 = ''
 classic_config.InteractiveShell.colors = 'NoColor'
 classic_config.InteractiveShell.xmode = 'Plain'
 
 frontend_flags['classic']=(
     classic_config,
     "Gives IPython a similar feel to the classic Python prompt."
 )
 # # log doesn't make so much sense this way anymore
 # paa('--log','-l',
 #     action='store_true', dest='InteractiveShell.logstart',
 #     help="Start logging to the default log file (./ipython_log.py).")
 #
 # # quick is harder to implement
 frontend_flags['quick']=(
     {'TerminalIPythonApp' : {'quick' : True}},
     "Enable quick startup with no config files."
 )
 
 frontend_flags['i'] = (
     {'TerminalIPythonApp' : {'force_interact' : True}},
     """If running code from the command line, become interactive afterwards.
     It is often useful to follow this with `--` to treat remaining flags as
     script arguments.
     """
 )
 flags.update(frontend_flags)
 
 aliases = dict(base_aliases)
-aliases.update(shell_aliases)
+aliases.update(shell_aliases)  # type: ignore[arg-type]
 
 #-----------------------------------------------------------------------------
 # Main classes and functions
 #-----------------------------------------------------------------------------
 
 
 class LocateIPythonApp(BaseIPythonApplication):
     description = """print the path to the IPython dir"""
     subcommands = dict(
         profile=('IPython.core.profileapp.ProfileLocate',
             "print the path to an IPython profile directory",
         ),
     )
     def start(self):
         if self.subapp is not None:
             return self.subapp.start()
         else:
             print(self.ipython_dir)
 
 
 class TerminalIPythonApp(BaseIPythonApplication, InteractiveShellApp):
     name = u'ipython'
     description = usage.cl_usage
-    crash_handler_class = IPAppCrashHandler
+    crash_handler_class = IPAppCrashHandler  # typing: ignore[assignment]
     examples = _examples
 
     flags = flags
     aliases = aliases
     classes = List()
 
     interactive_shell_class = Type(
         klass=object,   # use default_value otherwise which only allow subclasses.
         default_value=TerminalInteractiveShell,
         help="Class to use to instantiate the TerminalInteractiveShell object. Useful for custom Frontends"
     ).tag(config=True)
 
     @default('classes')
     def _classes_default(self):
         """This has to be in a method, for TerminalIPythonApp to be available."""
         return [
             InteractiveShellApp, # ShellApp comes before TerminalApp, because
             self.__class__,      # it will also affect subclasses (e.g. QtConsole)
             TerminalInteractiveShell,
             HistoryManager,
             MagicsManager,
             ProfileDir,
             PlainTextFormatter,
             IPCompleter,
             ScriptMagics,
             LoggingMagics,
             StoreMagics,
         ]
 
     subcommands = dict(
         profile = ("IPython.core.profileapp.ProfileApp",
             "Create and manage IPython profiles."
         ),
         kernel = ("ipykernel.kernelapp.IPKernelApp",
             "Start a kernel without an attached frontend."
         ),
         locate=('IPython.terminal.ipapp.LocateIPythonApp',
             LocateIPythonApp.description
         ),
         history=('IPython.core.historyapp.HistoryApp',
             "Manage the IPython history database."
         ),
     )
 
 
     # *do* autocreate requested profile, but don't create the config file.
     auto_create=Bool(True)
     # configurables
     quick = Bool(False,
         help="""Start IPython quickly by skipping the loading of config files."""
     ).tag(config=True)
     @observe('quick')
     def _quick_changed(self, change):
         if change['new']:
             self.load_config_file = lambda *a, **kw: None
 
     display_banner = Bool(True,
         help="Whether to display a banner upon starting IPython."
     ).tag(config=True)
 
     # if there is code of files to run from the cmd line, don't interact
     # unless the --i flag (App.force_interact) is true.
     force_interact = Bool(False,
         help="""If a command or file is given via the command-line,
         e.g. 'ipython foo.py', start an interactive shell after executing the
         file or command."""
     ).tag(config=True)
     @observe('force_interact')
     def _force_interact_changed(self, change):
         if change['new']:
             self.interact = True
 
     @observe('file_to_run', 'code_to_run', 'module_to_run')
     def _file_to_run_changed(self, change):
         new = change['new']
         if new:
             self.something_to_run = True
         if new and not self.force_interact:
                 self.interact = False
 
     # internal, not-configurable
     something_to_run=Bool(False)
 
     @catch_config_error
     def initialize(self, argv=None):
         """Do actions after construct, but before starting the app."""
         super(TerminalIPythonApp, self).initialize(argv)
         if self.subapp is not None:
             # don't bother initializing further, starting subapp
             return
         # print self.extra_args
         if self.extra_args and not self.something_to_run:
             self.file_to_run = self.extra_args[0]
         self.init_path()
         # create the shell
         self.init_shell()
         # and draw the banner
         self.init_banner()
         # Now a variety of things that happen after the banner is printed.
         self.init_gui_pylab()
         self.init_extensions()
         self.init_code()
 
     def init_shell(self):
         """initialize the InteractiveShell instance"""
         # Create an InteractiveShell instance.
         # shell.display_banner should always be False for the terminal
         # based app, because we call shell.show_banner() by hand below
         # so the banner shows *before* all extension loading stuff.
         self.shell = self.interactive_shell_class.instance(parent=self,
                         profile_dir=self.profile_dir,
                         ipython_dir=self.ipython_dir, user_ns=self.user_ns)
         self.shell.configurables.append(self)
 
     def init_banner(self):
         """optionally display the banner"""
         if self.display_banner and self.interact:
             self.shell.show_banner()
         # Make sure there is a space below the banner.
         if self.log_level <= logging.INFO: print()
 
     def _pylab_changed(self, name, old, new):
         """Replace --pylab='inline' with --pylab='auto'"""
         if new == 'inline':
             warnings.warn("'inline' not available as pylab backend, "
                       "using 'auto' instead.")
             self.pylab = 'auto'
 
     def start(self):
         if self.subapp is not None:
             return self.subapp.start()
         # perform any prexec steps:
         if self.interact:
             self.log.debug("Starting IPython's mainloop...")
             self.shell.mainloop()
         else:
             self.log.debug("IPython not interactive...")
             self.shell.restore_term_title()
             if not self.shell.last_execution_succeeded:
                 sys.exit(1)
 
 def load_default_config(ipython_dir=None):
     """Load the default config file from the default ipython_dir.
 
     This is useful for embedded shells.
     """
     if ipython_dir is None:
         ipython_dir = get_ipython_dir()
 
     profile_dir = os.path.join(ipython_dir, 'profile_default')
     app = TerminalIPythonApp()
     app.config_file_paths.append(profile_dir)
     app.load_config_file()
     return app.config
 
 launch_new_instance = TerminalIPythonApp.launch_instance
 
 
 if __name__ == '__main__':
     launch_new_instance()