Merge pull request #13522 from Carreau/auto-backport-of-pr-13506-on-7.x

Backport PR  #13506: Allow to configure lazy loadable magics.

IPython/core/interactiveshell.py

@@ -1,3851 +1,3882 b''
 # -*- coding: utf-8 -*-
 """Main IPython class."""
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2001 Janko Hauser <[e-mail]>
 #  Copyright (C) 2001-2007 Fernando Perez. <[e-mail]>
 #  Copyright (C) 2008-2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 
 import abc
 import ast
 import atexit
 import builtins as builtin_mod
 import functools
 import inspect
 import os
 import re
 import runpy
 import sys
 import tempfile
 import traceback
 import types
 import subprocess
 import warnings
 from io import open as io_open
 
 from pathlib import Path
 from pickleshare import PickleShareDB
 
 from traitlets.config.configurable import SingletonConfigurable
 from traitlets.utils.importstring import import_item
 from IPython.core import oinspect
 from IPython.core import magic
 from IPython.core import page
 from IPython.core import prefilter
 from IPython.core import ultratb
 from IPython.core.alias import Alias, AliasManager
 from IPython.core.autocall import ExitAutocall
 from IPython.core.builtin_trap import BuiltinTrap
 from IPython.core.events import EventManager, available_events
 from IPython.core.compilerop import CachingCompiler, check_linecache_ipython
 from IPython.core.debugger import InterruptiblePdb
 from IPython.core.display_trap import DisplayTrap
 from IPython.core.displayhook import DisplayHook
 from IPython.core.displaypub import DisplayPublisher
 from IPython.core.error import InputRejected, UsageError
 from IPython.core.extensions import ExtensionManager
 from IPython.core.formatters import DisplayFormatter
 from IPython.core.history import HistoryManager
 from IPython.core.inputtransformer2 import ESC_MAGIC, ESC_MAGIC2
 from IPython.core.logger import Logger
 from IPython.core.macro import Macro
 from IPython.core.payload import PayloadManager
 from IPython.core.prefilter import PrefilterManager
 from IPython.core.profiledir import ProfileDir
 from IPython.core.usage import default_banner
 from IPython.display import display
 from IPython.testing.skipdoctest import skip_doctest
 from IPython.utils import PyColorize
 from IPython.utils import io
 from IPython.utils import py3compat
 from IPython.utils import openpy
 from IPython.utils.decorators import undoc
 from IPython.utils.io import ask_yes_no
 from IPython.utils.ipstruct import Struct
 from IPython.paths import get_ipython_dir
 from IPython.utils.path import get_home_dir, get_py_filename, ensure_dir_exists
 from IPython.utils.process import system, getoutput
 from IPython.utils.strdispatch import StrDispatch
 from IPython.utils.syspathcontext import prepended_to_syspath
 from IPython.utils.text import format_screen, LSString, SList, DollarFormatter
 from IPython.utils.tempdir import TemporaryDirectory
 from traitlets import (
     Integer, Bool, CaselessStrEnum, Enum, List, Dict, Unicode, Instance, Type,
     observe, default, validate, Any
 )
 from warnings import warn
 from logging import error
 import IPython.core.hooks
 
 from typing import List as ListType, Tuple, Optional
 from ast import AST
 
 # NoOpContext is deprecated, but ipykernel imports it from here.
 # See https://github.com/ipython/ipykernel/issues/157
 # (2016, let's try to remove than in IPython 8.0)
 from IPython.utils.contexts import NoOpContext
 
 try:
     import docrepr.sphinxify as sphx
 
     def sphinxify(oinfo):
         wrapped_docstring = sphx.wrap_main_docstring(oinfo)
 
         def sphinxify_docstring(docstring):
             with TemporaryDirectory() as dirname:
                 return {
                     "text/html": sphx.sphinxify(wrapped_docstring, dirname),
                     "text/plain": docstring,
                 }
 
         return sphinxify_docstring
 except ImportError:
     sphinxify = None
 
 
 class ProvisionalWarning(DeprecationWarning):
     """
     Warning class for unstable features
     """
     pass
 
 if sys.version_info > (3,8):
     from ast import Module
 else :
     # mock the new API, ignore second argument
     # see https://github.com/ipython/ipython/issues/11590
     from ast import Module as OriginalModule
     Module = lambda nodelist, type_ignores: OriginalModule(nodelist)
 
 if sys.version_info > (3,6):
     _assign_nodes         = (ast.AugAssign, ast.AnnAssign, ast.Assign)
     _single_targets_nodes = (ast.AugAssign, ast.AnnAssign)
 else:
     _assign_nodes         = (ast.AugAssign, ast.Assign )
     _single_targets_nodes = (ast.AugAssign, )
 
 #-----------------------------------------------------------------------------
 # Await Helpers
 #-----------------------------------------------------------------------------
 
 def removed_co_newlocals(function:types.FunctionType) -> types.FunctionType:
     """Return a function that do not create a new local scope. 
 
     Given a function, create a clone of this function where the co_newlocal flag
     has been removed, making this function code actually run in the sourounding
     scope. 
 
     We need this in order to run asynchronous code in user level namespace.
     """
     from types import CodeType, FunctionType
     CO_NEWLOCALS = 0x0002
     code = function.__code__
     new_co_flags = code.co_flags & ~CO_NEWLOCALS
     if sys.version_info > (3, 8, 0, 'alpha', 3):
         new_code = code.replace(co_flags=new_co_flags)
     else:
         new_code = CodeType(
             code.co_argcount,
             code.co_kwonlyargcount,
             code.co_nlocals,
             code.co_stacksize,
             new_co_flags,
             code.co_code,
             code.co_consts,
             code.co_names,
             code.co_varnames,
             code.co_filename,
             code.co_name,
             code.co_firstlineno,
             code.co_lnotab,
             code.co_freevars,
             code.co_cellvars
         )
     return FunctionType(new_code, globals(), function.__name__, function.__defaults__)
 
 
 # we still need to run things using the asyncio eventloop, but there is no
 # async integration
 from .async_helpers import (_asyncio_runner,  _asyncify, _pseudo_sync_runner)
 from .async_helpers import _curio_runner, _trio_runner, _should_be_async
 
 
 def _ast_asyncify(cell:str, wrapper_name:str) -> ast.Module:
     """
     Parse a cell with top-level await and modify the AST to be able to run it later.
 
     Parameter
     ---------
 
     cell: str
         The code cell to asyncronify
     wrapper_name: str
         The name of the function to be used to wrap the passed `cell`. It is
         advised to **not** use a python identifier in order to not pollute the
         global namespace in which the function will be ran.
 
     Return
     ------
 
     A module object AST containing **one** function named `wrapper_name`.
 
     The given code is wrapped in a async-def function, parsed into an AST, and
     the resulting function definition AST is modified to return the last
     expression.
 
     The last expression or await node is moved into a return statement at the
     end of the function, and removed from its original location. If the last
     node is not Expr or Await nothing is done.
 
     The function `__code__` will need to be later modified  (by
     ``removed_co_newlocals``) in a subsequent step to not create new `locals()`
     meaning that the local and global scope are the same, ie as if the body of
     the function was at module level.
 
     Lastly a call to `locals()` is made just before the last expression of the
     function, or just after the last assignment or statement to make sure the
     global dict is updated as python function work with a local fast cache which
     is updated only on `local()` calls.
     """
 
     from ast import Expr, Await, Return
     if sys.version_info >= (3,8):
         return ast.parse(cell)
     tree = ast.parse(_asyncify(cell))
 
     function_def = tree.body[0]
     function_def.name = wrapper_name
     try_block = function_def.body[0]
     lastexpr = try_block.body[-1]
     if isinstance(lastexpr, (Expr, Await)):
         try_block.body[-1] = Return(lastexpr.value)
     ast.fix_missing_locations(tree)
     return tree
 #-----------------------------------------------------------------------------
 # Globals
 #-----------------------------------------------------------------------------
 
 # compiled regexps for autoindent management
 dedent_re = re.compile(r'^\s+raise|^\s+return|^\s+pass')
 
 #-----------------------------------------------------------------------------
 # Utilities
 #-----------------------------------------------------------------------------
 
 @undoc
 def softspace(file, newvalue):
     """Copied from code.py, to remove the dependency"""
 
     oldvalue = 0
     try:
         oldvalue = file.softspace
     except AttributeError:
         pass
     try:
         file.softspace = newvalue
     except (AttributeError, TypeError):
         # "attribute-less object" or "read-only attributes"
         pass
     return oldvalue
 
 @undoc
 def no_op(*a, **kw):
     pass
 
 
 class SpaceInInput(Exception): pass
 
 
 def get_default_colors():
     "DEPRECATED"
     warn('get_default_color is deprecated since IPython 5.0, and returns `Neutral` on all platforms.',
         DeprecationWarning, stacklevel=2)
     return 'Neutral'
 
 
 class SeparateUnicode(Unicode):
     r"""A Unicode subclass to validate separate_in, separate_out, etc.
 
     This is a Unicode based trait that converts '0'->'' and ``'\\n'->'\n'``.
     """
 
     def validate(self, obj, value):
         if value == '0': value = ''
         value = value.replace('\\n','\n')
         return super(SeparateUnicode, self).validate(obj, value)
 
 
 @undoc
 class DummyMod(object):
     """A dummy module used for IPython's interactive module when
     a namespace must be assigned to the module's __dict__."""
     __spec__ = None
 
 
 class ExecutionInfo(object):
     """The arguments used for a call to :meth:`InteractiveShell.run_cell`
 
     Stores information about what is going to happen.
     """
     raw_cell = None
     store_history = False
     silent = False
     shell_futures = True
 
     def __init__(self, raw_cell, store_history, silent, shell_futures):
         self.raw_cell = raw_cell
         self.store_history = store_history
         self.silent = silent
         self.shell_futures = shell_futures
 
     def __repr__(self):
         name = self.__class__.__qualname__
         raw_cell = ((self.raw_cell[:50] + '..')
                     if len(self.raw_cell) > 50 else self.raw_cell)
         return '<%s object at %x, raw_cell="%s" store_history=%s silent=%s shell_futures=%s>' %\
                (name, id(self), raw_cell, self.store_history, self.silent, self.shell_futures)
 
 
 class ExecutionResult(object):
     """The result of a call to :meth:`InteractiveShell.run_cell`
 
     Stores information about what took place.
     """
     execution_count = None
     error_before_exec = None
     error_in_exec = None
     info = None
     result = None
 
     def __init__(self, info):
         self.info = info
 
     @property
     def success(self):
         return (self.error_before_exec is None) and (self.error_in_exec is None)
 
     def raise_error(self):
         """Reraises error if `success` is `False`, otherwise does nothing"""
         if self.error_before_exec is not None:
             raise self.error_before_exec
         if self.error_in_exec is not None:
             raise self.error_in_exec
 
     def __repr__(self):
         name = self.__class__.__qualname__
         return '<%s object at %x, execution_count=%s error_before_exec=%s error_in_exec=%s info=%s result=%s>' %\
                 (name, id(self), self.execution_count, self.error_before_exec, self.error_in_exec, repr(self.info), repr(self.result))
 
 
 class InteractiveShell(SingletonConfigurable):
     """An enhanced, interactive shell for Python."""
 
     _instance = None
 
     ast_transformers = List([], help=
         """
         A list of ast.NodeTransformer subclass instances, which will be applied
         to user input before code is run.
         """
     ).tag(config=True)
 
     autocall = Enum((0,1,2), default_value=0, help=
         """
         Make IPython automatically call any callable object even if you didn't
         type explicit parentheses. For example, 'str 43' becomes 'str(43)'
         automatically. The value can be '0' to disable the feature, '1' for
         'smart' autocall, where it is not applied if there are no more
         arguments on the line, and '2' for 'full' autocall, where all callable
         objects are automatically called (even if no arguments are present).
         """
     ).tag(config=True)
 
     autoindent = Bool(True, help=
         """
         Autoindent IPython code entered interactively.
         """
     ).tag(config=True)
 
     autoawait = Bool(True, help=
         """
         Automatically run await statement in the top level repl.
         """
     ).tag(config=True)
 
     loop_runner_map ={
         'asyncio':(_asyncio_runner, True),
         'curio':(_curio_runner, True),
         'trio':(_trio_runner, True),
         'sync': (_pseudo_sync_runner, False)
     }
 
     loop_runner = Any(default_value="IPython.core.interactiveshell._asyncio_runner",
         allow_none=True,
         help="""Select the loop runner that will be used to execute top-level asynchronous code"""
     ).tag(config=True)
 
     @default('loop_runner')
     def _default_loop_runner(self):
         return import_item("IPython.core.interactiveshell._asyncio_runner")
 
     @validate('loop_runner')
     def _import_runner(self, proposal):
         if isinstance(proposal.value, str):
             if proposal.value in self.loop_runner_map:
                 runner, autoawait = self.loop_runner_map[proposal.value]
                 self.autoawait = autoawait
                 return runner
             runner = import_item(proposal.value)
             if not callable(runner):
                 raise ValueError('loop_runner must be callable')
             return runner
         if not callable(proposal.value):
             raise ValueError('loop_runner must be callable')
         return proposal.value
 
     automagic = Bool(True, help=
         """
         Enable magic commands to be called without the leading %.
         """
     ).tag(config=True)
 
     banner1 = Unicode(default_banner,
         help="""The part of the banner to be printed before the profile"""
     ).tag(config=True)
     banner2 = Unicode('',
         help="""The part of the banner to be printed after the profile"""
     ).tag(config=True)
 
     cache_size = Integer(1000, help=
         """
         Set the size of the output cache.  The default is 1000, you can
         change it permanently in your config file.  Setting it to 0 completely
         disables the caching system, and the minimum value accepted is 3 (if
         you provide a value less than 3, it is reset to 0 and a warning is
         issued).  This limit is defined because otherwise you'll spend more
         time re-flushing a too small cache than working
         """
     ).tag(config=True)
     color_info = Bool(True, help=
         """
         Use colors for displaying information about objects. Because this
         information is passed through a pager (like 'less'), and some pagers
         get confused with color codes, this capability can be turned off.
         """
     ).tag(config=True)
     colors = CaselessStrEnum(('Neutral', 'NoColor','LightBG','Linux'),
                              default_value='Neutral',
         help="Set the color scheme (NoColor, Neutral, Linux, or LightBG)."
     ).tag(config=True)
     debug = Bool(False).tag(config=True)
     disable_failing_post_execute = Bool(False,
         help="Don't call post-execute functions that have failed in the past."
     ).tag(config=True)
     display_formatter = Instance(DisplayFormatter, allow_none=True)
     displayhook_class = Type(DisplayHook)
     display_pub_class = Type(DisplayPublisher)
     compiler_class = Type(CachingCompiler)
 
     sphinxify_docstring = Bool(False, help=
         """
         Enables rich html representation of docstrings. (This requires the
         docrepr module).
         """).tag(config=True)
 
     @observe("sphinxify_docstring")
     def _sphinxify_docstring_changed(self, change):
         if change['new']:
             warn("`sphinxify_docstring` is provisional since IPython 5.0 and might change in future versions." , ProvisionalWarning)
 
     enable_html_pager = Bool(False, help=
         """
         (Provisional API) enables html representation in mime bundles sent
         to pagers.
         """).tag(config=True)
 
     @observe("enable_html_pager")
     def _enable_html_pager_changed(self, change):
         if change['new']:
             warn("`enable_html_pager` is provisional since IPython 5.0 and might change in future versions.", ProvisionalWarning)
 
     data_pub_class = None
 
     exit_now = Bool(False)
     exiter = Instance(ExitAutocall)
     @default('exiter')
     def _exiter_default(self):
         return ExitAutocall(self)
     # Monotonically increasing execution counter
     execution_count = Integer(1)
     filename = Unicode("<ipython console>")
     ipython_dir= Unicode('').tag(config=True) # Set to get_ipython_dir() in __init__
 
     # Used to transform cells before running them, and check whether code is complete
     input_transformer_manager = Instance('IPython.core.inputtransformer2.TransformerManager',
                                          ())
 
     @property
     def input_transformers_cleanup(self):
         return self.input_transformer_manager.cleanup_transforms
 
     input_transformers_post = List([],
         help="A list of string input transformers, to be applied after IPython's "
              "own input transformations."
     )
 
     @property
     def input_splitter(self):
         """Make this available for backward compatibility (pre-7.0 release) with existing code.
 
         For example, ipykernel ipykernel currently uses
         `shell.input_splitter.check_complete`
         """
         from warnings import warn
         warn("`input_splitter` is deprecated since IPython 7.0, prefer `input_transformer_manager`.",
              DeprecationWarning, stacklevel=2
         )
         return self.input_transformer_manager
 
     logstart = Bool(False, help=
         """
         Start logging to the default log file in overwrite mode.
         Use `logappend` to specify a log file to **append** logs to.
         """
     ).tag(config=True)
     logfile = Unicode('', help=
         """
         The name of the logfile to use.
         """
     ).tag(config=True)
     logappend = Unicode('', help=
         """
         Start logging to the given file in append mode.
         Use `logfile` to specify a log file to **overwrite** logs to.
         """
     ).tag(config=True)
     object_info_string_level = Enum((0,1,2), default_value=0,
     ).tag(config=True)
     pdb = Bool(False, help=
         """
         Automatically call the pdb debugger after every exception.
         """
     ).tag(config=True)
     display_page = Bool(False,
         help="""If True, anything that would be passed to the pager
         will be displayed as regular output instead."""
     ).tag(config=True)
 
     # deprecated prompt traits:
 
     prompt_in1 = Unicode('In [\\#]: ',
         help="Deprecated since IPython 4.0 and ignored since 5.0, set TerminalInteractiveShell.prompts object directly."
     ).tag(config=True)
     prompt_in2 = Unicode('   .\\D.: ',
         help="Deprecated since IPython 4.0 and ignored since 5.0, set TerminalInteractiveShell.prompts object directly."
     ).tag(config=True)
     prompt_out = Unicode('Out[\\#]: ',
         help="Deprecated since IPython 4.0 and ignored since 5.0, set TerminalInteractiveShell.prompts object directly."
     ).tag(config=True)
     prompts_pad_left = Bool(True,
         help="Deprecated since IPython 4.0 and ignored since 5.0, set TerminalInteractiveShell.prompts object directly."
     ).tag(config=True)
 
     @observe('prompt_in1', 'prompt_in2', 'prompt_out', 'prompt_pad_left')
     def _prompt_trait_changed(self, change):
         name = change['name']
         warn("InteractiveShell.{name} is deprecated since IPython 4.0"
              " and ignored since 5.0, set TerminalInteractiveShell.prompts"
              " object directly.".format(name=name))
 
         # protect against weird cases where self.config may not exist:
 
     show_rewritten_input = Bool(True,
         help="Show rewritten input, e.g. for autocall."
     ).tag(config=True)
 
     quiet = Bool(False).tag(config=True)
 
     history_length = Integer(10000,
         help='Total length of command history'
     ).tag(config=True)
 
     history_load_length = Integer(1000, help=
         """
         The number of saved history entries to be loaded
         into the history buffer at startup.
         """
     ).tag(config=True)
 
     ast_node_interactivity = Enum(['all', 'last', 'last_expr', 'none', 'last_expr_or_assign'],
                                   default_value='last_expr',
                                   help="""
         'all', 'last', 'last_expr' or 'none', 'last_expr_or_assign' specifying
         which nodes should be run interactively (displaying output from expressions).
         """
     ).tag(config=True)
 
     # TODO: this part of prompt management should be moved to the frontends.
     # Use custom TraitTypes that convert '0'->'' and '\\n'->'\n'
     separate_in = SeparateUnicode('\n').tag(config=True)
     separate_out = SeparateUnicode('').tag(config=True)
     separate_out2 = SeparateUnicode('').tag(config=True)
     wildcards_case_sensitive = Bool(True).tag(config=True)
     xmode = CaselessStrEnum(('Context', 'Plain', 'Verbose', 'Minimal'),
                             default_value='Context',
                             help="Switch modes for the IPython exception handlers."
                             ).tag(config=True)
 
     # Subcomponents of InteractiveShell
     alias_manager = Instance('IPython.core.alias.AliasManager', allow_none=True)
     prefilter_manager = Instance('IPython.core.prefilter.PrefilterManager', allow_none=True)
     builtin_trap = Instance('IPython.core.builtin_trap.BuiltinTrap', allow_none=True)
     display_trap = Instance('IPython.core.display_trap.DisplayTrap', allow_none=True)
     extension_manager = Instance('IPython.core.extensions.ExtensionManager', allow_none=True)
     payload_manager = Instance('IPython.core.payload.PayloadManager', allow_none=True)
     history_manager = Instance('IPython.core.history.HistoryAccessorBase', allow_none=True)
     magics_manager = Instance('IPython.core.magic.MagicsManager', allow_none=True)
 
     profile_dir = Instance('IPython.core.application.ProfileDir', allow_none=True)
     @property
     def profile(self):
         if self.profile_dir is not None:
             name = os.path.basename(self.profile_dir.location)
             return name.replace('profile_','')
 
 
     # Private interface
     _post_execute = Dict()
 
     # Tracks any GUI loop loaded for pylab
     pylab_gui_select = None
 
     last_execution_succeeded = Bool(True, help='Did last executed command succeeded')
 
     last_execution_result = Instance('IPython.core.interactiveshell.ExecutionResult', help='Result of executing the last command', allow_none=True)
 
     def __init__(self, ipython_dir=None, profile_dir=None,
                  user_module=None, user_ns=None,
                  custom_exceptions=((), None), **kwargs):
-
         # This is where traits with a config_key argument are updated
         # from the values on config.
         super(InteractiveShell, self).__init__(**kwargs)
         if 'PromptManager' in self.config:
             warn('As of IPython 5.0 `PromptManager` config will have no effect'
                  ' and has been replaced by TerminalInteractiveShell.prompts_class')
         self.configurables = [self]
 
         # These are relatively independent and stateless
         self.init_ipython_dir(ipython_dir)
         self.init_profile_dir(profile_dir)
         self.init_instance_attrs()
         self.init_environment()
 
         # Check if we're in a virtualenv, and set up sys.path.
         self.init_virtualenv()
 
         # Create namespaces (user_ns, user_global_ns, etc.)
         self.init_create_namespaces(user_module, user_ns)
         # This has to be done after init_create_namespaces because it uses
         # something in self.user_ns, but before init_sys_modules, which
         # is the first thing to modify sys.
         # TODO: When we override sys.stdout and sys.stderr before this class
         # is created, we are saving the overridden ones here. Not sure if this
         # is what we want to do.
         self.save_sys_module_state()
         self.init_sys_modules()
 
         # While we're trying to have each part of the code directly access what
         # it needs without keeping redundant references to objects, we have too
         # much legacy code that expects ip.db to exist.
         self.db = PickleShareDB(os.path.join(self.profile_dir.location, 'db'))
 
         self.init_history()
         self.init_encoding()
         self.init_prefilter()
 
         self.init_syntax_highlighting()
         self.init_hooks()
         self.init_events()
         self.init_pushd_popd_magic()
         self.init_user_ns()
         self.init_logger()
         self.init_builtins()
 
         # The following was in post_config_initialization
         self.init_inspector()
         self.raw_input_original = input
         self.init_completer()
         # TODO: init_io() needs to happen before init_traceback handlers
         # because the traceback handlers hardcode the stdout/stderr streams.
         # This logic in in debugger.Pdb and should eventually be changed.
         self.init_io()
         self.init_traceback_handlers(custom_exceptions)
         self.init_prompts()
         self.init_display_formatter()
         self.init_display_pub()
         self.init_data_pub()
         self.init_displayhook()
         self.init_magics()
         self.init_alias()
         self.init_logstart()
         self.init_pdb()
         self.init_extension_manager()
         self.init_payload()
         self.init_deprecation_warnings()
         self.hooks.late_startup_hook()
         self.events.trigger('shell_initialized', self)
         atexit.register(self.atexit_operations)
 
         # The trio runner is used for running Trio in the foreground thread. It
         # is different from `_trio_runner(async_fn)` in `async_helpers.py`
         # which calls `trio.run()` for every cell. This runner runs all cells
         # inside a single Trio event loop. If used, it is set from
         # `ipykernel.kernelapp`.
         self.trio_runner = None
 
     def get_ipython(self):
         """Return the currently running IPython instance."""
         return self
 
     #-------------------------------------------------------------------------
     # Trait changed handlers
     #-------------------------------------------------------------------------
     @observe('ipython_dir')
     def _ipython_dir_changed(self, change):
         ensure_dir_exists(change['new'])
 
     def set_autoindent(self,value=None):
         """Set the autoindent flag.
 
         If called with no arguments, it acts as a toggle."""
         if value is None:
             self.autoindent = not self.autoindent
         else:
             self.autoindent = value
 
     def set_trio_runner(self, tr):
         self.trio_runner = tr
 
     #-------------------------------------------------------------------------
     # init_* methods called by __init__
     #-------------------------------------------------------------------------
 
     def init_ipython_dir(self, ipython_dir):
         if ipython_dir is not None:
             self.ipython_dir = ipython_dir
             return
 
         self.ipython_dir = get_ipython_dir()
 
     def init_profile_dir(self, profile_dir):
         if profile_dir is not None:
             self.profile_dir = profile_dir
             return
         self.profile_dir = ProfileDir.create_profile_dir_by_name(
             self.ipython_dir, "default"
         )
 
     def init_instance_attrs(self):
         self.more = False
 
         # command compiler
         self.compile = self.compiler_class()
 
         # Make an empty namespace, which extension writers can rely on both
         # existing and NEVER being used by ipython itself.  This gives them a
         # convenient location for storing additional information and state
         # their extensions may require, without fear of collisions with other
         # ipython names that may develop later.
         self.meta = Struct()
 
         # Temporary files used for various purposes.  Deleted at exit.
         self.tempfiles = []
         self.tempdirs = []
 
         # keep track of where we started running (mainly for crash post-mortem)
         # This is not being used anywhere currently.
         self.starting_dir = os.getcwd()
 
         # Indentation management
         self.indent_current_nsp = 0
 
         # Dict to track post-execution functions that have been registered
         self._post_execute = {}
 
     def init_environment(self):
         """Any changes we need to make to the user's environment."""
         pass
 
     def init_encoding(self):
         # Get system encoding at startup time.  Certain terminals (like Emacs
         # under Win32 have it set to None, and we need to have a known valid
         # encoding to use in the raw_input() method
         try:
             self.stdin_encoding = sys.stdin.encoding or 'ascii'
         except AttributeError:
             self.stdin_encoding = 'ascii'
 
 
     @observe('colors')
     def init_syntax_highlighting(self, changes=None):
         # Python source parser/formatter for syntax highlighting
         pyformat = PyColorize.Parser(style=self.colors, parent=self).format
         self.pycolorize = lambda src: pyformat(src,'str')
 
     def refresh_style(self):
         # No-op here, used in subclass
         pass
 
     def init_pushd_popd_magic(self):
         # for pushd/popd management
         self.home_dir = get_home_dir()
 
         self.dir_stack = []
 
     def init_logger(self):
         self.logger = Logger(self.home_dir, logfname='ipython_log.py',
                              logmode='rotate')
 
     def init_logstart(self):
         """Initialize logging in case it was requested at the command line.
         """
         if self.logappend:
             self.magic('logstart %s append' % self.logappend)
         elif self.logfile:
             self.magic('logstart %s' % self.logfile)
         elif self.logstart:
             self.magic('logstart')
 
     def init_deprecation_warnings(self):
         """
         register default filter for deprecation warning.
 
         This will allow deprecation warning of function used interactively to show
         warning to users, and still hide deprecation warning from libraries import.
         """
         if sys.version_info < (3,7):
             warnings.filterwarnings("default", category=DeprecationWarning, module=self.user_ns.get("__name__"))
 
 
     def init_builtins(self):
         # A single, static flag that we set to True.  Its presence indicates
         # that an IPython shell has been created, and we make no attempts at
         # removing on exit or representing the existence of more than one
         # IPython at a time.
         builtin_mod.__dict__['__IPYTHON__'] = True
         builtin_mod.__dict__['display'] = display
 
         self.builtin_trap = BuiltinTrap(shell=self)
 
     @observe('colors')
     def init_inspector(self, changes=None):
         # Object inspector
         self.inspector = oinspect.Inspector(oinspect.InspectColors,
                                             PyColorize.ANSICodeColors,
                                             self.colors,
                                             self.object_info_string_level)
 
     def init_io(self):
         # This will just use sys.stdout and sys.stderr. If you want to
         # override sys.stdout and sys.stderr themselves, you need to do that
         # *before* instantiating this class, because io holds onto
         # references to the underlying streams.
         # io.std* are deprecated, but don't show our own deprecation warnings
         # during initialization of the deprecated API.
         with warnings.catch_warnings():
             warnings.simplefilter('ignore', DeprecationWarning)
             io.stdout = io.IOStream(sys.stdout)
             io.stderr = io.IOStream(sys.stderr)
 
     def init_prompts(self):
         # Set system prompts, so that scripts can decide if they are running
         # interactively.
         sys.ps1 = 'In : '
         sys.ps2 = '...: '
         sys.ps3 = 'Out: '
 
     def init_display_formatter(self):
         self.display_formatter = DisplayFormatter(parent=self)
         self.configurables.append(self.display_formatter)
 
     def init_display_pub(self):
         self.display_pub = self.display_pub_class(parent=self, shell=self)
         self.configurables.append(self.display_pub)
 
     def init_data_pub(self):
         if not self.data_pub_class:
             self.data_pub = None
             return
         self.data_pub = self.data_pub_class(parent=self)
         self.configurables.append(self.data_pub)
 
     def init_displayhook(self):
         # Initialize displayhook, set in/out prompts and printing system
         self.displayhook = self.displayhook_class(
             parent=self,
             shell=self,
             cache_size=self.cache_size,
         )
         self.configurables.append(self.displayhook)
         # This is a context manager that installs/revmoes the displayhook at
         # the appropriate time.
         self.display_trap = DisplayTrap(hook=self.displayhook)
 
     def init_virtualenv(self):
         """Add the current virtualenv to sys.path so the user can import modules from it.
         This isn't perfect: it doesn't use the Python interpreter with which the
         virtualenv was built, and it ignores the --no-site-packages option. A
         warning will appear suggesting the user installs IPython in the
         virtualenv, but for many cases, it probably works well enough.
         Adapted from code snippets online.
         http://blog.ufsoft.org/2009/1/29/ipython-and-virtualenv
         """
         if 'VIRTUAL_ENV' not in os.environ:
             # Not in a virtualenv
             return
         elif os.environ["VIRTUAL_ENV"] == "":
             warn("Virtual env path set to '', please check if this is intended.")
             return
 
         p = Path(sys.executable)
         p_venv = Path(os.environ["VIRTUAL_ENV"])
 
         # fallback venv detection:
         # stdlib venv may symlink sys.executable, so we can't use realpath.
         # but others can symlink *to* the venv Python, so we can't just use sys.executable.
         # So we just check every item in the symlink tree (generally <= 3)
         paths = [p]
         while p.is_symlink():
             p = Path(os.readlink(p))
             paths.append(p.resolve())
 
         # In Cygwin paths like "c:\..." and '\cygdrive\c\...' are possible
         if p_venv.parts[1] == "cygdrive":
             drive_name = p_venv.parts[2]
             p_venv = (drive_name + ":/") / Path(*p_venv.parts[3:])
 
         if any(p_venv == p.parents[1] for p in paths):
             # Our exe is inside or has access to the virtualenv, don't need to do anything.
             return
 
         if sys.platform == "win32":
             virtual_env = str(Path(os.environ["VIRTUAL_ENV"], "Lib", "site-packages"))
         else:
             virtual_env_path = Path(
                 os.environ["VIRTUAL_ENV"], "lib", "python{}.{}", "site-packages"
             )
             p_ver = sys.version_info[:2]
 
             # Predict version from py[thon]-x.x in the $VIRTUAL_ENV
             re_m = re.search(r"\bpy(?:thon)?([23])\.(\d+)\b", os.environ["VIRTUAL_ENV"])
             if re_m:
                 predicted_path = Path(str(virtual_env_path).format(*re_m.groups()))
                 if predicted_path.exists():
                     p_ver = re_m.groups()
 
             virtual_env = str(virtual_env_path).format(*p_ver)
 
         warn(
             "Attempting to work in a virtualenv. If you encounter problems, "
             "please install IPython inside the virtualenv."
         )
         import site
         sys.path.insert(0, virtual_env)
         site.addsitedir(virtual_env)
 
     #-------------------------------------------------------------------------
     # Things related to injections into the sys module
     #-------------------------------------------------------------------------
 
     def save_sys_module_state(self):
         """Save the state of hooks in the sys module.
 
         This has to be called after self.user_module is created.
         """
         self._orig_sys_module_state = {'stdin': sys.stdin,
                                        'stdout': sys.stdout,
                                        'stderr': sys.stderr,
                                        'excepthook': sys.excepthook}
         self._orig_sys_modules_main_name = self.user_module.__name__
         self._orig_sys_modules_main_mod = sys.modules.get(self.user_module.__name__)
 
     def restore_sys_module_state(self):
         """Restore the state of the sys module."""
         try:
             for k, v in self._orig_sys_module_state.items():
                 setattr(sys, k, v)
         except AttributeError:
             pass
         # Reset what what done in self.init_sys_modules
         if self._orig_sys_modules_main_mod is not None:
             sys.modules[self._orig_sys_modules_main_name] = self._orig_sys_modules_main_mod
 
     #-------------------------------------------------------------------------
     # Things related to the banner
     #-------------------------------------------------------------------------
 
     @property
     def banner(self):
         banner = self.banner1
         if self.profile and self.profile != 'default':
             banner += '\nIPython profile: %s\n' % self.profile
         if self.banner2:
             banner += '\n' + self.banner2
         return banner
 
     def show_banner(self, banner=None):
         if banner is None:
             banner = self.banner
         sys.stdout.write(banner)
 
     #-------------------------------------------------------------------------
     # Things related to hooks
     #-------------------------------------------------------------------------
 
     def init_hooks(self):
         # hooks holds pointers used for user-side customizations
         self.hooks = Struct()
 
         self.strdispatchers = {}
 
         # Set all default hooks, defined in the IPython.hooks module.
         hooks = IPython.core.hooks
         for hook_name in hooks.__all__:
             # default hooks have priority 100, i.e. low; user hooks should have
             # 0-100 priority
             self.set_hook(hook_name,getattr(hooks,hook_name), 100, _warn_deprecated=False)
 
         if self.display_page:
             self.set_hook('show_in_pager', page.as_hook(page.display_page), 90)
 
     def set_hook(self,name,hook, priority=50, str_key=None, re_key=None,
                  _warn_deprecated=True):
         """set_hook(name,hook) -> sets an internal IPython hook.
 
         IPython exposes some of its internal API as user-modifiable hooks.  By
         adding your function to one of these hooks, you can modify IPython's
         behavior to call at runtime your own routines."""
 
         # At some point in the future, this should validate the hook before it
         # accepts it.  Probably at least check that the hook takes the number
         # of args it's supposed to.
 
         f = types.MethodType(hook,self)
 
         # check if the hook is for strdispatcher first
         if str_key is not None:
             sdp = self.strdispatchers.get(name, StrDispatch())
             sdp.add_s(str_key, f, priority )
             self.strdispatchers[name] = sdp
             return
         if re_key is not None:
             sdp = self.strdispatchers.get(name, StrDispatch())
             sdp.add_re(re.compile(re_key), f, priority )
             self.strdispatchers[name] = sdp
             return
 
         dp = getattr(self.hooks, name, None)
         if name not in IPython.core.hooks.__all__:
             print("Warning! Hook '%s' is not one of %s" % \
                   (name, IPython.core.hooks.__all__ ))
 
         if _warn_deprecated and (name in IPython.core.hooks.deprecated):
             alternative = IPython.core.hooks.deprecated[name]
             warn("Hook {} is deprecated. Use {} instead.".format(name, alternative), stacklevel=2)
 
         if not dp:
             dp = IPython.core.hooks.CommandChainDispatcher()
 
         try:
             dp.add(f,priority)
         except AttributeError:
             # it was not commandchain, plain old func - replace
             dp = f
 
         setattr(self.hooks,name, dp)
 
     #-------------------------------------------------------------------------
     # Things related to events
     #-------------------------------------------------------------------------
 
     def init_events(self):
         self.events = EventManager(self, available_events)
 
         self.events.register("pre_execute", self._clear_warning_registry)
 
     def register_post_execute(self, func):
         """DEPRECATED: Use ip.events.register('post_run_cell', func)
         
         Register a function for calling after code execution.
         """
         warn("ip.register_post_execute is deprecated, use "
              "ip.events.register('post_run_cell', func) instead.", stacklevel=2)
         self.events.register('post_run_cell', func)
 
     def _clear_warning_registry(self):
         # clear the warning registry, so that different code blocks with
         # overlapping line number ranges don't cause spurious suppression of
         # warnings (see gh-6611 for details)
         if "__warningregistry__" in self.user_global_ns:
             del self.user_global_ns["__warningregistry__"]
 
     #-------------------------------------------------------------------------
     # Things related to the "main" module
     #-------------------------------------------------------------------------
 
     def new_main_mod(self, filename, modname):
         """Return a new 'main' module object for user code execution.
         
         ``filename`` should be the path of the script which will be run in the
         module. Requests with the same filename will get the same module, with
         its namespace cleared.
         
         ``modname`` should be the module name - normally either '__main__' or
         the basename of the file without the extension.
         
         When scripts are executed via %run, we must keep a reference to their
         __main__ module around so that Python doesn't
         clear it, rendering references to module globals useless.
 
         This method keeps said reference in a private dict, keyed by the
         absolute path of the script. This way, for multiple executions of the
         same script we only keep one copy of the namespace (the last one),
         thus preventing memory leaks from old references while allowing the
         objects from the last execution to be accessible.
         """
         filename = os.path.abspath(filename)
         try:
             main_mod = self._main_mod_cache[filename]
         except KeyError:
             main_mod = self._main_mod_cache[filename] = types.ModuleType(
                         modname,
                         doc="Module created for script run in IPython")
         else:
             main_mod.__dict__.clear()
             main_mod.__name__ = modname
 
         main_mod.__file__ = filename
         # It seems pydoc (and perhaps others) needs any module instance to
         # implement a __nonzero__ method
         main_mod.__nonzero__ = lambda : True
 
         return main_mod
 
     def clear_main_mod_cache(self):
         """Clear the cache of main modules.
 
         Mainly for use by utilities like %reset.
 
         Examples
         --------
 
         In [15]: import IPython
 
         In [16]: m = _ip.new_main_mod(IPython.__file__, 'IPython')
 
         In [17]: len(_ip._main_mod_cache) > 0
         Out[17]: True
 
         In [18]: _ip.clear_main_mod_cache()
 
         In [19]: len(_ip._main_mod_cache) == 0
         Out[19]: True
         """
         self._main_mod_cache.clear()
 
     #-------------------------------------------------------------------------
     # Things related to debugging
     #-------------------------------------------------------------------------
 
     def init_pdb(self):
         # Set calling of pdb on exceptions
         # self.call_pdb is a property
         self.call_pdb = self.pdb
 
     def _get_call_pdb(self):
         return self._call_pdb
 
     def _set_call_pdb(self,val):
 
         if val not in (0,1,False,True):
             raise ValueError('new call_pdb value must be boolean')
 
         # store value in instance
         self._call_pdb = val
 
         # notify the actual exception handlers
         self.InteractiveTB.call_pdb = val
 
     call_pdb = property(_get_call_pdb,_set_call_pdb,None,
                         'Control auto-activation of pdb at exceptions')
 
     def debugger(self,force=False):
         """Call the pdb debugger.
 
         Keywords:
 
           - force(False): by default, this routine checks the instance call_pdb
             flag and does not actually invoke the debugger if the flag is false.
             The 'force' option forces the debugger to activate even if the flag
             is false.
         """
 
         if not (force or self.call_pdb):
             return
 
         if not hasattr(sys,'last_traceback'):
             error('No traceback has been produced, nothing to debug.')
             return
 
         self.InteractiveTB.debugger(force=True)
 
     #-------------------------------------------------------------------------
     # Things related to IPython's various namespaces
     #-------------------------------------------------------------------------
     default_user_namespaces = True
 
     def init_create_namespaces(self, user_module=None, user_ns=None):
         # Create the namespace where the user will operate.  user_ns is
         # normally the only one used, and it is passed to the exec calls as
         # the locals argument.  But we do carry a user_global_ns namespace
         # given as the exec 'globals' argument,  This is useful in embedding
         # situations where the ipython shell opens in a context where the
         # distinction between locals and globals is meaningful.  For
         # non-embedded contexts, it is just the same object as the user_ns dict.
 
         # FIXME. For some strange reason, __builtins__ is showing up at user
         # level as a dict instead of a module. This is a manual fix, but I
         # should really track down where the problem is coming from. Alex
         # Schmolck reported this problem first.
 
         # A useful post by Alex Martelli on this topic:
         # Re: inconsistent value from __builtins__
         # Von: Alex Martelli <[e-mail]>
         # Datum: Freitag 01 Oktober 2004 04:45:34 nachmittags/abends
         # Gruppen: comp.lang.python
 
         # Michael Hohn <[e-mail]> wrote:
         # > >>> print type(builtin_check.get_global_binding('__builtins__'))
         # > <type 'dict'>
         # > >>> print type(__builtins__)
         # > <type 'module'>
         # > Is this difference in return value intentional?
 
         # Well, it's documented that '__builtins__' can be either a dictionary
         # or a module, and it's been that way for a long time. Whether it's
         # intentional (or sensible), I don't know. In any case, the idea is
         # that if you need to access the built-in namespace directly, you
         # should start with "import __builtin__" (note, no 's') which will
         # definitely give you a module. Yeah, it's somewhat confusing:-(.
 
         # These routines return a properly built module and dict as needed by
         # the rest of the code, and can also be used by extension writers to
         # generate properly initialized namespaces.
         if (user_ns is not None) or (user_module is not None):
             self.default_user_namespaces = False
         self.user_module, self.user_ns = self.prepare_user_module(user_module, user_ns)
 
         # A record of hidden variables we have added to the user namespace, so
         # we can list later only variables defined in actual interactive use.
         self.user_ns_hidden = {}
 
         # Now that FakeModule produces a real module, we've run into a nasty
         # problem: after script execution (via %run), the module where the user
         # code ran is deleted.  Now that this object is a true module (needed
         # so doctest and other tools work correctly), the Python module
         # teardown mechanism runs over it, and sets to None every variable
         # present in that module.  Top-level references to objects from the
         # script survive, because the user_ns is updated with them.  However,
         # calling functions defined in the script that use other things from
         # the script will fail, because the function's closure had references
         # to the original objects, which are now all None.  So we must protect
         # these modules from deletion by keeping a cache.
         #
         # To avoid keeping stale modules around (we only need the one from the
         # last run), we use a dict keyed with the full path to the script, so
         # only the last version of the module is held in the cache.  Note,
         # however, that we must cache the module *namespace contents* (their
         # __dict__).  Because if we try to cache the actual modules, old ones
         # (uncached) could be destroyed while still holding references (such as
         # those held by GUI objects that tend to be long-lived)>
         #
         # The %reset command will flush this cache.  See the cache_main_mod()
         # and clear_main_mod_cache() methods for details on use.
 
         # This is the cache used for 'main' namespaces
         self._main_mod_cache = {}
 
         # A table holding all the namespaces IPython deals with, so that
         # introspection facilities can search easily.
         self.ns_table = {'user_global':self.user_module.__dict__,
                          'user_local':self.user_ns,
                          'builtin':builtin_mod.__dict__
                          }
 
     @property
     def user_global_ns(self):
         return self.user_module.__dict__
 
     def prepare_user_module(self, user_module=None, user_ns=None):
         """Prepare the module and namespace in which user code will be run.
         
         When IPython is started normally, both parameters are None: a new module
         is created automatically, and its __dict__ used as the namespace.
         
         If only user_module is provided, its __dict__ is used as the namespace.
         If only user_ns is provided, a dummy module is created, and user_ns
         becomes the global namespace. If both are provided (as they may be
         when embedding), user_ns is the local namespace, and user_module
         provides the global namespace.
 
         Parameters
         ----------
         user_module : module, optional
             The current user module in which IPython is being run. If None,
             a clean module will be created.
         user_ns : dict, optional
             A namespace in which to run interactive commands.
 
         Returns
         -------
         A tuple of user_module and user_ns, each properly initialised.
         """
         if user_module is None and user_ns is not None:
             user_ns.setdefault("__name__", "__main__")
             user_module = DummyMod()
             user_module.__dict__ = user_ns
 
         if user_module is None:
             user_module = types.ModuleType("__main__",
                 doc="Automatically created module for IPython interactive environment")
 
         # We must ensure that __builtin__ (without the final 's') is always
         # available and pointing to the __builtin__ *module*.  For more details:
         # http://mail.python.org/pipermail/python-dev/2001-April/014068.html
         user_module.__dict__.setdefault('__builtin__', builtin_mod)
         user_module.__dict__.setdefault('__builtins__', builtin_mod)
 
         if user_ns is None:
             user_ns = user_module.__dict__
 
         return user_module, user_ns
 
     def init_sys_modules(self):
         # We need to insert into sys.modules something that looks like a
         # module but which accesses the IPython namespace, for shelve and
         # pickle to work interactively. Normally they rely on getting
         # everything out of __main__, but for embedding purposes each IPython
         # instance has its own private namespace, so we can't go shoving
         # everything into __main__.
 
         # note, however, that we should only do this for non-embedded
         # ipythons, which really mimic the __main__.__dict__ with their own
         # namespace.  Embedded instances, on the other hand, should not do
         # this because they need to manage the user local/global namespaces
         # only, but they live within a 'normal' __main__ (meaning, they
         # shouldn't overtake the execution environment of the script they're
         # embedded in).
 
         # This is overridden in the InteractiveShellEmbed subclass to a no-op.
         main_name = self.user_module.__name__
         sys.modules[main_name] = self.user_module
 
     def init_user_ns(self):
         """Initialize all user-visible namespaces to their minimum defaults.
 
         Certain history lists are also initialized here, as they effectively
         act as user namespaces.
 
         Notes
         -----
         All data structures here are only filled in, they are NOT reset by this
         method.  If they were not empty before, data will simply be added to
         them.
         """
         # This function works in two parts: first we put a few things in
         # user_ns, and we sync that contents into user_ns_hidden so that these
         # initial variables aren't shown by %who.  After the sync, we add the
         # rest of what we *do* want the user to see with %who even on a new
         # session (probably nothing, so they really only see their own stuff)
 
         # The user dict must *always* have a __builtin__ reference to the
         # Python standard __builtin__ namespace,  which must be imported.
         # This is so that certain operations in prompt evaluation can be
         # reliably executed with builtins.  Note that we can NOT use
         # __builtins__ (note the 's'),  because that can either be a dict or a
         # module, and can even mutate at runtime, depending on the context
         # (Python makes no guarantees on it).  In contrast, __builtin__ is
         # always a module object, though it must be explicitly imported.
 
         # For more details:
         # http://mail.python.org/pipermail/python-dev/2001-April/014068.html
         ns = {}
 
         # make global variables for user access to the histories
         ns['_ih'] = self.history_manager.input_hist_parsed
         ns['_oh'] = self.history_manager.output_hist
         ns['_dh'] = self.history_manager.dir_hist
 
         # user aliases to input and output histories.  These shouldn't show up
         # in %who, as they can have very large reprs.
         ns['In']  = self.history_manager.input_hist_parsed
         ns['Out'] = self.history_manager.output_hist
 
         # Store myself as the public api!!!
         ns['get_ipython'] = self.get_ipython
 
         ns['exit'] = self.exiter
         ns['quit'] = self.exiter
 
         # Sync what we've added so far to user_ns_hidden so these aren't seen
         # by %who
         self.user_ns_hidden.update(ns)
 
         # Anything put into ns now would show up in %who.  Think twice before
         # putting anything here, as we really want %who to show the user their
         # stuff, not our variables.
 
         # Finally, update the real user's namespace
         self.user_ns.update(ns)
 
     @property
     def all_ns_refs(self):
         """Get a list of references to all the namespace dictionaries in which
         IPython might store a user-created object.
         
         Note that this does not include the displayhook, which also caches
         objects from the output."""
         return [self.user_ns, self.user_global_ns, self.user_ns_hidden] + \
                [m.__dict__ for m in self._main_mod_cache.values()]
 
     def reset(self, new_session=True, aggressive=False):
         """Clear all internal namespaces, and attempt to release references to
         user objects.
 
         If new_session is True, a new history session will be opened.
         """
         # Clear histories
         self.history_manager.reset(new_session)
         # Reset counter used to index all histories
         if new_session:
             self.execution_count = 1
 
         # Reset last execution result
         self.last_execution_succeeded = True
         self.last_execution_result = None
 
         # Flush cached output items
         if self.displayhook.do_full_cache:
             self.displayhook.flush()
 
         # The main execution namespaces must be cleared very carefully,
         # skipping the deletion of the builtin-related keys, because doing so
         # would cause errors in many object's __del__ methods.
         if self.user_ns is not self.user_global_ns:
             self.user_ns.clear()
         ns = self.user_global_ns
         drop_keys = set(ns.keys())
         drop_keys.discard('__builtin__')
         drop_keys.discard('__builtins__')
         drop_keys.discard('__name__')
         for k in drop_keys:
             del ns[k]
 
         self.user_ns_hidden.clear()
 
         # Restore the user namespaces to minimal usability
         self.init_user_ns()
         if aggressive and not hasattr(self, "_sys_modules_keys"):
             print("Cannot restore sys.module, no snapshot")
         elif aggressive:
             print("culling sys module...")
             current_keys = set(sys.modules.keys())
             for k in current_keys - self._sys_modules_keys:
                 if k.startswith("multiprocessing"):
                     continue
                 del sys.modules[k]
 
         # Restore the default and user aliases
         self.alias_manager.clear_aliases()
         self.alias_manager.init_aliases()
 
         # Now define aliases that only make sense on the terminal, because they
         # need direct access to the console in a way that we can't emulate in
         # GUI or web frontend
         if os.name == 'posix':
             for cmd in ('clear', 'more', 'less', 'man'):
                 if cmd not in self.magics_manager.magics['line']:
                     self.alias_manager.soft_define_alias(cmd, cmd)
 
         # Flush the private list of module references kept for script
         # execution protection
         self.clear_main_mod_cache()
 
     def del_var(self, varname, by_name=False):
         """Delete a variable from the various namespaces, so that, as
         far as possible, we're not keeping any hidden references to it.
 
         Parameters
         ----------
         varname : str
             The name of the variable to delete.
         by_name : bool
             If True, delete variables with the given name in each
             namespace. If False (default), find the variable in the user
             namespace, and delete references to it.
         """
         if varname in ('__builtin__', '__builtins__'):
             raise ValueError("Refusing to delete %s" % varname)
 
         ns_refs = self.all_ns_refs
 
         if by_name:                    # Delete by name
             for ns in ns_refs:
                 try:
                     del ns[varname]
                 except KeyError:
                     pass
         else:                         # Delete by object
             try:
                 obj = self.user_ns[varname]
             except KeyError:
                 raise NameError("name '%s' is not defined" % varname)
             # Also check in output history
             ns_refs.append(self.history_manager.output_hist)
             for ns in ns_refs:
                 to_delete = [n for n, o in ns.items() if o is obj]
                 for name in to_delete:
                     del ns[name]
 
             # Ensure it is removed from the last execution result
             if self.last_execution_result.result is obj:
                 self.last_execution_result = None
 
             # displayhook keeps extra references, but not in a dictionary
             for name in ('_', '__', '___'):
                 if getattr(self.displayhook, name) is obj:
                     setattr(self.displayhook, name, None)
 
     def reset_selective(self, regex=None):
         """Clear selective variables from internal namespaces based on a
         specified regular expression.
 
         Parameters
         ----------
         regex : string or compiled pattern, optional
             A regular expression pattern that will be used in searching
             variable names in the users namespaces.
         """
         if regex is not None:
             try:
                 m = re.compile(regex)
             except TypeError:
                 raise TypeError('regex must be a string or compiled pattern')
             # Search for keys in each namespace that match the given regex
             # If a match is found, delete the key/value pair.
             for ns in self.all_ns_refs:
                 for var in ns:
                     if m.search(var):
                         del ns[var]
 
     def push(self, variables, interactive=True):
         """Inject a group of variables into the IPython user namespace.
 
         Parameters
         ----------
         variables : dict, str or list/tuple of str
             The variables to inject into the user's namespace.  If a dict, a
             simple update is done.  If a str, the string is assumed to have
             variable names separated by spaces.  A list/tuple of str can also
             be used to give the variable names.  If just the variable names are
             give (list/tuple/str) then the variable values looked up in the
             callers frame.
         interactive : bool
             If True (default), the variables will be listed with the ``who``
             magic.
         """
         vdict = None
 
         # We need a dict of name/value pairs to do namespace updates.
         if isinstance(variables, dict):
             vdict = variables
         elif isinstance(variables, (str, list, tuple)):
             if isinstance(variables, str):
                 vlist = variables.split()
             else:
                 vlist = variables
             vdict = {}
             cf = sys._getframe(1)
             for name in vlist:
                 try:
                     vdict[name] = eval(name, cf.f_globals, cf.f_locals)
                 except:
                     print('Could not get variable %s from %s' %
                            (name,cf.f_code.co_name))
         else:
             raise ValueError('variables must be a dict/str/list/tuple')
 
         # Propagate variables to user namespace
         self.user_ns.update(vdict)
 
         # And configure interactive visibility
         user_ns_hidden = self.user_ns_hidden
         if interactive:
             for name in vdict:
                 user_ns_hidden.pop(name, None)
         else:
             user_ns_hidden.update(vdict)
 
     def drop_by_id(self, variables):
         """Remove a dict of variables from the user namespace, if they are the
         same as the values in the dictionary.
         
         This is intended for use by extensions: variables that they've added can
         be taken back out if they are unloaded, without removing any that the
         user has overwritten.
         
         Parameters
         ----------
         variables : dict
           A dictionary mapping object names (as strings) to the objects.
         """
         for name, obj in variables.items():
             if name in self.user_ns and self.user_ns[name] is obj:
                 del self.user_ns[name]
                 self.user_ns_hidden.pop(name, None)
 
     #-------------------------------------------------------------------------
     # Things related to object introspection
     #-------------------------------------------------------------------------
 
     def _ofind(self, oname, namespaces=None):
         """Find an object in the available namespaces.
 
         self._ofind(oname) -> dict with keys: found,obj,ospace,ismagic
 
         Has special code to detect magic functions.
         """
         oname = oname.strip()
         if not oname.startswith(ESC_MAGIC) and \
                 not oname.startswith(ESC_MAGIC2) and \
                 not all(a.isidentifier() for a in oname.split(".")):
             return {'found': False}
 
         if namespaces is None:
             # Namespaces to search in:
             # Put them in a list. The order is important so that we
             # find things in the same order that Python finds them.
             namespaces = [ ('Interactive', self.user_ns),
                            ('Interactive (global)', self.user_global_ns),
                            ('Python builtin', builtin_mod.__dict__),
                            ]
 
         ismagic = False
         isalias = False
         found = False
         ospace = None
         parent = None
         obj = None
 
 
         # Look for the given name by splitting it in parts.  If the head is
         # found, then we look for all the remaining parts as members, and only
         # declare success if we can find them all.
         oname_parts = oname.split('.')
         oname_head, oname_rest = oname_parts[0],oname_parts[1:]
         for nsname,ns in namespaces:
             try:
                 obj = ns[oname_head]
             except KeyError:
                 continue
             else:
                 for idx, part in enumerate(oname_rest):
                     try:
                         parent = obj
                         # The last part is looked up in a special way to avoid
                         # descriptor invocation as it may raise or have side
                         # effects.
                         if idx == len(oname_rest) - 1:
                             obj = self._getattr_property(obj, part)
                         else:
                             obj = getattr(obj, part)
                     except:
                         # Blanket except b/c some badly implemented objects
                         # allow __getattr__ to raise exceptions other than
                         # AttributeError, which then crashes IPython.
                         break
                 else:
                     # If we finish the for loop (no break), we got all members
                     found = True
                     ospace = nsname
                     break  # namespace loop
 
         # Try to see if it's magic
         if not found:
             obj = None
             if oname.startswith(ESC_MAGIC2):
                 oname = oname.lstrip(ESC_MAGIC2)
                 obj = self.find_cell_magic(oname)
             elif oname.startswith(ESC_MAGIC):
                 oname = oname.lstrip(ESC_MAGIC)
                 obj = self.find_line_magic(oname)
             else:
                 # search without prefix, so run? will find %run?
                 obj = self.find_line_magic(oname)
                 if obj is None:
                     obj = self.find_cell_magic(oname)
             if obj is not None:
                 found = True
                 ospace = 'IPython internal'
                 ismagic = True
                 isalias = isinstance(obj, Alias)
 
         # Last try: special-case some literals like '', [], {}, etc:
         if not found and oname_head in ["''",'""','[]','{}','()']:
             obj = eval(oname_head)
             found = True
             ospace = 'Interactive'
 
         return {
                 'obj':obj,
                 'found':found,
                 'parent':parent,
                 'ismagic':ismagic,
                 'isalias':isalias,
                 'namespace':ospace
                }
 
     @staticmethod
     def _getattr_property(obj, attrname):
         """Property-aware getattr to use in object finding.
 
         If attrname represents a property, return it unevaluated (in case it has
         side effects or raises an error.
 
         """
         if not isinstance(obj, type):
             try:
                 # `getattr(type(obj), attrname)` is not guaranteed to return
                 # `obj`, but does so for property:
                 #
                 # property.__get__(self, None, cls) -> self
                 #
                 # The universal alternative is to traverse the mro manually
                 # searching for attrname in class dicts.
                 attr = getattr(type(obj), attrname)
             except AttributeError:
                 pass
             else:
                 # This relies on the fact that data descriptors (with both
                 # __get__ & __set__ magic methods) take precedence over
                 # instance-level attributes:
                 #
                 #    class A(object):
                 #        @property
                 #        def foobar(self): return 123
                 #    a = A()
                 #    a.__dict__['foobar'] = 345
                 #    a.foobar  # == 123
                 #
                 # So, a property may be returned right away.
                 if isinstance(attr, property):
                     return attr
 
         # Nothing helped, fall back.
         return getattr(obj, attrname)
 
     def _object_find(self, oname, namespaces=None):
         """Find an object and return a struct with info about it."""
         return Struct(self._ofind(oname, namespaces))
 
     def _inspect(self, meth, oname, namespaces=None, **kw):
         """Generic interface to the inspector system.
 
         This function is meant to be called by pdef, pdoc & friends.
         """
         info = self._object_find(oname, namespaces)
         docformat = (
             sphinxify(self.object_inspect(oname)) if self.sphinxify_docstring else None
         )
         if info.found:
             pmethod = getattr(self.inspector, meth)
             # TODO: only apply format_screen to the plain/text repr of the mime
             # bundle.
             formatter = format_screen if info.ismagic else docformat
             if meth == 'pdoc':
                 pmethod(info.obj, oname, formatter)
             elif meth == 'pinfo':
                 pmethod(
                     info.obj,
                     oname,
                     formatter,
                     info,
                     enable_html_pager=self.enable_html_pager,
-                    **kw
+                    **kw,
                 )
             else:
                 pmethod(info.obj, oname)
         else:
             print('Object `%s` not found.' % oname)
             return 'not found'  # so callers can take other action
 
     def object_inspect(self, oname, detail_level=0):
         """Get object info about oname"""
         with self.builtin_trap:
             info = self._object_find(oname)
             if info.found:
                 return self.inspector.info(info.obj, oname, info=info,
                             detail_level=detail_level
                 )
             else:
                 return oinspect.object_info(name=oname, found=False)
 
     def object_inspect_text(self, oname, detail_level=0):
         """Get object info as formatted text"""
         return self.object_inspect_mime(oname, detail_level)['text/plain']
 
     def object_inspect_mime(self, oname, detail_level=0):
         """Get object info as a mimebundle of formatted representations.
 
         A mimebundle is a dictionary, keyed by mime-type.
         It must always have the key `'text/plain'`.
         """
         with self.builtin_trap:
             info = self._object_find(oname)
             if info.found:
                 docformat = (
                     sphinxify(self.object_inspect(oname))
                     if self.sphinxify_docstring
                     else None
                 )
                 return self.inspector._get_info(
                     info.obj,
                     oname,
                     info=info,
                     detail_level=detail_level,
                     formatter=docformat,
                 )
             else:
                 raise KeyError(oname)
 
     #-------------------------------------------------------------------------
     # Things related to history management
     #-------------------------------------------------------------------------
 
     def init_history(self):
         """Sets up the command history, and starts regular autosaves."""
         self.history_manager = HistoryManager(shell=self, parent=self)
         self.configurables.append(self.history_manager)
 
     #-------------------------------------------------------------------------
     # Things related to exception handling and tracebacks (not debugging)
     #-------------------------------------------------------------------------
 
     debugger_cls = InterruptiblePdb
 
     def init_traceback_handlers(self, custom_exceptions):
         # Syntax error handler.
         self.SyntaxTB = ultratb.SyntaxTB(color_scheme='NoColor', parent=self)
 
         # The interactive one is initialized with an offset, meaning we always
         # want to remove the topmost item in the traceback, which is our own
         # internal code. Valid modes: ['Plain','Context','Verbose','Minimal']
         self.InteractiveTB = ultratb.AutoFormattedTB(mode = 'Plain',
                                                      color_scheme='NoColor',
                                                      tb_offset = 1,
                                    check_cache=check_linecache_ipython,
                                    debugger_cls=self.debugger_cls, parent=self)
 
         # The instance will store a pointer to the system-wide exception hook,
         # so that runtime code (such as magics) can access it.  This is because
         # during the read-eval loop, it may get temporarily overwritten.
         self.sys_excepthook = sys.excepthook
 
         # and add any custom exception handlers the user may have specified
         self.set_custom_exc(*custom_exceptions)
 
         # Set the exception mode
         self.InteractiveTB.set_mode(mode=self.xmode)
 
     def set_custom_exc(self, exc_tuple, handler):
         """set_custom_exc(exc_tuple, handler)
 
         Set a custom exception handler, which will be called if any of the
         exceptions in exc_tuple occur in the mainloop (specifically, in the
         run_code() method).
 
         Parameters
         ----------
 
         exc_tuple : tuple of exception classes
             A *tuple* of exception classes, for which to call the defined
             handler.  It is very important that you use a tuple, and NOT A
             LIST here, because of the way Python's except statement works.  If
             you only want to trap a single exception, use a singleton tuple::
 
                 exc_tuple == (MyCustomException,)
 
         handler : callable
             handler must have the following signature::
 
                 def my_handler(self, etype, value, tb, tb_offset=None):
                     ...
                     return structured_traceback
 
             Your handler must return a structured traceback (a list of strings),
             or None.
 
             This will be made into an instance method (via types.MethodType)
             of IPython itself, and it will be called if any of the exceptions
             listed in the exc_tuple are caught. If the handler is None, an
             internal basic one is used, which just prints basic info.
 
             To protect IPython from crashes, if your handler ever raises an
             exception or returns an invalid result, it will be immediately
             disabled.
 
         Notes
         -----
 
         WARNING: by putting in your own exception handler into IPython's main
         execution loop, you run a very good chance of nasty crashes.  This
         facility should only be used if you really know what you are doing."""
         if not isinstance(exc_tuple, tuple):
             raise TypeError("The custom exceptions must be given as a tuple.")
 
         def dummy_handler(self, etype, value, tb, tb_offset=None):
             print('*** Simple custom exception handler ***')
             print('Exception type :', etype)
             print('Exception value:', value)
             print('Traceback      :', tb)
 
         def validate_stb(stb):
             """validate structured traceback return type
             
             return type of CustomTB *should* be a list of strings, but allow
             single strings or None, which are harmless.
             
             This function will *always* return a list of strings,
             and will raise a TypeError if stb is inappropriate.
             """
             msg = "CustomTB must return list of strings, not %r" % stb
             if stb is None:
                 return []
             elif isinstance(stb, str):
                 return [stb]
             elif not isinstance(stb, list):
                 raise TypeError(msg)
             # it's a list
             for line in stb:
                 # check every element
                 if not isinstance(line, str):
                     raise TypeError(msg)
             return stb
 
         if handler is None:
             wrapped = dummy_handler
         else:
             def wrapped(self,etype,value,tb,tb_offset=None):
                 """wrap CustomTB handler, to protect IPython from user code
 
                 This makes it harder (but not impossible) for custom exception
                 handlers to crash IPython.
                 """
                 try:
                     stb = handler(self,etype,value,tb,tb_offset=tb_offset)
                     return validate_stb(stb)
                 except:
                     # clear custom handler immediately
                     self.set_custom_exc((), None)
                     print("Custom TB Handler failed, unregistering", file=sys.stderr)
                     # show the exception in handler first
                     stb = self.InteractiveTB.structured_traceback(*sys.exc_info())
                     print(self.InteractiveTB.stb2text(stb))
                     print("The original exception:")
                     stb = self.InteractiveTB.structured_traceback(
                                             (etype,value,tb), tb_offset=tb_offset
                     )
                 return stb
 
         self.CustomTB = types.MethodType(wrapped,self)
         self.custom_exceptions = exc_tuple
 
     def excepthook(self, etype, value, tb):
         """One more defense for GUI apps that call sys.excepthook.
 
         GUI frameworks like wxPython trap exceptions and call
         sys.excepthook themselves.  I guess this is a feature that
         enables them to keep running after exceptions that would
         otherwise kill their mainloop. This is a bother for IPython
         which expects to catch all of the program exceptions with a try:
         except: statement.
 
         Normally, IPython sets sys.excepthook to a CrashHandler instance, so if
         any app directly invokes sys.excepthook, it will look to the user like
         IPython crashed.  In order to work around this, we can disable the
         CrashHandler and replace it with this excepthook instead, which prints a
         regular traceback using our InteractiveTB.  In this fashion, apps which
         call sys.excepthook will generate a regular-looking exception from
         IPython, and the CrashHandler will only be triggered by real IPython
         crashes.
 
         This hook should be used sparingly, only in places which are not likely
         to be true IPython errors.
         """
         self.showtraceback((etype, value, tb), tb_offset=0)
 
     def _get_exc_info(self, exc_tuple=None):
         """get exc_info from a given tuple, sys.exc_info() or sys.last_type etc.
 
         Ensures sys.last_type,value,traceback hold the exc_info we found,
         from whichever source.
 
         raises ValueError if none of these contain any information
         """
         if exc_tuple is None:
             etype, value, tb = sys.exc_info()
         else:
             etype, value, tb = exc_tuple
 
         if etype is None:
             if hasattr(sys, 'last_type'):
                 etype, value, tb = sys.last_type, sys.last_value, \
                                    sys.last_traceback
 
         if etype is None:
             raise ValueError("No exception to find")
 
         # Now store the exception info in sys.last_type etc.
         # WARNING: these variables are somewhat deprecated and not
         # necessarily safe to use in a threaded environment, but tools
         # like pdb depend on their existence, so let's set them.  If we
         # find problems in the field, we'll need to revisit their use.
         sys.last_type = etype
         sys.last_value = value
         sys.last_traceback = tb
 
         return etype, value, tb
 
     def show_usage_error(self, exc):
         """Show a short message for UsageErrors
 
         These are special exceptions that shouldn't show a traceback.
         """
         print("UsageError: %s" % exc, file=sys.stderr)
 
     def get_exception_only(self, exc_tuple=None):
         """
         Return as a string (ending with a newline) the exception that
         just occurred, without any traceback.
         """
         etype, value, tb = self._get_exc_info(exc_tuple)
         msg = traceback.format_exception_only(etype, value)
         return ''.join(msg)
 
     def showtraceback(self, exc_tuple=None, filename=None, tb_offset=None,
                       exception_only=False, running_compiled_code=False):
         """Display the exception that just occurred.
 
         If nothing is known about the exception, this is the method which
         should be used throughout the code for presenting user tracebacks,
         rather than directly invoking the InteractiveTB object.
 
         A specific showsyntaxerror() also exists, but this method can take
         care of calling it if needed, so unless you are explicitly catching a
         SyntaxError exception, don't try to analyze the stack manually and
         simply call this method."""
 
         try:
             try:
                 etype, value, tb = self._get_exc_info(exc_tuple)
             except ValueError:
                 print('No traceback available to show.', file=sys.stderr)
                 return
 
             if issubclass(etype, SyntaxError):
                 # Though this won't be called by syntax errors in the input
                 # line, there may be SyntaxError cases with imported code.
                 self.showsyntaxerror(filename, running_compiled_code)
             elif etype is UsageError:
                 self.show_usage_error(value)
             else:
                 if exception_only:
                     stb = ['An exception has occurred, use %tb to see '
                            'the full traceback.\n']
                     stb.extend(self.InteractiveTB.get_exception_only(etype,
                                                                      value))
                 else:
                     try:
                         # Exception classes can customise their traceback - we
                         # use this in IPython.parallel for exceptions occurring
                         # in the engines. This should return a list of strings.
                         stb = value._render_traceback_()
                     except Exception:
                         stb = self.InteractiveTB.structured_traceback(etype,
                                             value, tb, tb_offset=tb_offset)
 
                     self._showtraceback(etype, value, stb)
                     if self.call_pdb:
                         # drop into debugger
                         self.debugger(force=True)
                     return
 
                 # Actually show the traceback
                 self._showtraceback(etype, value, stb)
 
         except KeyboardInterrupt:
             print('\n' + self.get_exception_only(), file=sys.stderr)
 
     def _showtraceback(self, etype, evalue, stb: str):
         """Actually show a traceback.
 
         Subclasses may override this method to put the traceback on a different
         place, like a side channel.
         """
         val = self.InteractiveTB.stb2text(stb)
         try:
             print(val)
         except UnicodeEncodeError:
             print(val.encode("utf-8", "backslashreplace").decode())
 
     def showsyntaxerror(self, filename=None, running_compiled_code=False):
         """Display the syntax error that just occurred.
 
         This doesn't display a stack trace because there isn't one.
 
         If a filename is given, it is stuffed in the exception instead
         of what was there before (because Python's parser always uses
         "<string>" when reading from a string).
 
         If the syntax error occurred when running a compiled code (i.e. running_compile_code=True),
         longer stack trace will be displayed.
          """
         etype, value, last_traceback = self._get_exc_info()
 
         if filename and issubclass(etype, SyntaxError):
             try:
                 value.filename = filename
             except:
                 # Not the format we expect; leave it alone
                 pass
 
         # If the error occurred when executing compiled code, we should provide full stacktrace.
         elist = traceback.extract_tb(last_traceback) if running_compiled_code else []
         stb = self.SyntaxTB.structured_traceback(etype, value, elist)
         self._showtraceback(etype, value, stb)
 
     # This is overridden in TerminalInteractiveShell to show a message about
     # the %paste magic.
     def showindentationerror(self):
         """Called by _run_cell when there's an IndentationError in code entered
         at the prompt.
 
         This is overridden in TerminalInteractiveShell to show a message about
         the %paste magic."""
         self.showsyntaxerror()
 
     #-------------------------------------------------------------------------
     # Things related to readline
     #-------------------------------------------------------------------------
 
     def init_readline(self):
         """DEPRECATED
 
         Moved to terminal subclass, here only to simplify the init logic."""
         # Set a number of methods that depend on readline to be no-op
         warnings.warn('`init_readline` is no-op since IPython 5.0 and is Deprecated',
                 DeprecationWarning, stacklevel=2)
         self.set_custom_completer = no_op
 
     @skip_doctest
     def set_next_input(self, s, replace=False):
         """ Sets the 'default' input string for the next command line.
 
         Example::
 
             In [1]: _ip.set_next_input("Hello Word")
             In [2]: Hello Word_  # cursor is here
         """
         self.rl_next_input = s
 
     def _indent_current_str(self):
         """return the current level of indentation as a string"""
         return self.input_splitter.get_indent_spaces() * ' '
 
     #-------------------------------------------------------------------------
     # Things related to text completion
     #-------------------------------------------------------------------------
 
     def init_completer(self):
         """Initialize the completion machinery.
 
         This creates completion machinery that can be used by client code,
         either interactively in-process (typically triggered by the readline
         library), programmatically (such as in test suites) or out-of-process
         (typically over the network by remote frontends).
         """
         from IPython.core.completer import IPCompleter
         from IPython.core.completerlib import (module_completer,
                 magic_run_completer, cd_completer, reset_completer)
 
         self.Completer = IPCompleter(shell=self,
                                      namespace=self.user_ns,
                                      global_namespace=self.user_global_ns,
                                      parent=self,
                                      )
         self.configurables.append(self.Completer)
 
         # Add custom completers to the basic ones built into IPCompleter
         sdisp = self.strdispatchers.get('complete_command', StrDispatch())
         self.strdispatchers['complete_command'] = sdisp
         self.Completer.custom_completers = sdisp
 
         self.set_hook('complete_command', module_completer, str_key = 'import')
         self.set_hook('complete_command', module_completer, str_key = 'from')
         self.set_hook('complete_command', module_completer, str_key = '%aimport')
         self.set_hook('complete_command', magic_run_completer, str_key = '%run')
         self.set_hook('complete_command', cd_completer, str_key = '%cd')
         self.set_hook('complete_command', reset_completer, str_key = '%reset')
 
     @skip_doctest
     def complete(self, text, line=None, cursor_pos=None):
         """Return the completed text and a list of completions.
 
         Parameters
         ----------
 
            text : string
              A string of text to be completed on.  It can be given as empty and
              instead a line/position pair are given.  In this case, the
              completer itself will split the line like readline does.
 
            line : string, optional
              The complete line that text is part of.
 
            cursor_pos : int, optional
              The position of the cursor on the input line.
 
         Returns
         -------
           text : string
             The actual text that was completed.
 
           matches : list
             A sorted list with all possible completions.
 
         The optional arguments allow the completion to take more context into
         account, and are part of the low-level completion API.
 
         This is a wrapper around the completion mechanism, similar to what
         readline does at the command line when the TAB key is hit.  By
         exposing it as a method, it can be used by other non-readline
         environments (such as GUIs) for text completion.
 
         Simple usage example:
 
         In [1]: x = 'hello'
 
         In [2]: _ip.complete('x.l')
         Out[2]: ('x.l', ['x.ljust', 'x.lower', 'x.lstrip'])
         """
 
         # Inject names into __builtin__ so we can complete on the added names.
         with self.builtin_trap:
             return self.Completer.complete(text, line, cursor_pos)
 
     def set_custom_completer(self, completer, pos=0) -> None:
         """Adds a new custom completer function.
 
         The position argument (defaults to 0) is the index in the completers
         list where you want the completer to be inserted.
 
         `completer` should have the following signature::
 
             def completion(self: Completer, text: string) -> List[str]:
                 raise NotImplementedError
 
         It will be bound to the current Completer instance and pass some text
         and return a list with current completions to suggest to the user.
         """
 
         newcomp = types.MethodType(completer, self.Completer)
         self.Completer.custom_matchers.insert(pos,newcomp)
 
     def set_completer_frame(self, frame=None):
         """Set the frame of the completer."""
         if frame:
             self.Completer.namespace = frame.f_locals
             self.Completer.global_namespace = frame.f_globals
         else:
             self.Completer.namespace = self.user_ns
             self.Completer.global_namespace = self.user_global_ns
 
     #-------------------------------------------------------------------------
     # Things related to magics
     #-------------------------------------------------------------------------
 
     def init_magics(self):
         from IPython.core import magics as m
         self.magics_manager = magic.MagicsManager(shell=self,
                                    parent=self,
                                    user_magics=m.UserMagics(self))
         self.configurables.append(self.magics_manager)
 
         # Expose as public API from the magics manager
         self.register_magics = self.magics_manager.register
 
         self.register_magics(m.AutoMagics, m.BasicMagics, m.CodeMagics,
             m.ConfigMagics, m.DisplayMagics, m.ExecutionMagics,
             m.ExtensionMagics, m.HistoryMagics, m.LoggingMagics,
             m.NamespaceMagics, m.OSMagics, m.PackagingMagics,
             m.PylabMagics, m.ScriptMagics,
         )
         self.register_magics(m.AsyncMagics)
 
         # Register Magic Aliases
         mman = self.magics_manager
         # FIXME: magic aliases should be defined by the Magics classes
         # or in MagicsManager, not here
         mman.register_alias('ed', 'edit')
         mman.register_alias('hist', 'history')
         mman.register_alias('rep', 'recall')
         mman.register_alias('SVG', 'svg', 'cell')
         mman.register_alias('HTML', 'html', 'cell')
         mman.register_alias('file', 'writefile', 'cell')
 
         # FIXME: Move the color initialization to the DisplayHook, which
         # should be split into a prompt manager and displayhook. We probably
         # even need a centralize colors management object.
         self.run_line_magic('colors', self.colors)
 
     # Defined here so that it's included in the documentation
     @functools.wraps(magic.MagicsManager.register_function)
     def register_magic_function(self, func, magic_kind='line', magic_name=None):
         self.magics_manager.register_function(
             func, magic_kind=magic_kind, magic_name=magic_name
         )
 
-    def run_line_magic(self, magic_name, line, _stack_depth=1):
+    def _find_with_lazy_load(self, type_, magic_name: str):
+        """
+        Try to find a magic potentially lazy-loading it.
+
+        Parameters
+        ----------
+
+        type_: "line"|"cell"
+            the type of magics we are trying to find/lazy load.
+        magic_name: str
+            The name of the magic we are trying to find/lazy load
+
+
+        Note that this may have any side effects
+        """
+        finder = {"line": self.find_line_magic, "cell": self.find_cell_magic}[type_]
+        fn = finder(magic_name)
+        if fn is not None:
+            return fn
+        lazy = self.magics_manager.lazy_magics.get(magic_name)
+        if lazy is None:
+            return None
+
+        self.run_line_magic("load_ext", lazy)
+        res = finder(magic_name)
+        return res
+
+    def run_line_magic(self, magic_name: str, line, _stack_depth=1):
         """Execute the given line magic.
 
         Parameters
         ----------
         magic_name : str
           Name of the desired magic function, without '%' prefix.
 
         line : str
           The rest of the input line as a single string.
           
         _stack_depth : int
           If run_line_magic() is called from magic() then _stack_depth=2.
           This is added to ensure backward compatibility for use of 'get_ipython().magic()'
         """
-        fn = self.find_line_magic(magic_name)
+        fn = self._find_with_lazy_load("line", magic_name)
+        if fn is None:
+            lazy = self.magics_manager.lazy_magics.get(magic_name)
+            if lazy:
+                self.run_line_magic("load_ext", lazy)
+                fn = self.find_line_magic(magic_name)
         if fn is None:
             cm = self.find_cell_magic(magic_name)
             etpl = "Line magic function `%%%s` not found%s."
             extra = '' if cm is None else (' (But cell magic `%%%%%s` exists, '
                                     'did you mean that instead?)' % magic_name )
             raise UsageError(etpl % (magic_name, extra))
         else:
             # Note: this is the distance in the stack to the user's frame.
             # This will need to be updated if the internal calling logic gets
             # refactored, or else we'll be expanding the wrong variables.
 
             # Determine stack_depth depending on where run_line_magic() has been called
             stack_depth = _stack_depth
             if getattr(fn, magic.MAGIC_NO_VAR_EXPAND_ATTR, False):
                 # magic has opted out of var_expand
                 magic_arg_s = line
             else:
                 magic_arg_s = self.var_expand(line, stack_depth)
             # Put magic args in a list so we can call with f(*a) syntax
             args = [magic_arg_s]
             kwargs = {}
             # Grab local namespace if we need it:
             if getattr(fn, "needs_local_scope", False):
                 kwargs['local_ns'] = self.get_local_scope(stack_depth)
             with self.builtin_trap:
                 result = fn(*args, **kwargs)
             return result
 
     def get_local_scope(self, stack_depth):
         """Get local scope at given stack depth.
 
         Parameters
         ----------
         stack_depth : int
           Depth relative to calling frame
         """
         return sys._getframe(stack_depth + 1).f_locals
 
     def run_cell_magic(self, magic_name, line, cell):
         """Execute the given cell magic.
 
         Parameters
         ----------
         magic_name : str
           Name of the desired magic function, without '%' prefix.
 
         line : str
           The rest of the first input line as a single string.
 
         cell : str
           The body of the cell as a (possibly multiline) string.
         """
-        fn = self.find_cell_magic(magic_name)
+        fn = self._find_with_lazy_load("cell", magic_name)
         if fn is None:
             lm = self.find_line_magic(magic_name)
             etpl = "Cell magic `%%{0}` not found{1}."
             extra = '' if lm is None else (' (But line magic `%{0}` exists, '
                             'did you mean that instead?)'.format(magic_name))
             raise UsageError(etpl.format(magic_name, extra))
         elif cell == '':
             message = '%%{0} is a cell magic, but the cell body is empty.'.format(magic_name)
             if self.find_line_magic(magic_name) is not None:
                 message += ' Did you mean the line magic %{0} (single %)?'.format(magic_name)
             raise UsageError(message)
         else:
             # Note: this is the distance in the stack to the user's frame.
             # This will need to be updated if the internal calling logic gets
             # refactored, or else we'll be expanding the wrong variables.
             stack_depth = 2
             if getattr(fn, magic.MAGIC_NO_VAR_EXPAND_ATTR, False):
                 # magic has opted out of var_expand
                 magic_arg_s = line
             else:
                 magic_arg_s = self.var_expand(line, stack_depth)
             kwargs = {}
             if getattr(fn, "needs_local_scope", False):
                 kwargs['local_ns'] = self.user_ns
 
             with self.builtin_trap:
                 args = (magic_arg_s, cell)
                 result = fn(*args, **kwargs)
             return result
 
     def find_line_magic(self, magic_name):
         """Find and return a line magic by name.
 
         Returns None if the magic isn't found."""
         return self.magics_manager.magics['line'].get(magic_name)
 
     def find_cell_magic(self, magic_name):
         """Find and return a cell magic by name.
 
         Returns None if the magic isn't found."""
         return self.magics_manager.magics['cell'].get(magic_name)
 
     def find_magic(self, magic_name, magic_kind='line'):
         """Find and return a magic of the given type by name.
 
         Returns None if the magic isn't found."""
         return self.magics_manager.magics[magic_kind].get(magic_name)
 
     def magic(self, arg_s):
         """DEPRECATED. Use run_line_magic() instead.
 
         Call a magic function by name.
 
         Input: a string containing the name of the magic function to call and
         any additional arguments to be passed to the magic.
 
         magic('name -opt foo bar') is equivalent to typing at the ipython
         prompt:
 
         In[1]: %name -opt foo bar
 
         To call a magic without arguments, simply use magic('name').
 
         This provides a proper Python function to call IPython's magics in any
         valid Python code you can type at the interpreter, including loops and
         compound statements.
         """
         # TODO: should we issue a loud deprecation warning here?
         magic_name, _, magic_arg_s = arg_s.partition(' ')
         magic_name = magic_name.lstrip(prefilter.ESC_MAGIC)
         return self.run_line_magic(magic_name, magic_arg_s, _stack_depth=2)
 
     #-------------------------------------------------------------------------
     # Things related to macros
     #-------------------------------------------------------------------------
 
     def define_macro(self, name, themacro):
         """Define a new macro
 
         Parameters
         ----------
         name : str
             The name of the macro.
         themacro : str or Macro
             The action to do upon invoking the macro.  If a string, a new
             Macro object is created by passing the string to it.
         """
 
         from IPython.core import macro
 
         if isinstance(themacro, str):
             themacro = macro.Macro(themacro)
         if not isinstance(themacro, macro.Macro):
             raise ValueError('A macro must be a string or a Macro instance.')
         self.user_ns[name] = themacro
 
     #-------------------------------------------------------------------------
     # Things related to the running of system commands
     #-------------------------------------------------------------------------
 
     def system_piped(self, cmd):
         """Call the given cmd in a subprocess, piping stdout/err
 
         Parameters
         ----------
         cmd : str
           Command to execute (can not end in '&', as background processes are
           not supported.  Should not be a command that expects input
           other than simple text.
         """
         if cmd.rstrip().endswith('&'):
             # this is *far* from a rigorous test
             # We do not support backgrounding processes because we either use
             # pexpect or pipes to read from.  Users can always just call
             # os.system() or use ip.system=ip.system_raw
             # if they really want a background process.
             raise OSError("Background processes not supported.")
 
         # we explicitly do NOT return the subprocess status code, because
         # a non-None value would trigger :func:`sys.displayhook` calls.
         # Instead, we store the exit_code in user_ns.
         self.user_ns['_exit_code'] = system(self.var_expand(cmd, depth=1))
 
     def system_raw(self, cmd):
         """Call the given cmd in a subprocess using os.system on Windows or
         subprocess.call using the system shell on other platforms.
 
         Parameters
         ----------
         cmd : str
           Command to execute.
         """
         cmd = self.var_expand(cmd, depth=1)
         # protect os.system from UNC paths on Windows, which it can't handle:
         if sys.platform == 'win32':
             from IPython.utils._process_win32 import AvoidUNCPath
             with AvoidUNCPath() as path:
                 if path is not None:
                     cmd = '"pushd %s &&"%s' % (path, cmd)
                 try:
                     ec = os.system(cmd)
                 except KeyboardInterrupt:
                     print('\n' + self.get_exception_only(), file=sys.stderr)
                     ec = -2
         else:
             # For posix the result of the subprocess.call() below is an exit
             # code, which by convention is zero for success, positive for
             # program failure.  Exit codes above 128 are reserved for signals,
             # and the formula for converting a signal to an exit code is usually
             # signal_number+128.  To more easily differentiate between exit
             # codes and signals, ipython uses negative numbers.  For instance
             # since control-c is signal 2 but exit code 130, ipython's
             # _exit_code variable will read -2.  Note that some shells like
             # csh and fish don't follow sh/bash conventions for exit codes.
             executable = os.environ.get('SHELL', None)
             try:
                 # Use env shell instead of default /bin/sh
                 ec = subprocess.call(cmd, shell=True, executable=executable)
             except KeyboardInterrupt:
                 # intercept control-C; a long traceback is not useful here
                 print('\n' + self.get_exception_only(), file=sys.stderr)
                 ec = 130
             if ec > 128:
                 ec = -(ec - 128)
 
         # We explicitly do NOT return the subprocess status code, because
         # a non-None value would trigger :func:`sys.displayhook` calls.
         # Instead, we store the exit_code in user_ns.  Note the semantics
         # of _exit_code: for control-c, _exit_code == -signal.SIGNIT,
         # but raising SystemExit(_exit_code) will give status 254!
         self.user_ns['_exit_code'] = ec
 
     # use piped system by default, because it is better behaved
     system = system_piped
 
     def getoutput(self, cmd, split=True, depth=0):
         """Get output (possibly including stderr) from a subprocess.
 
         Parameters
         ----------
         cmd : str
           Command to execute (can not end in '&', as background processes are
           not supported.
         split : bool, optional
           If True, split the output into an IPython SList.  Otherwise, an
           IPython LSString is returned.  These are objects similar to normal
           lists and strings, with a few convenience attributes for easier
           manipulation of line-based output.  You can use '?' on them for
           details.
         depth : int, optional
           How many frames above the caller are the local variables which should
           be expanded in the command string? The default (0) assumes that the
           expansion variables are in the stack frame calling this function.
         """
         if cmd.rstrip().endswith('&'):
             # this is *far* from a rigorous test
             raise OSError("Background processes not supported.")
         out = getoutput(self.var_expand(cmd, depth=depth+1))
         if split:
             out = SList(out.splitlines())
         else:
             out = LSString(out)
         return out
 
     #-------------------------------------------------------------------------
     # Things related to aliases
     #-------------------------------------------------------------------------
 
     def init_alias(self):
         self.alias_manager = AliasManager(shell=self, parent=self)
         self.configurables.append(self.alias_manager)
 
     #-------------------------------------------------------------------------
     # Things related to extensions
     #-------------------------------------------------------------------------
 
     def init_extension_manager(self):
         self.extension_manager = ExtensionManager(shell=self, parent=self)
         self.configurables.append(self.extension_manager)
 
     #-------------------------------------------------------------------------
     # Things related to payloads
     #-------------------------------------------------------------------------
 
     def init_payload(self):
         self.payload_manager = PayloadManager(parent=self)
         self.configurables.append(self.payload_manager)
 
     #-------------------------------------------------------------------------
     # Things related to the prefilter
     #-------------------------------------------------------------------------
 
     def init_prefilter(self):
         self.prefilter_manager = PrefilterManager(shell=self, parent=self)
         self.configurables.append(self.prefilter_manager)
         # Ultimately this will be refactored in the new interpreter code, but
         # for now, we should expose the main prefilter method (there's legacy
         # code out there that may rely on this).
         self.prefilter = self.prefilter_manager.prefilter_lines
 
     def auto_rewrite_input(self, cmd):
         """Print to the screen the rewritten form of the user's command.
 
         This shows visual feedback by rewriting input lines that cause
         automatic calling to kick in, like::
 
           /f x
 
         into::
 
           ------> f(x)
 
         after the user's input prompt.  This helps the user understand that the
         input line was transformed automatically by IPython.
         """
         if not self.show_rewritten_input:
             return
 
         # This is overridden in TerminalInteractiveShell to use fancy prompts
         print("------> " + cmd)
 
     #-------------------------------------------------------------------------
     # Things related to extracting values/expressions from kernel and user_ns
     #-------------------------------------------------------------------------
 
     def _user_obj_error(self):
         """return simple exception dict
 
         for use in user_expressions
         """
 
         etype, evalue, tb = self._get_exc_info()
         stb = self.InteractiveTB.get_exception_only(etype, evalue)
 
         exc_info = {
             u'status' : 'error',
             u'traceback' : stb,
             u'ename' : etype.__name__,
             u'evalue' : py3compat.safe_unicode(evalue),
         }
 
         return exc_info
 
     def _format_user_obj(self, obj):
         """format a user object to display dict
 
         for use in user_expressions
         """
 
         data, md = self.display_formatter.format(obj)
         value = {
             'status' : 'ok',
             'data' : data,
             'metadata' : md,
         }
         return value
 
     def user_expressions(self, expressions):
         """Evaluate a dict of expressions in the user's namespace.
 
         Parameters
         ----------
         expressions : dict
           A dict with string keys and string values.  The expression values
           should be valid Python expressions, each of which will be evaluated
           in the user namespace.
 
         Returns
         -------
         A dict, keyed like the input expressions dict, with the rich mime-typed
         display_data of each value.
         """
         out = {}
         user_ns = self.user_ns
         global_ns = self.user_global_ns
 
         for key, expr in expressions.items():
             try:
                 value = self._format_user_obj(eval(expr, global_ns, user_ns))
             except:
                 value = self._user_obj_error()
             out[key] = value
         return out
 
     #-------------------------------------------------------------------------
     # Things related to the running of code
     #-------------------------------------------------------------------------
 
     def ex(self, cmd):
         """Execute a normal python statement in user namespace."""
         with self.builtin_trap:
             exec(cmd, self.user_global_ns, self.user_ns)
 
     def ev(self, expr):
         """Evaluate python expression expr in user namespace.
 
         Returns the result of evaluation
         """
         with self.builtin_trap:
             return eval(expr, self.user_global_ns, self.user_ns)
 
     def safe_execfile(self, fname, *where, exit_ignore=False, raise_exceptions=False, shell_futures=False):
         """A safe version of the builtin execfile().
 
         This version will never throw an exception, but instead print
         helpful error messages to the screen.  This only works on pure
         Python files with the .py extension.
 
         Parameters
         ----------
         fname : string
             The name of the file to be executed.
         where : tuple
             One or two namespaces, passed to execfile() as (globals,locals).
             If only one is given, it is passed as both.
         exit_ignore : bool (False)
             If True, then silence SystemExit for non-zero status (it is always
             silenced for zero status, as it is so common).
         raise_exceptions : bool (False)
             If True raise exceptions everywhere. Meant for testing.
         shell_futures : bool (False)
             If True, the code will share future statements with the interactive
             shell. It will both be affected by previous __future__ imports, and
             any __future__ imports in the code will affect the shell. If False,
             __future__ imports are not shared in either direction.
 
         """
         fname = os.path.abspath(os.path.expanduser(fname))
 
         # Make sure we can open the file
         try:
             with open(fname):
                 pass
         except:
             warn('Could not open file <%s> for safe execution.' % fname)
             return
 
         # Find things also in current directory.  This is needed to mimic the
         # behavior of running a script from the system command line, where
         # Python inserts the script's directory into sys.path
         dname = os.path.dirname(fname)
 
         with prepended_to_syspath(dname), self.builtin_trap:
             try:
                 glob, loc = (where + (None, ))[:2]
                 py3compat.execfile(
                     fname, glob, loc,
                     self.compile if shell_futures else None)
             except SystemExit as status:
                 # If the call was made with 0 or None exit status (sys.exit(0)
                 # or sys.exit() ), don't bother showing a traceback, as both of
                 # these are considered normal by the OS:
                 # > python -c'import sys;sys.exit(0)'; echo $?
                 # 0
                 # > python -c'import sys;sys.exit()'; echo $?
                 # 0
                 # For other exit status, we show the exception unless
                 # explicitly silenced, but only in short form.
                 if status.code:
                     if raise_exceptions:
                         raise
                     if not exit_ignore:
                         self.showtraceback(exception_only=True)
             except:
                 if raise_exceptions:
                     raise
                 # tb offset is 2 because we wrap execfile
                 self.showtraceback(tb_offset=2)
 
     def safe_execfile_ipy(self, fname, shell_futures=False, raise_exceptions=False):
         """Like safe_execfile, but for .ipy or .ipynb files with IPython syntax.
 
         Parameters
         ----------
         fname : str
             The name of the file to execute.  The filename must have a
             .ipy or .ipynb extension.
         shell_futures : bool (False)
             If True, the code will share future statements with the interactive
             shell. It will both be affected by previous __future__ imports, and
             any __future__ imports in the code will affect the shell. If False,
             __future__ imports are not shared in either direction.
         raise_exceptions : bool (False)
             If True raise exceptions everywhere.  Meant for testing.
         """
         fname = os.path.abspath(os.path.expanduser(fname))
 
         # Make sure we can open the file
         try:
             with open(fname):
                 pass
         except:
             warn('Could not open file <%s> for safe execution.' % fname)
             return
 
         # Find things also in current directory.  This is needed to mimic the
         # behavior of running a script from the system command line, where
         # Python inserts the script's directory into sys.path
         dname = os.path.dirname(fname)
         
         def get_cells():
             """generator for sequence of code blocks to run"""
             if fname.endswith('.ipynb'):
                 from nbformat import read
                 nb = read(fname, as_version=4)
                 if not nb.cells:
                     return
                 for cell in nb.cells:
                     if cell.cell_type == 'code':
                         yield cell.source
             else:
                 with open(fname) as f:
                     yield f.read()
 
         with prepended_to_syspath(dname):
             try:
                 for cell in get_cells():
                     result = self.run_cell(cell, silent=True, shell_futures=shell_futures)
                     if raise_exceptions:
                         result.raise_error()
                     elif not result.success:
                         break
             except:
                 if raise_exceptions:
                     raise
                 self.showtraceback()
                 warn('Unknown failure executing file: <%s>' % fname)
 
     def safe_run_module(self, mod_name, where):
         """A safe version of runpy.run_module().
 
         This version will never throw an exception, but instead print
         helpful error messages to the screen.
 
         `SystemExit` exceptions with status code 0 or None are ignored.
 
         Parameters
         ----------
         mod_name : string
             The name of the module to be executed.
         where : dict
             The globals namespace.
         """
         try:
             try:
                 where.update(
                     runpy.run_module(str(mod_name), run_name="__main__",
                                      alter_sys=True)
                             )
             except SystemExit as status:
                 if status.code:
                     raise
         except:
             self.showtraceback()
             warn('Unknown failure executing module: <%s>' % mod_name)
 
     def run_cell(self, raw_cell, store_history=False, silent=False, shell_futures=True):
         """Run a complete IPython cell.
 
         Parameters
         ----------
         raw_cell : str
           The code (including IPython code such as %magic functions) to run.
         store_history : bool
           If True, the raw and translated cell will be stored in IPython's
           history. For user code calling back into IPython's machinery, this
           should be set to False.
         silent : bool
           If True, avoid side-effects, such as implicit displayhooks and
           and logging.  silent=True forces store_history=False.
         shell_futures : bool
           If True, the code will share future statements with the interactive
           shell. It will both be affected by previous __future__ imports, and
           any __future__ imports in the code will affect the shell. If False,
           __future__ imports are not shared in either direction.
 
         Returns
         -------
         result : :class:`ExecutionResult`
         """
         result = None
         try:
             result = self._run_cell(
                 raw_cell, store_history, silent, shell_futures)
         finally:
             self.events.trigger('post_execute')
             if not silent:
                 self.events.trigger('post_run_cell', result)
         return result
 
     def _run_cell(self, raw_cell:str, store_history:bool, silent:bool, shell_futures:bool):
         """Internal method to run a complete IPython cell."""
 
         # we need to avoid calling self.transform_cell multiple time on the same thing
         # so we need to store some results:
         preprocessing_exc_tuple = None
         try:
             transformed_cell = self.transform_cell(raw_cell)
         except Exception:
             transformed_cell = raw_cell
             preprocessing_exc_tuple = sys.exc_info()
 
         assert transformed_cell is not None
         coro = self.run_cell_async(
             raw_cell,
             store_history=store_history,
             silent=silent,
             shell_futures=shell_futures,
             transformed_cell=transformed_cell,
             preprocessing_exc_tuple=preprocessing_exc_tuple,
         )
 
         # run_cell_async is async, but may not actually need an eventloop.
         # when this is the case, we want to run it using the pseudo_sync_runner
         # so that code can invoke eventloops (for example via the %run , and
         # `%paste` magic.
         if self.trio_runner:
             runner = self.trio_runner
         elif self.should_run_async(
             raw_cell,
             transformed_cell=transformed_cell,
             preprocessing_exc_tuple=preprocessing_exc_tuple,
         ):
             runner = self.loop_runner
         else:
             runner = _pseudo_sync_runner
 
         try:
             return runner(coro)
         except BaseException as e:
             info = ExecutionInfo(raw_cell, store_history, silent, shell_futures)
             result = ExecutionResult(info)
             result.error_in_exec = e
             self.showtraceback(running_compiled_code=True)
             return result
         return
 
     def should_run_async(
         self, raw_cell: str, *, transformed_cell=None, preprocessing_exc_tuple=None
     ) -> bool:
         """Return whether a cell should be run asynchronously via a coroutine runner
 
         Parameters
         ----------
         raw_cell: str
             The code to be executed
 
         Returns
         -------
         result: bool
             Whether the code needs to be run with a coroutine runner or not
 
         .. versionadded:: 7.0
         """
         if not self.autoawait:
             return False
         if preprocessing_exc_tuple is not None:
             return False
         assert preprocessing_exc_tuple is None
         if transformed_cell is None:
             warnings.warn(
                 "`should_run_async` will not call `transform_cell`"
                 " automatically in the future. Please pass the result to"
                 " `transformed_cell` argument and any exception that happen"
                 " during the"
                 "transform in `preprocessing_exc_tuple` in"
                 " IPython 7.17 and above.",
                 DeprecationWarning,
                 stacklevel=2,
             )
             try:
                 cell = self.transform_cell(raw_cell)
             except Exception:
                 # any exception during transform will be raised
                 # prior to execution
                 return False
         else:
             cell = transformed_cell
         return _should_be_async(cell)
 
     async def run_cell_async(
         self,
         raw_cell: str,
         store_history=False,
         silent=False,
         shell_futures=True,
         *,
         transformed_cell: Optional[str] = None,
         preprocessing_exc_tuple: Optional[Any] = None
     ) -> ExecutionResult:
         """Run a complete IPython cell asynchronously.
 
         Parameters
         ----------
         raw_cell : str
           The code (including IPython code such as %magic functions) to run.
         store_history : bool
           If True, the raw and translated cell will be stored in IPython's
           history. For user code calling back into IPython's machinery, this
           should be set to False.
         silent : bool
           If True, avoid side-effects, such as implicit displayhooks and
           and logging.  silent=True forces store_history=False.
         shell_futures : bool
           If True, the code will share future statements with the interactive
           shell. It will both be affected by previous __future__ imports, and
           any __future__ imports in the code will affect the shell. If False,
           __future__ imports are not shared in either direction.
         transformed_cell: str
           cell that was passed through transformers
         preprocessing_exc_tuple:
           trace if the transformation failed.
 
         Returns
         -------
         result : :class:`ExecutionResult`
 
         .. versionadded:: 7.0
         """
         info = ExecutionInfo(
             raw_cell, store_history, silent, shell_futures)
         result = ExecutionResult(info)
 
         if (not raw_cell) or raw_cell.isspace():
             self.last_execution_succeeded = True
             self.last_execution_result = result
             return result
 
         if silent:
             store_history = False
 
         if store_history:
             result.execution_count = self.execution_count
 
         def error_before_exec(value):
             if store_history:
                 self.execution_count += 1
             result.error_before_exec = value
             self.last_execution_succeeded = False
             self.last_execution_result = result
             return result
 
         self.events.trigger('pre_execute')
         if not silent:
             self.events.trigger('pre_run_cell', info)
 
         if transformed_cell is None:
             warnings.warn(
                 "`run_cell_async` will not call `transform_cell`"
                 " automatically in the future. Please pass the result to"
                 " `transformed_cell` argument and any exception that happen"
                 " during the"
                 "transform in `preprocessing_exc_tuple` in"
                 " IPython 7.17 and above.",
                 DeprecationWarning,
                 stacklevel=2,
             )
             # If any of our input transformation (input_transformer_manager or
             # prefilter_manager) raises an exception, we store it in this variable
             # so that we can display the error after logging the input and storing
             # it in the history.
             try:
                 cell = self.transform_cell(raw_cell)
             except Exception:
                 preprocessing_exc_tuple = sys.exc_info()
                 cell = raw_cell  # cell has to exist so it can be stored/logged
             else:
                 preprocessing_exc_tuple = None
         else:
             if preprocessing_exc_tuple is None:
                 cell = transformed_cell
             else:
                 cell = raw_cell
 
         # Store raw and processed history
         if store_history:
             self.history_manager.store_inputs(self.execution_count,
                                               cell, raw_cell)
         if not silent:
             self.logger.log(cell, raw_cell)
 
         # Display the exception if input processing failed.
         if preprocessing_exc_tuple is not None:
             self.showtraceback(preprocessing_exc_tuple)
             if store_history:
                 self.execution_count += 1
             return error_before_exec(preprocessing_exc_tuple[1])
 
         # Our own compiler remembers the __future__ environment. If we want to
         # run code with a separate __future__ environment, use the default
         # compiler
         compiler = self.compile if shell_futures else self.compiler_class()
 
         _run_async = False
 
         with self.builtin_trap:
             cell_name = self.compile.cache(
                 cell, self.execution_count, raw_code=raw_cell
             )
 
             with self.display_trap:
                 # Compile to bytecode
                 try:
                     if sys.version_info < (3,8) and self.autoawait:
                         if _should_be_async(cell):
                             # the code AST below will not be user code: we wrap it
                             # in an `async def`. This will likely make some AST
                             # transformer below miss some transform opportunity and
                             # introduce a small coupling to run_code (in which we
                             # bake some assumptions of what _ast_asyncify returns.
                             # they are ways around (like grafting part of the ast
                             # later:
                             #    - Here, return code_ast.body[0].body[1:-1], as well
                             #    as last expression in  return statement which is
                             #    the user code part.
                             #    - Let it go through the AST transformers, and graft
                             #    - it back after the AST transform
                             # But that seem unreasonable, at least while we
                             # do not need it.
                             code_ast = _ast_asyncify(cell, 'async-def-wrapper')
                             _run_async = True
                         else:
                             code_ast = compiler.ast_parse(cell, filename=cell_name)
                     else:
                         code_ast = compiler.ast_parse(cell, filename=cell_name)
                 except self.custom_exceptions as e:
                     etype, value, tb = sys.exc_info()
                     self.CustomTB(etype, value, tb)
                     return error_before_exec(e)
                 except IndentationError as e:
                     self.showindentationerror()
                     return error_before_exec(e)
                 except (OverflowError, SyntaxError, ValueError, TypeError,
                         MemoryError) as e:
                     self.showsyntaxerror()
                     return error_before_exec(e)
 
                 # Apply AST transformations
                 try:
                     code_ast = self.transform_ast(code_ast)
                 except InputRejected as e:
                     self.showtraceback()
                     return error_before_exec(e)
 
                 # Give the displayhook a reference to our ExecutionResult so it
                 # can fill in the output value.
                 self.displayhook.exec_result = result
 
                 # Execute the user code
                 interactivity = "none" if silent else self.ast_node_interactivity
                 if _run_async:
                     interactivity = 'async'
 
                 has_raised = await self.run_ast_nodes(code_ast.body, cell_name,
                        interactivity=interactivity, compiler=compiler, result=result)
 
                 self.last_execution_succeeded = not has_raised
                 self.last_execution_result = result
 
                 # Reset this so later displayed values do not modify the
                 # ExecutionResult
                 self.displayhook.exec_result = None
 
         if store_history:
             # Write output to the database. Does nothing unless
             # history output logging is enabled.
             self.history_manager.store_output(self.execution_count)
             # Each cell is a *single* input, regardless of how many lines it has
             self.execution_count += 1
 
         return result
 
     def transform_cell(self, raw_cell):
         """Transform an input cell before parsing it.
 
         Static transformations, implemented in IPython.core.inputtransformer2,
         deal with things like ``%magic`` and ``!system`` commands.
         These run on all input.
         Dynamic transformations, for things like unescaped magics and the exit
         autocall, depend on the state of the interpreter.
         These only apply to single line inputs.
 
         These string-based transformations are followed by AST transformations;
         see :meth:`transform_ast`.
         """
         # Static input transformations
         cell = self.input_transformer_manager.transform_cell(raw_cell)
 
         if len(cell.splitlines()) == 1:
             # Dynamic transformations - only applied for single line commands
             with self.builtin_trap:
                 # use prefilter_lines to handle trailing newlines
                 # restore trailing newline for ast.parse
                 cell = self.prefilter_manager.prefilter_lines(cell) + '\n'
 
         lines = cell.splitlines(keepends=True)
         for transform in self.input_transformers_post:
             lines = transform(lines)
         cell = ''.join(lines)
 
         return cell
 
     def transform_ast(self, node):
         """Apply the AST transformations from self.ast_transformers
 
         Parameters
         ----------
         node : ast.Node
           The root node to be transformed. Typically called with the ast.Module
           produced by parsing user input.
 
         Returns
         -------
         An ast.Node corresponding to the node it was called with. Note that it
         may also modify the passed object, so don't rely on references to the
         original AST.
         """
         for transformer in self.ast_transformers:
             try:
                 node = transformer.visit(node)
             except InputRejected:
                 # User-supplied AST transformers can reject an input by raising
                 # an InputRejected.  Short-circuit in this case so that we
                 # don't unregister the transform.
                 raise
             except Exception:
                 warn("AST transformer %r threw an error. It will be unregistered." % transformer)
                 self.ast_transformers.remove(transformer)
 
         if self.ast_transformers:
             ast.fix_missing_locations(node)
         return node
 
     async def run_ast_nodes(self, nodelist:ListType[AST], cell_name:str, interactivity='last_expr',
                         compiler=compile, result=None):
         """Run a sequence of AST nodes. The execution mode depends on the
         interactivity parameter.
 
         Parameters
         ----------
         nodelist : list
           A sequence of AST nodes to run.
         cell_name : str
           Will be passed to the compiler as the filename of the cell. Typically
           the value returned by ip.compile.cache(cell).
         interactivity : str
           'all', 'last', 'last_expr' , 'last_expr_or_assign' or 'none',
           specifying which nodes should be run interactively (displaying output
           from expressions). 'last_expr' will run the last node interactively
           only if it is an expression (i.e. expressions in loops or other blocks
           are not displayed) 'last_expr_or_assign' will run the last expression
           or the last assignment. Other values for this parameter will raise a
           ValueError.
 
           Experimental value: 'async' Will try to run top level interactive
           async/await code in default runner, this will not respect the
           interactivity setting and will only run the last node if it is an
           expression.
 
         compiler : callable
           A function with the same interface as the built-in compile(), to turn
           the AST nodes into code objects. Default is the built-in compile().
         result : ExecutionResult, optional
           An object to store exceptions that occur during execution.
 
         Returns
         -------
         True if an exception occurred while running code, False if it finished
         running.
         """
         if not nodelist:
             return
 
         if interactivity == 'last_expr_or_assign':
             if isinstance(nodelist[-1], _assign_nodes):
                 asg = nodelist[-1]
                 if isinstance(asg, ast.Assign) and len(asg.targets) == 1:
                     target = asg.targets[0]
                 elif isinstance(asg, _single_targets_nodes):
                     target = asg.target
                 else:
                     target = None
                 if isinstance(target, ast.Name):
                     nnode = ast.Expr(ast.Name(target.id, ast.Load()))
                     ast.fix_missing_locations(nnode)
                     nodelist.append(nnode)
             interactivity = 'last_expr'
 
         _async = False
         if interactivity == 'last_expr':
             if isinstance(nodelist[-1], ast.Expr):
                 interactivity = "last"
             else:
                 interactivity = "none"
 
         if interactivity == 'none':
             to_run_exec, to_run_interactive = nodelist, []
         elif interactivity == 'last':
             to_run_exec, to_run_interactive = nodelist[:-1], nodelist[-1:]
         elif interactivity == 'all':
             to_run_exec, to_run_interactive = [], nodelist
         elif interactivity == 'async':
             to_run_exec, to_run_interactive = [], nodelist
             _async = True
         else:
             raise ValueError("Interactivity was %r" % interactivity)
 
         try:
             if _async and sys.version_info > (3,8):
                 raise ValueError("This branch should never happen on Python 3.8 and above, "
                                  "please try to upgrade IPython and open a bug report with your case.")
             if _async:
                 # If interactivity is async the semantics of run_code are
                 # completely different Skip usual machinery.
                 mod = Module(nodelist, [])
                 async_wrapper_code = compiler(mod, cell_name, 'exec')
                 exec(async_wrapper_code, self.user_global_ns, self.user_ns)
                 async_code = removed_co_newlocals(self.user_ns.pop('async-def-wrapper')).__code__
                 if (await self.run_code(async_code, result, async_=True)):
                     return True
             else:
                 if sys.version_info > (3, 8):
                     def compare(code):
                         is_async = (inspect.CO_COROUTINE & code.co_flags == inspect.CO_COROUTINE)
                         return is_async
                 else:
                     def compare(code):
                         return _async
 
                 # refactor that to just change the mod constructor.
                 to_run = []
                 for node in to_run_exec:
                     to_run.append((node, 'exec'))
 
                 for node in to_run_interactive:
                     to_run.append((node, 'single'))
 
                 for node,mode in to_run:
                     if mode == 'exec':
                         mod = Module([node], [])
                     elif mode == 'single':
                         mod = ast.Interactive([node])
                     with compiler.extra_flags(getattr(ast, 'PyCF_ALLOW_TOP_LEVEL_AWAIT', 0x0) if self.autoawait else 0x0):
                         code = compiler(mod, cell_name, mode)
                         asy = compare(code)
                     if (await self.run_code(code, result,  async_=asy)):
                         return True
 
             # Flush softspace
             if softspace(sys.stdout, 0):
                 print()
 
         except:
             # It's possible to have exceptions raised here, typically by
             # compilation of odd code (such as a naked 'return' outside a
             # function) that did parse but isn't valid. Typically the exception
             # is a SyntaxError, but it's safest just to catch anything and show
             # the user a traceback.
 
             # We do only one try/except outside the loop to minimize the impact
             # on runtime, and also because if any node in the node list is
             # broken, we should stop execution completely.
             if result:
                 result.error_before_exec = sys.exc_info()[1]
             self.showtraceback()
             return True
 
         return False
 
     def _async_exec(self, code_obj: types.CodeType, user_ns: dict):
         """
         Evaluate an asynchronous code object using a code runner
 
         Fake asynchronous execution of code_object in a namespace via a proxy namespace.
 
         Returns coroutine object, which can be executed via async loop runner
 
         WARNING: The semantics of `async_exec` are quite different from `exec`,
         in particular you can only pass a single namespace. It also return a
         handle to the value of the last things returned by code_object.
         """
 
         return eval(code_obj, user_ns)
 
     async def run_code(self, code_obj, result=None, *, async_=False):
         """Execute a code object.
 
         When an exception occurs, self.showtraceback() is called to display a
         traceback.
 
         Parameters
         ----------
         code_obj : code object
           A compiled code object, to be executed
         result : ExecutionResult, optional
           An object to store exceptions that occur during execution.
         async_ :  Bool (Experimental)
           Attempt to run top-level asynchronous code in a default loop.
 
         Returns
         -------
         False : successful execution.
         True : an error occurred.
         """
         # special value to say that anything above is IPython and should be
         # hidden.
         __tracebackhide__ = "__ipython_bottom__"
         # Set our own excepthook in case the user code tries to call it
         # directly, so that the IPython crash handler doesn't get triggered
         old_excepthook, sys.excepthook = sys.excepthook, self.excepthook
 
         # we save the original sys.excepthook in the instance, in case config
         # code (such as magics) needs access to it.
         self.sys_excepthook = old_excepthook
         outflag = True  # happens in more places, so it's easier as default
         try:
             try:
                 self.hooks.pre_run_code_hook()
                 if async_ and sys.version_info < (3,8):
                     last_expr = (await self._async_exec(code_obj, self.user_ns))
                     code = compile('last_expr', 'fake', "single")
                     exec(code, {'last_expr': last_expr})
                 elif async_ :
                     await eval(code_obj, self.user_global_ns, self.user_ns)
                 else:
                     exec(code_obj, self.user_global_ns, self.user_ns)
             finally:
                 # Reset our crash handler in place
                 sys.excepthook = old_excepthook
         except SystemExit as e:
             if result is not None:
                 result.error_in_exec = e
             self.showtraceback(exception_only=True)
             warn("To exit: use 'exit', 'quit', or Ctrl-D.", stacklevel=1)
         except self.custom_exceptions:
             etype, value, tb = sys.exc_info()
             if result is not None:
                 result.error_in_exec = value
             self.CustomTB(etype, value, tb)
         except:
             if result is not None:
                 result.error_in_exec = sys.exc_info()[1]
             self.showtraceback(running_compiled_code=True)
         else:
             outflag = False
         return outflag
 
     # For backwards compatibility
     runcode = run_code
 
     def check_complete(self, code: str) -> Tuple[str, str]:
         """Return whether a block of code is ready to execute, or should be continued
 
         Parameters
         ----------
         source : string
           Python input code, which can be multiline.
 
         Returns
         -------
         status : str
           One of 'complete', 'incomplete', or 'invalid' if source is not a
           prefix of valid code.
         indent : str
           When status is 'incomplete', this is some whitespace to insert on
           the next line of the prompt.
         """
         status, nspaces = self.input_transformer_manager.check_complete(code)
         return status, ' ' * (nspaces or 0)
 
     #-------------------------------------------------------------------------
     # Things related to GUI support and pylab
     #-------------------------------------------------------------------------
 
     active_eventloop = None
 
     def enable_gui(self, gui=None):
         raise NotImplementedError('Implement enable_gui in a subclass')
 
     def enable_matplotlib(self, gui=None):
         """Enable interactive matplotlib and inline figure support.
 
         This takes the following steps:
 
         1. select the appropriate eventloop and matplotlib backend
         2. set up matplotlib for interactive use with that backend
         3. configure formatters for inline figure display
         4. enable the selected gui eventloop
 
         Parameters
         ----------
         gui : optional, string
           If given, dictates the choice of matplotlib GUI backend to use
           (should be one of IPython's supported backends, 'qt', 'osx', 'tk',
           'gtk', 'wx' or 'inline'), otherwise we use the default chosen by
           matplotlib (as dictated by the matplotlib build-time options plus the
           user's matplotlibrc configuration file).  Note that not all backends
           make sense in all contexts, for example a terminal ipython can't
           display figures inline.
         """
         from IPython.core import pylabtools as pt
         from matplotlib_inline.backend_inline import configure_inline_support
         gui, backend = pt.find_gui_and_backend(gui, self.pylab_gui_select)
 
         if gui != 'inline':
             # If we have our first gui selection, store it
             if self.pylab_gui_select is None:
                 self.pylab_gui_select = gui
             # Otherwise if they are different
             elif gui != self.pylab_gui_select:
                 print('Warning: Cannot change to a different GUI toolkit: %s.'
                         ' Using %s instead.' % (gui, self.pylab_gui_select))
                 gui, backend = pt.find_gui_and_backend(self.pylab_gui_select)
 
         pt.activate_matplotlib(backend)
         configure_inline_support(self, backend)
 
         # Now we must activate the gui pylab wants to use, and fix %run to take
         # plot updates into account
         self.enable_gui(gui)
         self.magics_manager.registry['ExecutionMagics'].default_runner = \
             pt.mpl_runner(self.safe_execfile)
 
         return gui, backend
 
     def enable_pylab(self, gui=None, import_all=True, welcome_message=False):
         """Activate pylab support at runtime.
 
         This turns on support for matplotlib, preloads into the interactive
         namespace all of numpy and pylab, and configures IPython to correctly
         interact with the GUI event loop.  The GUI backend to be used can be
         optionally selected with the optional ``gui`` argument.
 
         This method only adds preloading the namespace to InteractiveShell.enable_matplotlib.
 
         Parameters
         ----------
         gui : optional, string
           If given, dictates the choice of matplotlib GUI backend to use
           (should be one of IPython's supported backends, 'qt', 'osx', 'tk',
           'gtk', 'wx' or 'inline'), otherwise we use the default chosen by
           matplotlib (as dictated by the matplotlib build-time options plus the
           user's matplotlibrc configuration file).  Note that not all backends
           make sense in all contexts, for example a terminal ipython can't
           display figures inline.
         import_all : optional, bool, default: True
           Whether to do `from numpy import *` and `from pylab import *`
           in addition to module imports.
         welcome_message : deprecated
           This argument is ignored, no welcome message will be displayed.
         """
         from IPython.core.pylabtools import import_pylab
 
         gui, backend = self.enable_matplotlib(gui)
 
         # We want to prevent the loading of pylab to pollute the user's
         # namespace as shown by the %who* magics, so we execute the activation
         # code in an empty namespace, and we update *both* user_ns and
         # user_ns_hidden with this information.
         ns = {}
         import_pylab(ns, import_all)
         # warn about clobbered names
         ignored = {"__builtins__"}
         both = set(ns).intersection(self.user_ns).difference(ignored)
         clobbered = [ name for name in both if self.user_ns[name] is not ns[name] ]
         self.user_ns.update(ns)
         self.user_ns_hidden.update(ns)
         return gui, backend, clobbered
 
     #-------------------------------------------------------------------------
     # Utilities
     #-------------------------------------------------------------------------
 
     def var_expand(self, cmd, depth=0, formatter=DollarFormatter()):
         """Expand python variables in a string.
 
         The depth argument indicates how many frames above the caller should
         be walked to look for the local namespace where to expand variables.
 
         The global namespace for expansion is always the user's interactive
         namespace.
         """
         ns = self.user_ns.copy()
         try:
             frame = sys._getframe(depth+1)
         except ValueError:
             # This is thrown if there aren't that many frames on the stack,
             # e.g. if a script called run_line_magic() directly.
             pass
         else:
             ns.update(frame.f_locals)
 
         try:
             # We have to use .vformat() here, because 'self' is a valid and common
             # name, and expanding **ns for .format() would make it collide with
             # the 'self' argument of the method.
             cmd = formatter.vformat(cmd, args=[], kwargs=ns)
         except Exception:
             # if formatter couldn't format, just let it go untransformed
             pass
         return cmd
 
     def mktempfile(self, data=None, prefix='ipython_edit_'):
         """Make a new tempfile and return its filename.
 
         This makes a call to tempfile.mkstemp (created in a tempfile.mkdtemp),
         but it registers the created filename internally so ipython cleans it up
         at exit time.
 
         Optional inputs:
 
           - data(None): if data is given, it gets written out to the temp file
             immediately, and the file is closed again."""
 
         dirname = tempfile.mkdtemp(prefix=prefix)
         self.tempdirs.append(dirname)
 
         handle, filename = tempfile.mkstemp('.py', prefix, dir=dirname)
         os.close(handle)  # On Windows, there can only be one open handle on a file
         self.tempfiles.append(filename)
 
         if data:
             with open(filename, 'w') as tmp_file:
                 tmp_file.write(data)
         return filename
 
     @undoc
     def write(self,data):
         """DEPRECATED: Write a string to the default output"""
         warn('InteractiveShell.write() is deprecated, use sys.stdout instead',
              DeprecationWarning, stacklevel=2)
         sys.stdout.write(data)
 
     @undoc
     def write_err(self,data):
         """DEPRECATED: Write a string to the default error output"""
         warn('InteractiveShell.write_err() is deprecated, use sys.stderr instead',
              DeprecationWarning, stacklevel=2)
         sys.stderr.write(data)
 
     def ask_yes_no(self, prompt, default=None, interrupt=None):
         if self.quiet:
             return True
         return ask_yes_no(prompt,default,interrupt)
 
     def show_usage(self):
         """Show a usage message"""
         page.page(IPython.core.usage.interactive_usage)
 
     def extract_input_lines(self, range_str, raw=False):
         """Return as a string a set of input history slices.
 
         Parameters
         ----------
         range_str : string
             The set of slices is given as a string, like "~5/6-~4/2 4:8 9",
             since this function is for use by magic functions which get their
             arguments as strings. The number before the / is the session
             number: ~n goes n back from the current session.
 
         raw : bool, optional
             By default, the processed input is used.  If this is true, the raw
             input history is used instead.
 
         Notes
         -----
 
         Slices can be described with two notations:
 
         * ``N:M`` -> standard python form, means including items N...(M-1).
         * ``N-M`` -> include items N..M (closed endpoint).
         """
         lines = self.history_manager.get_range_by_str(range_str, raw=raw)
         return "\n".join(x for _, _, x in lines)
 
     def find_user_code(self, target, raw=True, py_only=False, skip_encoding_cookie=True, search_ns=False):
         """Get a code string from history, file, url, or a string or macro.
 
         This is mainly used by magic functions.
 
         Parameters
         ----------
 
         target : str
 
           A string specifying code to retrieve. This will be tried respectively
           as: ranges of input history (see %history for syntax), url,
           corresponding .py file, filename, or an expression evaluating to a
           string or Macro in the user namespace.
 
         raw : bool
           If true (default), retrieve raw history. Has no effect on the other
           retrieval mechanisms.
 
         py_only : bool (default False)
           Only try to fetch python code, do not try alternative methods to decode file
           if unicode fails.
 
         Returns
         -------
         A string of code.
 
         ValueError is raised if nothing is found, and TypeError if it evaluates
         to an object of another type. In each case, .args[0] is a printable
         message.
         """
         code = self.extract_input_lines(target, raw=raw)  # Grab history
         if code:
             return code
         try:
             if target.startswith(('http://', 'https://')):
                 return openpy.read_py_url(target, skip_encoding_cookie=skip_encoding_cookie)
         except UnicodeDecodeError:
             if not py_only :
                 # Deferred import
                 from urllib.request import urlopen
                 response = urlopen(target)
                 return response.read().decode('latin1')
             raise ValueError(("'%s' seem to be unreadable.") % target)
 
         potential_target = [target]
         try :
             potential_target.insert(0,get_py_filename(target))
         except IOError:
             pass
 
         for tgt in potential_target :
             if os.path.isfile(tgt):                        # Read file
                 try :
                     return openpy.read_py_file(tgt, skip_encoding_cookie=skip_encoding_cookie)
                 except UnicodeDecodeError :
                     if not py_only :
                         with io_open(tgt,'r', encoding='latin1') as f :
                             return f.read()
                     raise ValueError(("'%s' seem to be unreadable.") % target)
             elif os.path.isdir(os.path.expanduser(tgt)):
                 raise ValueError("'%s' is a directory, not a regular file." % target)
 
         if search_ns:
             # Inspect namespace to load object source
             object_info = self.object_inspect(target, detail_level=1)
             if object_info['found'] and object_info['source']:
                 return object_info['source']
 
         try:                                              # User namespace
             codeobj = eval(target, self.user_ns)
         except Exception:
             raise ValueError(("'%s' was not found in history, as a file, url, "
                                 "nor in the user namespace.") % target)
 
         if isinstance(codeobj, str):
             return codeobj
         elif isinstance(codeobj, Macro):
             return codeobj.value
 
         raise TypeError("%s is neither a string nor a macro." % target,
                         codeobj)
 
     #-------------------------------------------------------------------------
     # Things related to IPython exiting
     #-------------------------------------------------------------------------
     def atexit_operations(self):
         """This will be executed at the time of exit.
 
         Cleanup operations and saving of persistent data that is done
         unconditionally by IPython should be performed here.
 
         For things that may depend on startup flags or platform specifics (such
         as having readline or not), register a separate atexit function in the
         code that has the appropriate information, rather than trying to
         clutter
         """
         # Close the history session (this stores the end time and line count)
         # this must be *before* the tempfile cleanup, in case of temporary
         # history db
         self.history_manager.end_session()
 
         # Cleanup all tempfiles and folders left around
         for tfile in self.tempfiles:
             try:
                 os.unlink(tfile)
             except OSError:
                 pass
 
         for tdir in self.tempdirs:
             try:
                 os.rmdir(tdir)
             except OSError:
                 pass
 
         # Clear all user namespaces to release all references cleanly.
         self.reset(new_session=False)
 
         # Run user hooks
         self.hooks.shutdown_hook()
 
     def cleanup(self):
         self.restore_sys_module_state()
 
 
     # Overridden in terminal subclass to change prompts
     def switch_doctest_mode(self, mode):
         pass
 
 
 class InteractiveShellABC(metaclass=abc.ABCMeta):
     """An abstract base class for InteractiveShell."""
 
 InteractiveShellABC.register(InteractiveShell)

IPython/core/magic.py

@@ -1,703 +1,749 b''
 # encoding: utf-8
 """Magic functions for InteractiveShell.
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2001 Janko Hauser <[e-mail]> and
 #  Copyright (C) 2001 Fernando Perez <[e-mail]>
 #  Copyright (C) 2008 The IPython Development Team
 
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 import os
 import re
 import sys
 from getopt import getopt, GetoptError
 
 from traitlets.config.configurable import Configurable
 from . import oinspect
 from .error import UsageError
 from .inputtransformer2 import ESC_MAGIC, ESC_MAGIC2
 from decorator import decorator
 from ..utils.ipstruct import Struct
 from ..utils.process import arg_split
 from ..utils.text import dedent
 from traitlets import Bool, Dict, Instance, observe
 from logging import error
 
 #-----------------------------------------------------------------------------
 # Globals
 #-----------------------------------------------------------------------------
 
 # A dict we'll use for each class that has magics, used as temporary storage to
 # pass information between the @line/cell_magic method decorators and the
 # @magics_class class decorator, because the method decorators have no
 # access to the class when they run.  See for more details:
 # http://stackoverflow.com/questions/2366713/can-a-python-decorator-of-an-instance-method-access-the-class
 
 magics = dict(line={}, cell={})
 
 magic_kinds = ('line', 'cell')
 magic_spec = ('line', 'cell', 'line_cell')
 magic_escapes = dict(line=ESC_MAGIC, cell=ESC_MAGIC2)
 
 #-----------------------------------------------------------------------------
 # Utility classes and functions
 #-----------------------------------------------------------------------------
 
 class Bunch: pass
 
 
 def on_off(tag):
     """Return an ON/OFF string for a 1/0 input. Simple utility function."""
     return ['OFF','ON'][tag]
 
 
 def compress_dhist(dh):
     """Compress a directory history into a new one with at most 20 entries.
 
     Return a new list made from the first and last 10 elements of dhist after
     removal of duplicates.
     """
     head, tail = dh[:-10], dh[-10:]
 
     newhead = []
     done = set()
     for h in head:
         if h in done:
             continue
         newhead.append(h)
         done.add(h)
 
     return newhead + tail
 
 
 def needs_local_scope(func):
     """Decorator to mark magic functions which need to local scope to run."""
     func.needs_local_scope = True
     return func
 
 #-----------------------------------------------------------------------------
 # Class and method decorators for registering magics
 #-----------------------------------------------------------------------------
 
 def magics_class(cls):
     """Class decorator for all subclasses of the main Magics class.
 
     Any class that subclasses Magics *must* also apply this decorator, to
     ensure that all the methods that have been decorated as line/cell magics
     get correctly registered in the class instance.  This is necessary because
     when method decorators run, the class does not exist yet, so they
     temporarily store their information into a module global.  Application of
     this class decorator copies that global data to the class instance and
     clears the global.
 
     Obviously, this mechanism is not thread-safe, which means that the
     *creation* of subclasses of Magic should only be done in a single-thread
     context.  Instantiation of the classes has no restrictions.  Given that
     these classes are typically created at IPython startup time and before user
     application code becomes active, in practice this should not pose any
     problems.
     """
     cls.registered = True
     cls.magics = dict(line = magics['line'],
                       cell = magics['cell'])
     magics['line'] = {}
     magics['cell'] = {}
     return cls
 
 
 def record_magic(dct, magic_kind, magic_name, func):
     """Utility function to store a function as a magic of a specific kind.
 
     Parameters
     ----------
     dct : dict
       A dictionary with 'line' and 'cell' subdicts.
 
     magic_kind : str
       Kind of magic to be stored.
 
     magic_name : str
       Key to store the magic as.
 
     func : function
       Callable object to store.
     """
     if magic_kind == 'line_cell':
         dct['line'][magic_name] = dct['cell'][magic_name] = func
     else:
         dct[magic_kind][magic_name] = func
 
 
 def validate_type(magic_kind):
     """Ensure that the given magic_kind is valid.
 
     Check that the given magic_kind is one of the accepted spec types (stored
     in the global `magic_spec`), raise ValueError otherwise.
     """
     if magic_kind not in magic_spec:
         raise ValueError('magic_kind must be one of %s, %s given' %
                          magic_kinds, magic_kind)
 
 
 # The docstrings for the decorator below will be fairly similar for the two
 # types (method and function), so we generate them here once and reuse the
 # templates below.
 _docstring_template = \
 """Decorate the given {0} as {1} magic.
 
 The decorator can be used with or without arguments, as follows.
 
 i) without arguments: it will create a {1} magic named as the {0} being
 decorated::
 
     @deco
     def foo(...)
 
 will create a {1} magic named `foo`.
 
 ii) with one string argument: which will be used as the actual name of the
 resulting magic::
 
     @deco('bar')
     def foo(...)
 
 will create a {1} magic named `bar`.
 
 To register a class magic use ``Interactiveshell.register_magic(class or instance)``.
 """
 
 # These two are decorator factories.  While they are conceptually very similar,
 # there are enough differences in the details that it's simpler to have them
 # written as completely standalone functions rather than trying to share code
 # and make a single one with convoluted logic.
 
 def _method_magic_marker(magic_kind):
     """Decorator factory for methods in Magics subclasses.
     """
 
     validate_type(magic_kind)
 
     # This is a closure to capture the magic_kind.  We could also use a class,
     # but it's overkill for just that one bit of state.
     def magic_deco(arg):
         call = lambda f, *a, **k: f(*a, **k)
 
         if callable(arg):
             # "Naked" decorator call (just @foo, no args)
             func = arg
             name = func.__name__
             retval = decorator(call, func)
             record_magic(magics, magic_kind, name, name)
         elif isinstance(arg, str):
             # Decorator called with arguments (@foo('bar'))
             name = arg
             def mark(func, *a, **kw):
                 record_magic(magics, magic_kind, name, func.__name__)
                 return decorator(call, func)
             retval = mark
         else:
             raise TypeError("Decorator can only be called with "
                             "string or function")
         return retval
 
     # Ensure the resulting decorator has a usable docstring
     magic_deco.__doc__ = _docstring_template.format('method', magic_kind)
     return magic_deco
 
 
 def _function_magic_marker(magic_kind):
     """Decorator factory for standalone functions.
     """
     validate_type(magic_kind)
     
     # This is a closure to capture the magic_kind.  We could also use a class,
     # but it's overkill for just that one bit of state.
     def magic_deco(arg):
         call = lambda f, *a, **k: f(*a, **k)
 
         # Find get_ipython() in the caller's namespace
         caller = sys._getframe(1)
         for ns in ['f_locals', 'f_globals', 'f_builtins']:
             get_ipython = getattr(caller, ns).get('get_ipython')
             if get_ipython is not None:
                 break
         else:
             raise NameError('Decorator can only run in context where '
                             '`get_ipython` exists')
 
         ip = get_ipython()
 
         if callable(arg):
             # "Naked" decorator call (just @foo, no args)
             func = arg
             name = func.__name__
             ip.register_magic_function(func, magic_kind, name)
             retval = decorator(call, func)
         elif isinstance(arg, str):
             # Decorator called with arguments (@foo('bar'))
             name = arg
             def mark(func, *a, **kw):
                 ip.register_magic_function(func, magic_kind, name)
                 return decorator(call, func)
             retval = mark
         else:
             raise TypeError("Decorator can only be called with "
                              "string or function")
         return retval
 
     # Ensure the resulting decorator has a usable docstring
     ds = _docstring_template.format('function', magic_kind)
 
     ds += dedent("""
     Note: this decorator can only be used in a context where IPython is already
     active, so that the `get_ipython()` call succeeds.  You can therefore use
     it in your startup files loaded after IPython initializes, but *not* in the
     IPython configuration file itself, which is executed before IPython is
     fully up and running.  Any file located in the `startup` subdirectory of
     your configuration profile will be OK in this sense.
     """)
     
     magic_deco.__doc__ = ds
     return magic_deco
 
 
 MAGIC_NO_VAR_EXPAND_ATTR = '_ipython_magic_no_var_expand'
 
 
 def no_var_expand(magic_func):
     """Mark a magic function as not needing variable expansion
 
     By default, IPython interprets `{a}` or `$a` in the line passed to magics
     as variables that should be interpolated from the interactive namespace
     before passing the line to the magic function.
     This is not always desirable, e.g. when the magic executes Python code
     (%timeit, %time, etc.).
     Decorate magics with `@no_var_expand` to opt-out of variable expansion.
 
     .. versionadded:: 7.3
     """
     setattr(magic_func, MAGIC_NO_VAR_EXPAND_ATTR, True)
     return magic_func
 
 
 # Create the actual decorators for public use
 
 # These three are used to decorate methods in class definitions
 line_magic = _method_magic_marker('line')
 cell_magic = _method_magic_marker('cell')
 line_cell_magic = _method_magic_marker('line_cell')
 
 # These three decorate standalone functions and perform the decoration
 # immediately.  They can only run where get_ipython() works
 register_line_magic = _function_magic_marker('line')
 register_cell_magic = _function_magic_marker('cell')
 register_line_cell_magic = _function_magic_marker('line_cell')
 
 #-----------------------------------------------------------------------------
 # Core Magic classes
 #-----------------------------------------------------------------------------
 
 class MagicsManager(Configurable):
     """Object that handles all magic-related functionality for IPython.
     """
     # Non-configurable class attributes
 
     # A two-level dict, first keyed by magic type, then by magic function, and
     # holding the actual callable object as value.  This is the dict used for
     # magic function dispatch
     magics = Dict()
+    lazy_magics = Dict(
+        help="""
+    Mapping from magic names to modules to load.
+
+    This can be used in IPython/IPykernel configuration to declare lazy magics
+    that will only be imported/registered on first use.
+
+    For example::
+
+        c.MagicsManger.lazy_magics = {
+          "my_magic": "slow.to.import",
+          "my_other_magic": "also.slow",
+        }
+
+    On first invocation of `%my_magic`, `%%my_magic`, `%%my_other_magic` or
+    `%%my_other_magic`, the corresponding module will be loaded as an ipython
+    extensions as if you had previously done `%load_ext ipython`.
+
+    Magics names should be without percent(s) as magics can be both cell
+    and line magics.
+
+    Lazy loading happen relatively late in execution process, and
+    complex extensions that manipulate Python/IPython internal state or global state
+    might not support lazy loading.
+    """
+    ).tag(
+        config=True,
+    )
 
     # A registry of the original objects that we've been given holding magics.
     registry = Dict()
 
     shell = Instance('IPython.core.interactiveshell.InteractiveShellABC', allow_none=True)
 
     auto_magic = Bool(True, help=
         "Automatically call line magics without requiring explicit % prefix"
     ).tag(config=True)
     @observe('auto_magic')
     def _auto_magic_changed(self, change):
         self.shell.automagic = change['new']
     
     _auto_status = [
         'Automagic is OFF, % prefix IS needed for line magics.',
         'Automagic is ON, % prefix IS NOT needed for line magics.']
 
     user_magics = Instance('IPython.core.magics.UserMagics', allow_none=True)
 
     def __init__(self, shell=None, config=None, user_magics=None, **traits):
 
         super(MagicsManager, self).__init__(shell=shell, config=config,
                                            user_magics=user_magics, **traits)
         self.magics = dict(line={}, cell={})
         # Let's add the user_magics to the registry for uniformity, so *all*
         # registered magic containers can be found there.
         self.registry[user_magics.__class__.__name__] = user_magics
 
     def auto_status(self):
         """Return descriptive string with automagic status."""
         return self._auto_status[self.auto_magic]
     
     def lsmagic(self):
         """Return a dict of currently available magic functions.
 
         The return dict has the keys 'line' and 'cell', corresponding to the
         two types of magics we support.  Each value is a list of names.
         """
         return self.magics
 
     def lsmagic_docs(self, brief=False, missing=''):
         """Return dict of documentation of magic functions.
 
         The return dict has the keys 'line' and 'cell', corresponding to the
         two types of magics we support. Each value is a dict keyed by magic
         name whose value is the function docstring. If a docstring is
         unavailable, the value of `missing` is used instead.
 
         If brief is True, only the first line of each docstring will be returned.
         """
         docs = {}
         for m_type in self.magics:
             m_docs = {}
             for m_name, m_func in self.magics[m_type].items():
                 if m_func.__doc__:
                     if brief:
                         m_docs[m_name] = m_func.__doc__.split('\n', 1)[0]
                     else:
                         m_docs[m_name] = m_func.__doc__.rstrip()
                 else:
                     m_docs[m_name] = missing
             docs[m_type] = m_docs
         return docs
 
+    def register_lazy(self, name: str, fully_qualified_name: str):
+        """
+        Lazily register a magic via an extension.
+
+
+        Parameters
+        ----------
+        name : str
+            Name of the magic you wish to register.
+        fully_qualified_name :
+            Fully qualified name of the module/submodule that should be loaded
+            as an extensions when the magic is first called.
+            It is assumed that loading this extensions will register the given
+            magic.
+        """
+
+        self.lazy_magics[name] = fully_qualified_name
+
     def register(self, *magic_objects):
         """Register one or more instances of Magics.
 
         Take one or more classes or instances of classes that subclass the main 
         `core.Magic` class, and register them with IPython to use the magic
         functions they provide.  The registration process will then ensure that
         any methods that have decorated to provide line and/or cell magics will
         be recognized with the `%x`/`%%x` syntax as a line/cell magic
         respectively.
 
         If classes are given, they will be instantiated with the default
         constructor.  If your classes need a custom constructor, you should
         instanitate them first and pass the instance.
 
         The provided arguments can be an arbitrary mix of classes and instances.
 
         Parameters
         ----------
         magic_objects : one or more classes or instances
         """
         # Start by validating them to ensure they have all had their magic
         # methods registered at the instance level
         for m in magic_objects:
             if not m.registered:
                 raise ValueError("Class of magics %r was constructed without "
                                  "the @register_magics class decorator")
             if isinstance(m, type):
                 # If we're given an uninstantiated class
                 m = m(shell=self.shell)
 
             # Now that we have an instance, we can register it and update the
             # table of callables
             self.registry[m.__class__.__name__] = m
             for mtype in magic_kinds:
                 self.magics[mtype].update(m.magics[mtype])
 
     def register_function(self, func, magic_kind='line', magic_name=None):
         """Expose a standalone function as magic function for IPython.
 
         This will create an IPython magic (line, cell or both) from a
         standalone function.  The functions should have the following
         signatures: 
 
         * For line magics: `def f(line)`
         * For cell magics: `def f(line, cell)`
         * For a function that does both: `def f(line, cell=None)`
 
         In the latter case, the function will be called with `cell==None` when
         invoked as `%f`, and with cell as a string when invoked as `%%f`.
 
         Parameters
         ----------
         func : callable
           Function to be registered as a magic.
 
         magic_kind : str
           Kind of magic, one of 'line', 'cell' or 'line_cell'
 
         magic_name : optional str
           If given, the name the magic will have in the IPython namespace.  By
           default, the name of the function itself is used.
         """
 
         # Create the new method in the user_magics and register it in the
         # global table
         validate_type(magic_kind)
         magic_name = func.__name__ if magic_name is None else magic_name
         setattr(self.user_magics, magic_name, func)
         record_magic(self.magics, magic_kind, magic_name, func)
 
     def register_alias(self, alias_name, magic_name, magic_kind='line', magic_params=None):
         """Register an alias to a magic function.
 
         The alias is an instance of :class:`MagicAlias`, which holds the
         name and kind of the magic it should call. Binding is done at
         call time, so if the underlying magic function is changed the alias
         will call the new function.
 
         Parameters
         ----------
         alias_name : str
           The name of the magic to be registered.
 
         magic_name : str
           The name of an existing magic.
 
         magic_kind : str
           Kind of magic, one of 'line' or 'cell'
         """
 
         # `validate_type` is too permissive, as it allows 'line_cell'
         # which we do not handle.
         if magic_kind not in magic_kinds:
             raise ValueError('magic_kind must be one of %s, %s given' %
                              magic_kinds, magic_kind)
 
         alias = MagicAlias(self.shell, magic_name, magic_kind, magic_params)
         setattr(self.user_magics, alias_name, alias)
         record_magic(self.magics, magic_kind, alias_name, alias)
 
 # Key base class that provides the central functionality for magics.
 
 
 class Magics(Configurable):
     """Base class for implementing magic functions.
 
     Shell functions which can be reached as %function_name. All magic
     functions should accept a string, which they can parse for their own
     needs. This can make some functions easier to type, eg `%cd ../`
     vs. `%cd("../")`
 
     Classes providing magic functions need to subclass this class, and they
     MUST:
 
     - Use the method decorators `@line_magic` and `@cell_magic` to decorate
       individual methods as magic functions, AND
 
     - Use the class decorator `@magics_class` to ensure that the magic
       methods are properly registered at the instance level upon instance
       initialization.
 
     See :mod:`magic_functions` for examples of actual implementation classes.
     """
     # Dict holding all command-line options for each magic.
     options_table = None
     # Dict for the mapping of magic names to methods, set by class decorator
     magics = None
     # Flag to check that the class decorator was properly applied
     registered = False
     # Instance of IPython shell
     shell = None
 
     def __init__(self, shell=None, **kwargs):
         if not(self.__class__.registered):
             raise ValueError('Magics subclass without registration - '
                              'did you forget to apply @magics_class?')
         if shell is not None:
             if hasattr(shell, 'configurables'):
                 shell.configurables.append(self)
             if hasattr(shell, 'config'):
                 kwargs.setdefault('parent', shell)
 
         self.shell = shell
         self.options_table = {}
         # The method decorators are run when the instance doesn't exist yet, so
         # they can only record the names of the methods they are supposed to
         # grab.  Only now, that the instance exists, can we create the proper
         # mapping to bound methods.  So we read the info off the original names
         # table and replace each method name by the actual bound method.
         # But we mustn't clobber the *class* mapping, in case of multiple instances.
         class_magics = self.magics
         self.magics = {}
         for mtype in magic_kinds:
             tab = self.magics[mtype] = {}
             cls_tab = class_magics[mtype]
             for magic_name, meth_name in cls_tab.items():
                 if isinstance(meth_name, str):
                     # it's a method name, grab it
                     tab[magic_name] = getattr(self, meth_name)
                 else:
                     # it's the real thing
                     tab[magic_name] = meth_name
         # Configurable **needs** to be initiated at the end or the config
         # magics get screwed up.
         super(Magics, self).__init__(**kwargs)
 
     def arg_err(self,func):
         """Print docstring if incorrect arguments were passed"""
         print('Error in arguments:')
         print(oinspect.getdoc(func))
 
     def format_latex(self, strng):
         """Format a string for latex inclusion."""
 
         # Characters that need to be escaped for latex:
         escape_re = re.compile(r'(%|_|\$|#|&)',re.MULTILINE)
         # Magic command names as headers:
         cmd_name_re = re.compile(r'^(%s.*?):' % ESC_MAGIC,
                                  re.MULTILINE)
         # Magic commands
         cmd_re = re.compile(r'(?P<cmd>%s.+?\b)(?!\}\}:)' % ESC_MAGIC,
                             re.MULTILINE)
         # Paragraph continue
         par_re = re.compile(r'\\$',re.MULTILINE)
 
         # The "\n" symbol
         newline_re = re.compile(r'\\n')
 
         # Now build the string for output:
         #strng = cmd_name_re.sub(r'\n\\texttt{\\textsl{\\large \1}}:',strng)
         strng = cmd_name_re.sub(r'\n\\bigskip\n\\texttt{\\textbf{ \1}}:',
                                 strng)
         strng = cmd_re.sub(r'\\texttt{\g<cmd>}',strng)
         strng = par_re.sub(r'\\\\',strng)
         strng = escape_re.sub(r'\\\1',strng)
         strng = newline_re.sub(r'\\textbackslash{}n',strng)
         return strng
 
     def parse_options(self, arg_str, opt_str, *long_opts, **kw):
         """Parse options passed to an argument string.
 
         The interface is similar to that of :func:`getopt.getopt`, but it
         returns a :class:`~IPython.utils.struct.Struct` with the options as keys
         and the stripped argument string still as a string.
 
         arg_str is quoted as a true sys.argv vector by using shlex.split.
         This allows us to easily expand variables, glob files, quote
         arguments, etc.
 
         Parameters
         ----------
 
         arg_str : str
           The arguments to parse.
 
         opt_str : str
           The options specification.
 
         mode : str, default 'string'
           If given as 'list', the argument string is returned as a list (split
           on whitespace) instead of a string.
 
         list_all : bool, default False
           Put all option values in lists. Normally only options
           appearing more than once are put in a list.
 
         posix : bool, default True
           Whether to split the input line in POSIX mode or not, as per the
           conventions outlined in the :mod:`shlex` module from the standard
           library.
         """
 
         # inject default options at the beginning of the input line
         caller = sys._getframe(1).f_code.co_name
         arg_str = '%s %s' % (self.options_table.get(caller,''),arg_str)
 
         mode = kw.get('mode','string')
         if mode not in ['string','list']:
             raise ValueError('incorrect mode given: %s' % mode)
         # Get options
         list_all = kw.get('list_all',0)
         posix = kw.get('posix', os.name == 'posix')
         strict = kw.get('strict', True)
 
         # Check if we have more than one argument to warrant extra processing:
         odict = {}  # Dictionary with options
         args = arg_str.split()
         if len(args) >= 1:
             # If the list of inputs only has 0 or 1 thing in it, there's no
             # need to look for options
             argv = arg_split(arg_str, posix, strict)
             # Do regular option processing
             try:
                 opts,args = getopt(argv, opt_str, long_opts)
             except GetoptError as e:
                 raise UsageError('%s ( allowed: "%s" %s)' % (e.msg,opt_str,
                                         " ".join(long_opts)))
             for o,a in opts:
                 if o.startswith('--'):
                     o = o[2:]
                 else:
                     o = o[1:]
                 try:
                     odict[o].append(a)
                 except AttributeError:
                     odict[o] = [odict[o],a]
                 except KeyError:
                     if list_all:
                         odict[o] = [a]
                     else:
                         odict[o] = a
 
         # Prepare opts,args for return
         opts = Struct(odict)
         if mode == 'string':
             args = ' '.join(args)
 
         return opts,args
 
     def default_option(self, fn, optstr):
         """Make an entry in the options_table for fn, with value optstr"""
 
         if fn not in self.lsmagic():
             error("%s is not a magic function" % fn)
         self.options_table[fn] = optstr
 
 
 class MagicAlias(object):
     """An alias to another magic function.
 
     An alias is determined by its magic name and magic kind. Lookup
     is done at call time, so if the underlying magic changes the alias
     will call the new function.
 
     Use the :meth:`MagicsManager.register_alias` method or the
     `%alias_magic` magic function to create and register a new alias.
     """
     def __init__(self, shell, magic_name, magic_kind, magic_params=None):
         self.shell = shell
         self.magic_name = magic_name
         self.magic_params = magic_params
         self.magic_kind = magic_kind
 
         self.pretty_target = '%s%s' % (magic_escapes[self.magic_kind], self.magic_name)
         self.__doc__ = "Alias for `%s`." % self.pretty_target
 
         self._in_call = False
 
     def __call__(self, *args, **kwargs):
         """Call the magic alias."""
         fn = self.shell.find_magic(self.magic_name, self.magic_kind)
         if fn is None:
             raise UsageError("Magic `%s` not found." % self.pretty_target)
 
         # Protect against infinite recursion.
         if self._in_call:
             raise UsageError("Infinite recursion detected; "
                              "magic aliases cannot call themselves.")
         self._in_call = True
         try:
             if self.magic_params:
                 args_list = list(args)
                 args_list[0] = self.magic_params + " " + args[0]
                 args = tuple(args_list)
             return fn(*args, **kwargs)
         finally:
             self._in_call = False

IPython/core/tests/test_magic.py

@@ -1,1238 +1,1286 b''
 # -*- coding: utf-8 -*-
 """Tests for various magic functions.
 
 Needs to be run by nose (to make ipython session available).
 """
 
 import io
 import os
 import re
 import sys
 import warnings
 from textwrap import dedent
 from unittest import TestCase
 from unittest import mock
 from importlib import invalidate_caches
 from io import StringIO
 
 import nose.tools as nt
 
 import shlex
 
 from IPython import get_ipython
 from IPython.core import magic
 from IPython.core.error import UsageError
 from IPython.core.magic import (Magics, magics_class, line_magic,
                                 cell_magic,
                                 register_line_magic, register_cell_magic)
 from IPython.core.magics import execution, script, code, logging, osm
 from IPython.testing import decorators as dec
 from IPython.testing import tools as tt
 from IPython.utils.io import capture_output
 from IPython.utils.tempdir import (TemporaryDirectory,
                                     TemporaryWorkingDirectory)
 from IPython.utils.process import find_cmd
+from IPython.utils.tempdir import TemporaryDirectory, TemporaryWorkingDirectory
+from IPython.utils.syspathcontext import prepended_to_syspath
+
 from .test_debugger import PdbTestInput
 
+from tempfile import NamedTemporaryFile
+
+import pytest
+from pathlib import Path
 
 @magic.magics_class
 class DummyMagics(magic.Magics): pass
 
 def test_extract_code_ranges():
     instr = "1 3 5-6 7-9 10:15 17: :10 10- -13 :"
     expected = [(0, 1), 
                 (2, 3),
                 (4, 6),
                 (6, 9),
                 (9, 14),
                 (16, None),
                 (None, 9),
                 (9, None),
                 (None, 13),
                 (None, None)]
     actual = list(code.extract_code_ranges(instr))
     nt.assert_equal(actual, expected)
 
 def test_extract_symbols():
     source = """import foo\na = 10\ndef b():\n    return 42\n\n\nclass A: pass\n\n\n"""
     symbols_args = ["a", "b", "A", "A,b", "A,a", "z"]
     expected = [([], ['a']),
                 (["def b():\n    return 42\n"], []),
                 (["class A: pass\n"], []),
                 (["class A: pass\n", "def b():\n    return 42\n"], []),
                 (["class A: pass\n"], ['a']),
                 ([], ['z'])]
     for symbols, exp in zip(symbols_args, expected):
         nt.assert_equal(code.extract_symbols(source, symbols), exp)
 
 
 def test_extract_symbols_raises_exception_with_non_python_code():
     source = ("=begin A Ruby program :)=end\n"
               "def hello\n"
               "puts 'Hello world'\n"
               "end")
     with nt.assert_raises(SyntaxError):
         code.extract_symbols(source, "hello")
 
 
 def test_magic_not_found():
     # magic not found raises UsageError
     with nt.assert_raises(UsageError):
         _ip.magic('doesntexist')
 
     # ensure result isn't success when a magic isn't found
     result = _ip.run_cell('%doesntexist')
     assert isinstance(result.error_in_exec, UsageError)
 
 
 def test_cell_magic_not_found():
     # magic not found raises UsageError
     with nt.assert_raises(UsageError):
         _ip.run_cell_magic('doesntexist', 'line', 'cell')
 
     # ensure result isn't success when a magic isn't found
     result = _ip.run_cell('%%doesntexist')
     assert isinstance(result.error_in_exec, UsageError)
 
 
 def test_magic_error_status():
     def fail(shell):
         1/0
     _ip.register_magic_function(fail)
     result = _ip.run_cell('%fail')
     assert isinstance(result.error_in_exec, ZeroDivisionError)
 
 
 def test_config():
     """ test that config magic does not raise
     can happen if Configurable init is moved too early into
     Magics.__init__ as then a Config object will be registered as a
     magic.
     """
     ## should not raise.
     _ip.magic('config')
 
 def test_config_available_configs():
     """ test that config magic prints available configs in unique and
     sorted order. """
     with capture_output() as captured:
         _ip.magic('config')
 
     stdout = captured.stdout
     config_classes = stdout.strip().split('\n')[1:]
     nt.assert_list_equal(config_classes, sorted(set(config_classes)))
 
 def test_config_print_class():
     """ test that config with a classname prints the class's options. """
     with capture_output() as captured:
         _ip.magic('config TerminalInteractiveShell')
 
     stdout = captured.stdout
     if not re.match("TerminalInteractiveShell.* options", stdout.splitlines()[0]):
         print(stdout)
         raise AssertionError("1st line of stdout not like "
                              "'TerminalInteractiveShell.* options'")
 
 def test_rehashx():
     # clear up everything
     _ip.alias_manager.clear_aliases()
     del _ip.db['syscmdlist']
     
     _ip.magic('rehashx')
     # Practically ALL ipython development systems will have more than 10 aliases
 
     nt.assert_true(len(_ip.alias_manager.aliases) > 10)
     for name, cmd in _ip.alias_manager.aliases:
         # we must strip dots from alias names
         nt.assert_not_in('.', name)
 
     # rehashx must fill up syscmdlist
     scoms = _ip.db['syscmdlist']
     nt.assert_true(len(scoms) > 10)
             
 
 
 def test_magic_parse_options():
     """Test that we don't mangle paths when parsing magic options."""
     ip = get_ipython()
     path = 'c:\\x'
     m = DummyMagics(ip)
     opts = m.parse_options('-f %s' % path,'f:')[0]
     # argv splitting is os-dependent
     if os.name == 'posix':
         expected = 'c:x'
     else:
         expected = path
     nt.assert_equal(opts['f'], expected)
 
 def test_magic_parse_long_options():
     """Magic.parse_options can handle --foo=bar long options"""
     ip = get_ipython()
     m = DummyMagics(ip)
     opts, _ = m.parse_options('--foo --bar=bubble', 'a', 'foo', 'bar=')
     nt.assert_in('foo', opts)
     nt.assert_in('bar', opts)
     nt.assert_equal(opts['bar'], "bubble")
 
 
 @dec.skip_without('sqlite3')
 def doctest_hist_f():
     """Test %hist -f with temporary filename.
 
     In [9]: import tempfile
 
     In [10]: tfile = tempfile.mktemp('.py','tmp-ipython-')
 
     In [11]: %hist -nl -f $tfile 3
 
     In [13]: import os; os.unlink(tfile)
     """
 
 
 @dec.skip_without('sqlite3')
 def doctest_hist_r():
     """Test %hist -r
 
     XXX - This test is not recording the output correctly.  For some reason, in
     testing mode the raw history isn't getting populated.  No idea why.
     Disabling the output checking for now, though at least we do run it.
 
     In [1]: 'hist' in _ip.lsmagic()
     Out[1]: True
 
     In [2]: x=1
 
     In [3]: %hist -rl 2
     x=1 # random
     %hist -r 2
     """
 
 
 @dec.skip_without('sqlite3')
 def doctest_hist_op():
     """Test %hist -op
 
     In [1]: class b(float):
        ...:     pass
        ...: 
 
     In [2]: class s(object):
        ...:     def __str__(self):
        ...:         return 's'
        ...: 
 
     In [3]: 
 
     In [4]: class r(b):
        ...:     def __repr__(self):
        ...:         return 'r'
        ...: 
 
     In [5]: class sr(s,r): pass
        ...: 
 
     In [6]: 
 
     In [7]: bb=b()
 
     In [8]: ss=s()
 
     In [9]: rr=r()
 
     In [10]: ssrr=sr()
 
     In [11]: 4.5
     Out[11]: 4.5
 
     In [12]: str(ss)
     Out[12]: 's'
 
     In [13]: 
 
     In [14]: %hist -op
     >>> class b:
     ...     pass
     ... 
     >>> class s(b):
     ...     def __str__(self):
     ...         return 's'
     ... 
     >>> 
     >>> class r(b):
     ...     def __repr__(self):
     ...         return 'r'
     ... 
     >>> class sr(s,r): pass
     >>> 
     >>> bb=b()
     >>> ss=s()
     >>> rr=r()
     >>> ssrr=sr()
     >>> 4.5
     4.5
     >>> str(ss)
     's'
     >>> 
     """
 
 def test_hist_pof():
     ip = get_ipython()
     ip.run_cell(u"1+2", store_history=True)
     #raise Exception(ip.history_manager.session_number)
     #raise Exception(list(ip.history_manager._get_range_session()))
     with TemporaryDirectory() as td:
         tf = os.path.join(td, 'hist.py')
         ip.run_line_magic('history', '-pof %s' % tf)
         assert os.path.isfile(tf)
 
 
 @dec.skip_without('sqlite3')
 def test_macro():
     ip = get_ipython()
     ip.history_manager.reset()   # Clear any existing history.
     cmds = ["a=1", "def b():\n  return a**2", "print(a,b())"]
     for i, cmd in enumerate(cmds, start=1):
         ip.history_manager.store_inputs(i, cmd)
     ip.magic("macro test 1-3")
     nt.assert_equal(ip.user_ns["test"].value, "\n".join(cmds)+"\n")
     
     # List macros
     nt.assert_in("test", ip.magic("macro"))
 
 
 @dec.skip_without('sqlite3')
 def test_macro_run():
     """Test that we can run a multi-line macro successfully."""
     ip = get_ipython()
     ip.history_manager.reset()
     cmds = ["a=10", "a+=1", "print(a)", "%macro test 2-3"]
     for cmd in cmds:
         ip.run_cell(cmd, store_history=True)
     nt.assert_equal(ip.user_ns["test"].value, "a+=1\nprint(a)\n")
     with tt.AssertPrints("12"):
         ip.run_cell("test")
     with tt.AssertPrints("13"):
         ip.run_cell("test")
 
 
 def test_magic_magic():
     """Test %magic"""
     ip = get_ipython()
     with capture_output() as captured:
         ip.magic("magic")
     
     stdout = captured.stdout
     nt.assert_in('%magic', stdout)
     nt.assert_in('IPython', stdout)
     nt.assert_in('Available', stdout)
 
 
 @dec.skipif_not_numpy
 def test_numpy_reset_array_undec():
     "Test '%reset array' functionality"
     _ip.ex('import numpy as np')
     _ip.ex('a = np.empty(2)')
     nt.assert_in('a', _ip.user_ns)
     _ip.magic('reset -f array')
     nt.assert_not_in('a', _ip.user_ns)
 
 def test_reset_out():
     "Test '%reset out' magic"
     _ip.run_cell("parrot = 'dead'", store_history=True)
     # test '%reset -f out', make an Out prompt
     _ip.run_cell("parrot", store_history=True)
     nt.assert_true('dead' in [_ip.user_ns[x] for x in ('_','__','___')])
     _ip.magic('reset -f out')
     nt.assert_false('dead' in [_ip.user_ns[x] for x in ('_','__','___')])
     nt.assert_equal(len(_ip.user_ns['Out']), 0)
 
 def test_reset_in():
     "Test '%reset in' magic"
     # test '%reset -f in'
     _ip.run_cell("parrot", store_history=True)
     nt.assert_true('parrot' in [_ip.user_ns[x] for x in ('_i','_ii','_iii')])
     _ip.magic('%reset -f in')
     nt.assert_false('parrot' in [_ip.user_ns[x] for x in ('_i','_ii','_iii')])
     nt.assert_equal(len(set(_ip.user_ns['In'])), 1)
 
 def test_reset_dhist():
     "Test '%reset dhist' magic"
     _ip.run_cell("tmp = [d for d in _dh]") # copy before clearing
     _ip.magic('cd ' + os.path.dirname(nt.__file__))
     _ip.magic('cd -')
     nt.assert_true(len(_ip.user_ns['_dh']) > 0)
     _ip.magic('reset -f dhist')
     nt.assert_equal(len(_ip.user_ns['_dh']), 0)
     _ip.run_cell("_dh = [d for d in tmp]") #restore
 
 def test_reset_in_length():
     "Test that '%reset in' preserves In[] length"
     _ip.run_cell("print 'foo'")
     _ip.run_cell("reset -f in")
     nt.assert_equal(len(_ip.user_ns['In']), _ip.displayhook.prompt_count+1)
 
 class TestResetErrors(TestCase):
 
     def test_reset_redefine(self):
 
         @magics_class
         class KernelMagics(Magics):
               @line_magic
               def less(self, shell): pass
 
         _ip.register_magics(KernelMagics)
 
         with self.assertLogs() as cm:
             # hack, we want to just capture logs, but assertLogs fails if not
             # logs get produce.
             # so log one things we ignore.
             import logging as log_mod
             log = log_mod.getLogger()
             log.info('Nothing')
             # end hack.
             _ip.run_cell("reset -f")
 
         assert len(cm.output) == 1
         for out in cm.output:
             assert "Invalid alias" not in out
 
 def test_tb_syntaxerror():
     """test %tb after a SyntaxError"""
     ip = get_ipython()
     ip.run_cell("for")
     
     # trap and validate stdout
     save_stdout = sys.stdout
     try:
         sys.stdout = StringIO()
         ip.run_cell("%tb")
         out = sys.stdout.getvalue()
     finally:
         sys.stdout = save_stdout
     # trim output, and only check the last line
     last_line = out.rstrip().splitlines()[-1].strip()
     nt.assert_equal(last_line, "SyntaxError: invalid syntax")
 
 
 def test_time():
     ip = get_ipython()
     
     with tt.AssertPrints("Wall time: "):
         ip.run_cell("%time None")
     
     ip.run_cell("def f(kmjy):\n"
                 "    %time print (2*kmjy)")
     
     with tt.AssertPrints("Wall time: "):
         with tt.AssertPrints("hihi", suppress=False):
             ip.run_cell("f('hi')")
 
 def test_time_last_not_expression():
     ip.run_cell("%%time\n"
                 "var_1 = 1\n"
                 "var_2 = 2\n")
     assert ip.user_ns['var_1'] == 1
     del ip.user_ns['var_1']
     assert ip.user_ns['var_2'] == 2
     del ip.user_ns['var_2']
     
 
 @dec.skip_win32
 def test_time2():
     ip = get_ipython()
     
     with tt.AssertPrints("CPU times: user "):
         ip.run_cell("%time None")
 
 def test_time3():
     """Erroneous magic function calls, issue gh-3334"""
     ip = get_ipython()
     ip.user_ns.pop('run', None)
     
     with tt.AssertNotPrints("not found", channel='stderr'):
         ip.run_cell("%%time\n"
                     "run = 0\n"
                     "run += 1")
 
 def test_multiline_time():
     """Make sure last statement from time return a value."""
     ip = get_ipython()
     ip.user_ns.pop('run', None)
 
     ip.run_cell(dedent("""\
         %%time
         a = "ho"
         b = "hey"
         a+b
         """))
     nt.assert_equal(ip.user_ns_hidden['_'], 'hohey')
 
 def test_time_local_ns():
     """
     Test that local_ns is actually global_ns when running a cell magic
     """
     ip = get_ipython()
     ip.run_cell("%%time\n"
                 "myvar = 1")
     nt.assert_equal(ip.user_ns['myvar'], 1)
     del ip.user_ns['myvar']
 
 def test_doctest_mode():
     "Toggle doctest_mode twice, it should be a no-op and run without error"
     _ip.magic('doctest_mode')
     _ip.magic('doctest_mode')
 
 
 def test_parse_options():
     """Tests for basic options parsing in magics."""
     # These are only the most minimal of tests, more should be added later.  At
     # the very least we check that basic text/unicode calls work OK.
     m = DummyMagics(_ip)
     nt.assert_equal(m.parse_options('foo', '')[1], 'foo')
     nt.assert_equal(m.parse_options(u'foo', '')[1], u'foo')
 
 
 def test_dirops():
     """Test various directory handling operations."""
     # curpath = lambda :os.path.splitdrive(os.getcwd())[1].replace('\\','/')
     curpath = os.getcwd
     startdir = os.getcwd()
     ipdir = os.path.realpath(_ip.ipython_dir)
     try:
         _ip.magic('cd "%s"' % ipdir)
         nt.assert_equal(curpath(), ipdir)
         _ip.magic('cd -')
         nt.assert_equal(curpath(), startdir)
         _ip.magic('pushd "%s"' % ipdir)
         nt.assert_equal(curpath(), ipdir)
         _ip.magic('popd')
         nt.assert_equal(curpath(), startdir)
     finally:
         os.chdir(startdir)
 
 
 def test_cd_force_quiet():
     """Test OSMagics.cd_force_quiet option"""
     _ip.config.OSMagics.cd_force_quiet = True
     osmagics = osm.OSMagics(shell=_ip)
 
     startdir = os.getcwd()
     ipdir = os.path.realpath(_ip.ipython_dir)
 
     try:
         with tt.AssertNotPrints(ipdir):
             osmagics.cd('"%s"' % ipdir)
         with tt.AssertNotPrints(startdir):
             osmagics.cd('-')
     finally:
         os.chdir(startdir)
 
 
 def test_xmode():
     # Calling xmode three times should be a no-op
     xmode = _ip.InteractiveTB.mode
     for i in range(4):
         _ip.magic("xmode")
     nt.assert_equal(_ip.InteractiveTB.mode, xmode)
     
 def test_reset_hard():
     monitor = []
     class A(object):
         def __del__(self):
             monitor.append(1)
         def __repr__(self):
             return "<A instance>"
             
     _ip.user_ns["a"] = A()
     _ip.run_cell("a")
     
     nt.assert_equal(monitor, [])
     _ip.magic("reset -f")
     nt.assert_equal(monitor, [1])
     
 class TestXdel(tt.TempFileMixin):
     def test_xdel(self):
         """Test that references from %run are cleared by xdel."""
         src = ("class A(object):\n"
                "    monitor = []\n"
                "    def __del__(self):\n"
                "        self.monitor.append(1)\n"
                "a = A()\n")
         self.mktmp(src)
         # %run creates some hidden references...
         _ip.magic("run %s" % self.fname)
         # ... as does the displayhook.
         _ip.run_cell("a")
         
         monitor = _ip.user_ns["A"].monitor
         nt.assert_equal(monitor, [])
         
         _ip.magic("xdel a")
         
         # Check that a's __del__ method has been called.
         nt.assert_equal(monitor, [1])
 
 def doctest_who():
     """doctest for %who
     
     In [1]: %reset -f
     
     In [2]: alpha = 123
     
     In [3]: beta = 'beta'
     
     In [4]: %who int
     alpha
     
     In [5]: %who str
     beta
     
     In [6]: %whos
     Variable   Type    Data/Info
     ----------------------------
     alpha      int     123
     beta       str     beta
     
     In [7]: %who_ls
     Out[7]: ['alpha', 'beta']
     """
 
 def test_whos():
     """Check that whos is protected against objects where repr() fails."""
     class A(object):
         def __repr__(self):
             raise Exception()
     _ip.user_ns['a'] = A()
     _ip.magic("whos")
 
 def doctest_precision():
     """doctest for %precision
     
     In [1]: f = get_ipython().display_formatter.formatters['text/plain']
     
     In [2]: %precision 5
     Out[2]: '%.5f'
     
     In [3]: f.float_format
     Out[3]: '%.5f'
     
     In [4]: %precision %e
     Out[4]: '%e'
     
     In [5]: f(3.1415927)
     Out[5]: '3.141593e+00'
     """
 
 def test_debug_magic():
     """Test debugging a small code with %debug
     
     In [1]: with PdbTestInput(['c']):
        ...:     %debug print("a b") #doctest: +ELLIPSIS
        ...:
     ...
     ipdb> c
     a b
     In [2]:
     """
 
 def test_psearch():
     with tt.AssertPrints("dict.fromkeys"):
         _ip.run_cell("dict.fr*?")
     with tt.AssertPrints("π.is_integer"):
         _ip.run_cell("π = 3.14;\nπ.is_integ*?")
 
 def test_timeit_shlex():
     """test shlex issues with timeit (#1109)"""
     _ip.ex("def f(*a,**kw): pass")
     _ip.magic('timeit -n1 "this is a bug".count(" ")')
     _ip.magic('timeit -r1 -n1 f(" ", 1)')
     _ip.magic('timeit -r1 -n1 f(" ", 1, " ", 2, " ")')
     _ip.magic('timeit -r1 -n1 ("a " + "b")')
     _ip.magic('timeit -r1 -n1 f("a " + "b")')
     _ip.magic('timeit -r1 -n1 f("a " + "b ")')
 
 
 def test_timeit_special_syntax():
     "Test %%timeit with IPython special syntax"
     @register_line_magic
     def lmagic(line):
         ip = get_ipython()
         ip.user_ns['lmagic_out'] = line
 
     # line mode test
     _ip.run_line_magic('timeit', '-n1 -r1 %lmagic my line')
     nt.assert_equal(_ip.user_ns['lmagic_out'], 'my line')
     # cell mode test
     _ip.run_cell_magic('timeit', '-n1 -r1', '%lmagic my line2')
     nt.assert_equal(_ip.user_ns['lmagic_out'], 'my line2')
 
 def test_timeit_return():
     """
     test whether timeit -o return object
     """
 
     res = _ip.run_line_magic('timeit','-n10 -r10 -o 1')
     assert(res is not None)
 
 def test_timeit_quiet():
     """
     test quiet option of timeit magic
     """
     with tt.AssertNotPrints("loops"):
         _ip.run_cell("%timeit -n1 -r1 -q 1")
 
 def test_timeit_return_quiet():
     with tt.AssertNotPrints("loops"):
         res = _ip.run_line_magic('timeit', '-n1 -r1 -q -o 1')
     assert (res is not None)
 
 def test_timeit_invalid_return():
     with nt.assert_raises_regex(SyntaxError, "outside function"):
         _ip.run_line_magic('timeit', 'return')
 
 @dec.skipif(execution.profile is None)
 def test_prun_special_syntax():
     "Test %%prun with IPython special syntax"
     @register_line_magic
     def lmagic(line):
         ip = get_ipython()
         ip.user_ns['lmagic_out'] = line
 
     # line mode test
     _ip.run_line_magic('prun', '-q %lmagic my line')
     nt.assert_equal(_ip.user_ns['lmagic_out'], 'my line')
     # cell mode test
     _ip.run_cell_magic('prun', '-q', '%lmagic my line2')
     nt.assert_equal(_ip.user_ns['lmagic_out'], 'my line2')
 
 @dec.skipif(execution.profile is None)
 def test_prun_quotes():
     "Test that prun does not clobber string escapes (GH #1302)"
     _ip.magic(r"prun -q x = '\t'")
     nt.assert_equal(_ip.user_ns['x'], '\t')
 
 def test_extension():
     # Debugging information for failures of this test
     print('sys.path:')
     for p in sys.path:
         print(' ', p)
     print('CWD', os.getcwd())
 
     nt.assert_raises(ImportError, _ip.magic, "load_ext daft_extension")
     daft_path = os.path.join(os.path.dirname(__file__), "daft_extension")
     sys.path.insert(0, daft_path)
     try:
         _ip.user_ns.pop('arq', None)
         invalidate_caches()   # Clear import caches
         _ip.magic("load_ext daft_extension")
         nt.assert_equal(_ip.user_ns['arq'], 185)
         _ip.magic("unload_ext daft_extension")
         assert 'arq' not in _ip.user_ns
     finally:
         sys.path.remove(daft_path)
 
 
 def test_notebook_export_json():
     _ip = get_ipython()
     _ip.history_manager.reset()   # Clear any existing history.
     cmds = [u"a=1", u"def b():\n  return a**2", u"print('noël, été', b())"]
     for i, cmd in enumerate(cmds, start=1):
         _ip.history_manager.store_inputs(i, cmd)
     with TemporaryDirectory() as td:
         outfile = os.path.join(td, "nb.ipynb")
         _ip.magic("notebook -e %s" % outfile)
 
 
 class TestEnv(TestCase):
 
     def test_env(self):
         env = _ip.magic("env")
         self.assertTrue(isinstance(env, dict))
 
     def test_env_secret(self):
         env = _ip.magic("env")
         hidden = "<hidden>"
         with mock.patch.dict(
             os.environ,
             {
                 "API_KEY": "abc123",
                 "SECRET_THING": "ssshhh",
                 "JUPYTER_TOKEN": "",
                 "VAR": "abc"
             }
         ):
             env = _ip.magic("env")
         assert env["API_KEY"] == hidden
         assert env["SECRET_THING"] == hidden
         assert env["JUPYTER_TOKEN"] == hidden
         assert env["VAR"] == "abc"
 
     def test_env_get_set_simple(self):
         env = _ip.magic("env var val1")
         self.assertEqual(env, None)
         self.assertEqual(os.environ['var'], 'val1')
         self.assertEqual(_ip.magic("env var"), 'val1')
         env = _ip.magic("env var=val2")
         self.assertEqual(env, None)
         self.assertEqual(os.environ['var'], 'val2')
 
     def test_env_get_set_complex(self):
         env = _ip.magic("env var 'val1 '' 'val2")
         self.assertEqual(env, None)
         self.assertEqual(os.environ['var'], "'val1 '' 'val2")
         self.assertEqual(_ip.magic("env var"), "'val1 '' 'val2")
         env = _ip.magic('env var=val2 val3="val4')
         self.assertEqual(env, None)
         self.assertEqual(os.environ['var'], 'val2 val3="val4')
 
     def test_env_set_bad_input(self):
         self.assertRaises(UsageError, lambda: _ip.magic("set_env var"))
 
     def test_env_set_whitespace(self):
         self.assertRaises(UsageError, lambda: _ip.magic("env var A=B"))
 
 
 class CellMagicTestCase(TestCase):
 
     def check_ident(self, magic):
         # Manually called, we get the result
         out = _ip.run_cell_magic(magic, 'a', 'b')
         nt.assert_equal(out, ('a','b'))
         # Via run_cell, it goes into the user's namespace via displayhook
         _ip.run_cell('%%' + magic +' c\nd\n')
         nt.assert_equal(_ip.user_ns['_'], ('c','d\n'))
 
     def test_cell_magic_func_deco(self):
         "Cell magic using simple decorator"
         @register_cell_magic
         def cellm(line, cell):
             return line, cell
 
         self.check_ident('cellm')
 
     def test_cell_magic_reg(self):
         "Cell magic manually registered"
         def cellm(line, cell):
             return line, cell
 
         _ip.register_magic_function(cellm, 'cell', 'cellm2')
         self.check_ident('cellm2')
 
     def test_cell_magic_class(self):
         "Cell magics declared via a class"
         @magics_class
         class MyMagics(Magics):
 
             @cell_magic
             def cellm3(self, line, cell):
                 return line, cell
 
         _ip.register_magics(MyMagics)
         self.check_ident('cellm3')
 
     def test_cell_magic_class2(self):
         "Cell magics declared via a class, #2"
         @magics_class
         class MyMagics2(Magics):
 
             @cell_magic('cellm4')
             def cellm33(self, line, cell):
                 return line, cell
             
         _ip.register_magics(MyMagics2)
         self.check_ident('cellm4')
         # Check that nothing is registered as 'cellm33'
         c33 = _ip.find_cell_magic('cellm33')
         nt.assert_equal(c33, None)
 
 def test_file():
     """Basic %%writefile"""
     ip = get_ipython()
     with TemporaryDirectory() as td:
         fname = os.path.join(td, 'file1')
         ip.run_cell_magic("writefile", fname, u'\n'.join([
             'line1',
             'line2',
         ]))
         with open(fname) as f:
             s = f.read()
         nt.assert_in('line1\n', s)
         nt.assert_in('line2', s)
 
 @dec.skip_win32
 def test_file_single_quote():
     """Basic %%writefile with embedded single quotes"""
     ip = get_ipython()
     with TemporaryDirectory() as td:
         fname = os.path.join(td, '\'file1\'')
         ip.run_cell_magic("writefile", fname, u'\n'.join([
             'line1',
             'line2',
         ]))
         with open(fname) as f:
             s = f.read()
         nt.assert_in('line1\n', s)
         nt.assert_in('line2', s)
 
 @dec.skip_win32
 def test_file_double_quote():
     """Basic %%writefile with embedded double quotes"""
     ip = get_ipython()
     with TemporaryDirectory() as td:
         fname = os.path.join(td, '"file1"')
         ip.run_cell_magic("writefile", fname, u'\n'.join([
             'line1',
             'line2',
         ]))
         with open(fname) as f:
             s = f.read()
         nt.assert_in('line1\n', s)
         nt.assert_in('line2', s)
 
 def test_file_var_expand():
     """%%writefile $filename"""
     ip = get_ipython()
     with TemporaryDirectory() as td:
         fname = os.path.join(td, 'file1')
         ip.user_ns['filename'] = fname
         ip.run_cell_magic("writefile", '$filename', u'\n'.join([
             'line1',
             'line2',
         ]))
         with open(fname) as f:
             s = f.read()
         nt.assert_in('line1\n', s)
         nt.assert_in('line2', s)
 
 def test_file_unicode():
     """%%writefile with unicode cell"""
     ip = get_ipython()
     with TemporaryDirectory() as td:
         fname = os.path.join(td, 'file1')
         ip.run_cell_magic("writefile", fname, u'\n'.join([
             u'liné1',
             u'liné2',
         ]))
         with io.open(fname, encoding='utf-8') as f:
             s = f.read()
         nt.assert_in(u'liné1\n', s)
         nt.assert_in(u'liné2', s)
 
 def test_file_amend():
     """%%writefile -a amends files"""
     ip = get_ipython()
     with TemporaryDirectory() as td:
         fname = os.path.join(td, 'file2')
         ip.run_cell_magic("writefile", fname, u'\n'.join([
             'line1',
             'line2',
         ]))
         ip.run_cell_magic("writefile", "-a %s" % fname, u'\n'.join([
             'line3',
             'line4',
         ]))
         with open(fname) as f:
             s = f.read()
         nt.assert_in('line1\n', s)
         nt.assert_in('line3\n', s)
 
 def test_file_spaces():
     """%%file with spaces in filename"""
     ip = get_ipython()
     with TemporaryWorkingDirectory() as td:
         fname = "file name"
         ip.run_cell_magic("file", '"%s"'%fname, u'\n'.join([
             'line1',
             'line2',
         ]))
         with open(fname) as f:
             s = f.read()
         nt.assert_in('line1\n', s)
         nt.assert_in('line2', s)
     
 def test_script_config():
     ip = get_ipython()
     ip.config.ScriptMagics.script_magics = ['whoda']
     sm = script.ScriptMagics(shell=ip)
     nt.assert_in('whoda', sm.magics['cell'])
 
 @dec.skip_win32
 def test_script_out():
     ip = get_ipython()
     ip.run_cell_magic("script", "--out output sh", "echo 'hi'")
     nt.assert_equal(ip.user_ns['output'], 'hi\n')
 
 @dec.skip_win32
 def test_script_err():
     ip = get_ipython()
     ip.run_cell_magic("script", "--err error sh", "echo 'hello' >&2")
     nt.assert_equal(ip.user_ns['error'], 'hello\n')
 
 @dec.skip_win32
 def test_script_out_err():
     ip = get_ipython()
     ip.run_cell_magic("script", "--out output --err error sh", "echo 'hi'\necho 'hello' >&2")
     nt.assert_equal(ip.user_ns['output'], 'hi\n')
     nt.assert_equal(ip.user_ns['error'], 'hello\n')
 
 @dec.skip_win32
 def test_script_bg_out():
     ip = get_ipython()
     ip.run_cell_magic("script", "--bg --out output sh", "echo 'hi'")
 
     nt.assert_equal(ip.user_ns['output'].read(), b'hi\n')
     ip.user_ns['output'].close()
 
 @dec.skip_win32
 def test_script_bg_err():
     ip = get_ipython()
     ip.run_cell_magic("script", "--bg --err error sh", "echo 'hello' >&2")
     nt.assert_equal(ip.user_ns['error'].read(), b'hello\n')
     ip.user_ns['error'].close()
 
 @dec.skip_win32
 def test_script_bg_out_err():
     ip = get_ipython()
     ip.run_cell_magic("script", "--bg --out output --err error sh", "echo 'hi'\necho 'hello' >&2")
     nt.assert_equal(ip.user_ns['output'].read(), b'hi\n')
     nt.assert_equal(ip.user_ns['error'].read(), b'hello\n')
     ip.user_ns['output'].close()
     ip.user_ns['error'].close()
 
 def test_script_defaults():
     ip = get_ipython()
     for cmd in ['sh', 'bash', 'perl', 'ruby']:
         try:
             find_cmd(cmd)
         except Exception:
             pass
         else:
             nt.assert_in(cmd, ip.magics_manager.magics['cell'])
 
 
 @magics_class
 class FooFoo(Magics):
     """class with both %foo and %%foo magics"""
     @line_magic('foo')
     def line_foo(self, line):
         "I am line foo"
         pass
 
     @cell_magic("foo")
     def cell_foo(self, line, cell):
         "I am cell foo, not line foo"
         pass
 
 def test_line_cell_info():
     """%%foo and %foo magics are distinguishable to inspect"""
     ip = get_ipython()
     ip.magics_manager.register(FooFoo)
     oinfo = ip.object_inspect('foo')
     nt.assert_true(oinfo['found'])
     nt.assert_true(oinfo['ismagic'])
     
     oinfo = ip.object_inspect('%%foo')
     nt.assert_true(oinfo['found'])
     nt.assert_true(oinfo['ismagic'])
     nt.assert_equal(oinfo['docstring'], FooFoo.cell_foo.__doc__)
 
     oinfo = ip.object_inspect('%foo')
     nt.assert_true(oinfo['found'])
     nt.assert_true(oinfo['ismagic'])
     nt.assert_equal(oinfo['docstring'], FooFoo.line_foo.__doc__)
 
 def test_multiple_magics():
     ip = get_ipython()
     foo1 = FooFoo(ip)
     foo2 = FooFoo(ip)
     mm = ip.magics_manager
     mm.register(foo1)
     nt.assert_true(mm.magics['line']['foo'].__self__ is foo1)
     mm.register(foo2)
     nt.assert_true(mm.magics['line']['foo'].__self__ is foo2)
 
 def test_alias_magic():
     """Test %alias_magic."""
     ip = get_ipython()
     mm = ip.magics_manager
 
     # Basic operation: both cell and line magics are created, if possible.
     ip.run_line_magic('alias_magic', 'timeit_alias timeit')
     nt.assert_in('timeit_alias', mm.magics['line'])
     nt.assert_in('timeit_alias', mm.magics['cell'])
 
     # --cell is specified, line magic not created.
     ip.run_line_magic('alias_magic', '--cell timeit_cell_alias timeit')
     nt.assert_not_in('timeit_cell_alias', mm.magics['line'])
     nt.assert_in('timeit_cell_alias', mm.magics['cell'])
 
     # Test that line alias is created successfully.
     ip.run_line_magic('alias_magic', '--line env_alias env')
     nt.assert_equal(ip.run_line_magic('env', ''),
                     ip.run_line_magic('env_alias', ''))
 
     # Test that line alias with parameters passed in is created successfully.
     ip.run_line_magic('alias_magic', '--line history_alias history --params ' + shlex.quote('3'))
     nt.assert_in('history_alias', mm.magics['line'])
 
 
 def test_save():
     """Test %save."""
     ip = get_ipython()
     ip.history_manager.reset()   # Clear any existing history.
     cmds = [u"a=1", u"def b():\n  return a**2", u"print(a, b())"]
     for i, cmd in enumerate(cmds, start=1):
         ip.history_manager.store_inputs(i, cmd)
     with TemporaryDirectory() as tmpdir:
         file = os.path.join(tmpdir, "testsave.py")
         ip.run_line_magic("save", "%s 1-10" % file)
         with open(file) as f:
             content = f.read()
             nt.assert_equal(content.count(cmds[0]), 1)
             nt.assert_in('coding: utf-8', content)
         ip.run_line_magic("save", "-a %s 1-10" % file)
         with open(file) as f:
             content = f.read()
             nt.assert_equal(content.count(cmds[0]), 2)
             nt.assert_in('coding: utf-8', content)
 
 
 def test_store():
     """Test %store."""
     ip = get_ipython()
     ip.run_line_magic('load_ext', 'storemagic')
     
     # make sure the storage is empty
     ip.run_line_magic('store', '-z')
     ip.user_ns['var'] = 42
     ip.run_line_magic('store', 'var')
     ip.user_ns['var'] = 39
     ip.run_line_magic('store', '-r')
     nt.assert_equal(ip.user_ns['var'], 42)
 
     ip.run_line_magic('store', '-d var')
     ip.user_ns['var'] = 39
     ip.run_line_magic('store' , '-r')
     nt.assert_equal(ip.user_ns['var'], 39)
 
 
 def _run_edit_test(arg_s, exp_filename=None,
                         exp_lineno=-1,
                         exp_contents=None,
                         exp_is_temp=None):
     ip = get_ipython()
     M = code.CodeMagics(ip)
     last_call = ['','']
     opts,args = M.parse_options(arg_s,'prxn:')
     filename, lineno, is_temp = M._find_edit_target(ip, args, opts, last_call)
         
     if exp_filename is not None:
         nt.assert_equal(exp_filename, filename)
     if exp_contents is not None:
         with io.open(filename, 'r', encoding='utf-8') as f:
             contents = f.read()
         nt.assert_equal(exp_contents, contents)
     if exp_lineno != -1:
         nt.assert_equal(exp_lineno, lineno)
     if exp_is_temp is not None:
         nt.assert_equal(exp_is_temp, is_temp)
 
 
 def test_edit_interactive():
     """%edit on interactively defined objects"""
     ip = get_ipython()
     n = ip.execution_count
     ip.run_cell(u"def foo(): return 1", store_history=True)
     
     try:
         _run_edit_test("foo")
     except code.InteractivelyDefined as e:
         nt.assert_equal(e.index, n)
     else:
         raise AssertionError("Should have raised InteractivelyDefined")
 
 
 def test_edit_cell():
     """%edit [cell id]"""
     ip = get_ipython()
     
     ip.run_cell(u"def foo(): return 1", store_history=True)
     
     # test
     _run_edit_test("1", exp_contents=ip.user_ns['In'][1], exp_is_temp=True)
 
 def test_bookmark():
     ip = get_ipython()
     ip.run_line_magic('bookmark', 'bmname')
     with tt.AssertPrints('bmname'):
         ip.run_line_magic('bookmark', '-l')
     ip.run_line_magic('bookmark', '-d bmname')
 
 def test_ls_magic():
     ip = get_ipython()
     json_formatter = ip.display_formatter.formatters['application/json']
     json_formatter.enabled = True
     lsmagic = ip.magic('lsmagic')
     with warnings.catch_warnings(record=True) as w:
         j = json_formatter(lsmagic)
     nt.assert_equal(sorted(j), ['cell', 'line'])
     nt.assert_equal(w, []) # no warnings
 
 def test_strip_initial_indent():
     def sii(s):
         lines = s.splitlines()
         return '\n'.join(code.strip_initial_indent(lines))
 
     nt.assert_equal(sii("  a = 1\nb = 2"), "a = 1\nb = 2")
     nt.assert_equal(sii("  a\n    b\nc"), "a\n  b\nc")
     nt.assert_equal(sii("a\n  b"), "a\n  b")
 
 def test_logging_magic_quiet_from_arg():
     _ip.config.LoggingMagics.quiet = False
     lm = logging.LoggingMagics(shell=_ip)
     with TemporaryDirectory() as td:
         try:
             with tt.AssertNotPrints(re.compile("Activating.*")):
                 lm.logstart('-q {}'.format(
                         os.path.join(td, "quiet_from_arg.log")))
         finally:
             _ip.logger.logstop()
 
 def test_logging_magic_quiet_from_config():
     _ip.config.LoggingMagics.quiet = True
     lm = logging.LoggingMagics(shell=_ip)
     with TemporaryDirectory() as td:
         try:
             with tt.AssertNotPrints(re.compile("Activating.*")):
                 lm.logstart(os.path.join(td, "quiet_from_config.log"))
         finally:
             _ip.logger.logstop()
 
 
 def test_logging_magic_not_quiet():
     _ip.config.LoggingMagics.quiet = False
     lm = logging.LoggingMagics(shell=_ip)
     with TemporaryDirectory() as td:
         try:
             with tt.AssertPrints(re.compile("Activating.*")):
                 lm.logstart(os.path.join(td, "not_quiet.log"))
         finally:
             _ip.logger.logstop()
 
 
 def test_time_no_var_expand():
     _ip.user_ns['a'] = 5
     _ip.user_ns['b'] = []
     _ip.magic('time b.append("{a}")')
     assert _ip.user_ns['b'] == ['{a}']
 
 
 # this is slow, put at the end for local testing.
 def test_timeit_arguments():
     "Test valid timeit arguments, should not cause SyntaxError (GH #1269)"
     if sys.version_info < (3,7):
         _ip.magic("timeit -n1 -r1 ('#')")
     else:
         # 3.7 optimize no-op statement like above out, and complain there is
         # nothing in the for loop.
         _ip.magic("timeit -n1 -r1 a=('#')")
+
+
+MINIMAL_LAZY_MAGIC = """
+from IPython.core.magic import (
+    Magics,
+    magics_class,
+    line_magic,
+    cell_magic,
+)
+
+
+@magics_class
+class LazyMagics(Magics):
+    @line_magic
+    def lazy_line(self, line):
+        print("Lazy Line")
+
+    @cell_magic
+    def lazy_cell(self, line, cell):
+        print("Lazy Cell")
+
+
+def load_ipython_extension(ipython):
+    ipython.register_magics(LazyMagics)
+"""
+
+
+def test_lazy_magics():
+    with pytest.raises(UsageError):
+        ip.run_line_magic("lazy_line", "")
+
+    startdir = os.getcwd()
+
+    with TemporaryDirectory() as tmpdir:
+        with prepended_to_syspath(tmpdir):
+            ptempdir = Path(tmpdir)
+            tf = ptempdir / "lazy_magic_module.py"
+            tf.write_text(MINIMAL_LAZY_MAGIC)
+            ip.magics_manager.register_lazy("lazy_line", Path(tf.name).name[:-3])
+            with tt.AssertPrints("Lazy Line"):
+                ip.run_line_magic("lazy_line", "")

IPython/core/tests/test_magic_terminal.py

@@ -1,192 +1,193 b''
 """Tests for various magic functions specific to the terminal frontend.
 
 Needs to be run by nose (to make ipython session available).
 """
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import sys
 from io import StringIO
 from unittest import TestCase
 
 import nose.tools as nt
 
 from IPython.testing import tools as tt
-
 #-----------------------------------------------------------------------------
 # Test functions begin
 #-----------------------------------------------------------------------------
 
+
 def check_cpaste(code, should_fail=False):
     """Execute code via 'cpaste' and ensure it was executed, unless
     should_fail is set.
     """
     ip.user_ns['code_ran'] = False
 
     src = StringIO()
     if not hasattr(src, 'encoding'):
         # IPython expects stdin to have an encoding attribute
         src.encoding = None
     src.write(code)
     src.write('\n--\n')
     src.seek(0)
 
     stdin_save = sys.stdin
     sys.stdin = src
 
     try:
         context = tt.AssertPrints if should_fail else tt.AssertNotPrints
         with context("Traceback (most recent call last)"):
-                ip.magic('cpaste')
+            ip.run_line_magic("cpaste", "")
 
         if not should_fail:
             assert ip.user_ns['code_ran'], "%r failed" % code
     finally:
         sys.stdin = stdin_save
 
 def test_cpaste():
     """Test cpaste magic"""
 
     def runf():
         """Marker function: sets a flag when executed.
         """
         ip.user_ns['code_ran'] = True
         return 'runf' # return string so '+ runf()' doesn't result in success
 
     tests = {'pass': ["runf()",
                       "In [1]: runf()",
                       "In [1]: if 1:\n   ...:     runf()",
                       "> > > runf()",
                       ">>> runf()",
                       "   >>> runf()",
                       ],
 
              'fail': ["1 + runf()",
                       "++ runf()",
              ]}
 
     ip.user_ns['runf'] = runf
 
     for code in tests['pass']:
         check_cpaste(code)
 
     for code in tests['fail']:
         check_cpaste(code, should_fail=True)
 
 
+
 class PasteTestCase(TestCase):
     """Multiple tests for clipboard pasting"""
 
     def paste(self, txt, flags='-q'):
         """Paste input text, by default in quiet mode"""
-        ip.hooks.clipboard_get = lambda : txt
-        ip.magic('paste '+flags)
+        ip.hooks.clipboard_get = lambda: txt
+        ip.run_line_magic("paste", flags)
 
     def setUp(self):
         # Inject fake clipboard hook but save original so we can restore it later
         self.original_clip = ip.hooks.clipboard_get
 
     def tearDown(self): 
         # Restore original hook
         ip.hooks.clipboard_get = self.original_clip
        
     def test_paste(self):
         ip.user_ns.pop('x', None)
         self.paste('x = 1')
         nt.assert_equal(ip.user_ns['x'], 1)
         ip.user_ns.pop('x')
 
     def test_paste_pyprompt(self):
         ip.user_ns.pop('x', None)
         self.paste('>>> x=2')
         nt.assert_equal(ip.user_ns['x'], 2)
         ip.user_ns.pop('x')
 
     def test_paste_py_multi(self):
         self.paste("""
         >>> x = [1,2,3]
         >>> y = []
         >>> for i in x:
         ...     y.append(i**2)
         ... 
         """)
         nt.assert_equal(ip.user_ns['x'], [1,2,3])
         nt.assert_equal(ip.user_ns['y'], [1,4,9])
 
     def test_paste_py_multi_r(self):
         "Now, test that self.paste -r works"
         self.test_paste_py_multi()
         nt.assert_equal(ip.user_ns.pop('x'), [1,2,3])
         nt.assert_equal(ip.user_ns.pop('y'), [1,4,9])
         nt.assert_false('x' in ip.user_ns)
         ip.magic('paste -r')
         nt.assert_equal(ip.user_ns['x'], [1,2,3])
         nt.assert_equal(ip.user_ns['y'], [1,4,9])
 
     def test_paste_email(self):
         "Test pasting of email-quoted contents"
         self.paste("""\
         >> def foo(x):
         >>     return x + 1
         >> xx = foo(1.1)""")
         nt.assert_equal(ip.user_ns['xx'], 2.1)
 
     def test_paste_email2(self):
         "Email again; some programs add a space also at each quoting level"
         self.paste("""\
         > > def foo(x):
         > >     return x + 1
         > > yy = foo(2.1)     """)
         nt.assert_equal(ip.user_ns['yy'], 3.1)
 
     def test_paste_email_py(self):
         "Email quoting of interactive input"
         self.paste("""\
         >> >>> def f(x):
         >> ...   return x+1
         >> ... 
         >> >>> zz = f(2.5)      """)
         nt.assert_equal(ip.user_ns['zz'], 3.5)
 
     def test_paste_echo(self):
         "Also test self.paste echoing, by temporarily faking the writer"
         w = StringIO()
         writer = ip.write
         ip.write = w.write
         code = """
         a = 100
         b = 200"""
         try:
             self.paste(code,'')
             out = w.getvalue()
         finally:
             ip.write = writer
         nt.assert_equal(ip.user_ns['a'], 100)
         nt.assert_equal(ip.user_ns['b'], 200)
         assert out == code+"\n## -- End pasted text --\n"
 
     def test_paste_leading_commas(self):
         "Test multiline strings with leading commas"
         tm = ip.magics_manager.registry['TerminalMagics']
         s = '''\
 a = """
 ,1,2,3
 """'''
         ip.user_ns.pop('foo', None)
         tm.store_or_execute(s, 'foo')
         nt.assert_in('foo', ip.user_ns)
 
 
     def test_paste_trailing_question(self):
         "Test pasting sources with trailing question marks"
         tm = ip.magics_manager.registry['TerminalMagics']
         s = '''\
 def funcfoo():
    if True: #am i true?
        return 'fooresult'
 '''
         ip.user_ns.pop('funcfoo', None)
         self.paste(s)
         nt.assert_equal(ip.user_ns['funcfoo'](), 'fooresult')

IPython/terminal/ipapp.py

@@ -1,380 +1,382 b''
 #!/usr/bin/env python
 # encoding: utf-8
 """
 The :class:`~IPython.core.application.Application` object for the command
 line :command:`ipython` program.
 """
 
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 
 import logging
 import os
 import sys
 import warnings
 
 from traitlets.config.loader import Config
 from traitlets.config.application import boolean_flag, catch_config_error
 from IPython.core import release
 from IPython.core import usage
 from IPython.core.completer import IPCompleter
 from IPython.core.crashhandler import CrashHandler
 from IPython.core.formatters import PlainTextFormatter
 from IPython.core.history import HistoryManager
 from IPython.core.application import (
     ProfileDir, BaseIPythonApplication, base_flags, base_aliases
 )
+from IPython.core.magic import MagicsManager
 from IPython.core.magics import (
     ScriptMagics, LoggingMagics
 )
 from IPython.core.shellapp import (
     InteractiveShellApp, shell_flags, shell_aliases
 )
 from IPython.extensions.storemagic import StoreMagics
 from .interactiveshell import TerminalInteractiveShell
 from IPython.paths import get_ipython_dir
 from traitlets import (
     Bool, List, default, observe, Type
 )
 
 #-----------------------------------------------------------------------------
 # Globals, utilities and helpers
 #-----------------------------------------------------------------------------
 
 _examples = """
 ipython --matplotlib       # enable matplotlib integration
 ipython --matplotlib=qt    # enable matplotlib integration with qt4 backend
 
 ipython --log-level=DEBUG  # set logging to DEBUG
 ipython --profile=foo      # start with profile foo
 
 ipython profile create foo # create profile foo w/ default config files
 ipython help profile       # show the help for the profile subcmd
 
 ipython locate             # print the path to the IPython directory
 ipython locate profile foo # print the path to the directory for profile `foo`
 """
 
 #-----------------------------------------------------------------------------
 # Crash handler for this application
 #-----------------------------------------------------------------------------
 
 class IPAppCrashHandler(CrashHandler):
     """sys.excepthook for IPython itself, leaves a detailed report on disk."""
 
     def __init__(self, app):
         contact_name = release.author
         contact_email = release.author_email
         bug_tracker = 'https://github.com/ipython/ipython/issues'
         super(IPAppCrashHandler,self).__init__(
             app, contact_name, contact_email, bug_tracker
         )
 
     def make_report(self,traceback):
         """Return a string containing a crash report."""
 
         sec_sep = self.section_sep
         # Start with parent report
         report = [super(IPAppCrashHandler, self).make_report(traceback)]
         # Add interactive-specific info we may have
         rpt_add = report.append
         try:
             rpt_add(sec_sep+"History of session input:")
             for line in self.app.shell.user_ns['_ih']:
                 rpt_add(line)
             rpt_add('\n*** Last line of input (may not be in above history):\n')
             rpt_add(self.app.shell._last_input_line+'\n')
         except:
             pass
 
         return ''.join(report)
 
 #-----------------------------------------------------------------------------
 # Aliases and Flags
 #-----------------------------------------------------------------------------
 flags = dict(base_flags)
 flags.update(shell_flags)
 frontend_flags = {}
 addflag = lambda *args: frontend_flags.update(boolean_flag(*args))
 addflag('autoedit-syntax', 'TerminalInteractiveShell.autoedit_syntax',
         'Turn on auto editing of files with syntax errors.',
         'Turn off auto editing of files with syntax errors.'
 )
 addflag('simple-prompt', 'TerminalInteractiveShell.simple_prompt',
         "Force simple minimal prompt using `raw_input`",
         "Use a rich interactive prompt with prompt_toolkit",
 )
 
 addflag('banner', 'TerminalIPythonApp.display_banner',
         "Display a banner upon starting IPython.",
         "Don't display a banner upon starting IPython."
 )
 addflag('confirm-exit', 'TerminalInteractiveShell.confirm_exit',
     """Set to confirm when you try to exit IPython with an EOF (Control-D
     in Unix, Control-Z/Enter in Windows). By typing 'exit' or 'quit',
     you can force a direct exit without any confirmation.""",
     "Don't prompt the user when exiting."
 )
 addflag('term-title', 'TerminalInteractiveShell.term_title',
     "Enable auto setting the terminal title.",
     "Disable auto setting the terminal title."
 )
 classic_config = Config()
 classic_config.InteractiveShell.cache_size = 0
 classic_config.PlainTextFormatter.pprint = False
 classic_config.TerminalInteractiveShell.prompts_class='IPython.terminal.prompts.ClassicPrompts'
 classic_config.InteractiveShell.separate_in = ''
 classic_config.InteractiveShell.separate_out = ''
 classic_config.InteractiveShell.separate_out2 = ''
 classic_config.InteractiveShell.colors = 'NoColor'
 classic_config.InteractiveShell.xmode = 'Plain'
 
 frontend_flags['classic']=(
     classic_config,
     "Gives IPython a similar feel to the classic Python prompt."
 )
 # # log doesn't make so much sense this way anymore
 # paa('--log','-l',
 #     action='store_true', dest='InteractiveShell.logstart',
 #     help="Start logging to the default log file (./ipython_log.py).")
 #
 # # quick is harder to implement
 frontend_flags['quick']=(
     {'TerminalIPythonApp' : {'quick' : True}},
     "Enable quick startup with no config files."
 )
 
 frontend_flags['i'] = (
     {'TerminalIPythonApp' : {'force_interact' : True}},
     """If running code from the command line, become interactive afterwards.
     It is often useful to follow this with `--` to treat remaining flags as
     script arguments.
     """
 )
 flags.update(frontend_flags)
 
 aliases = dict(base_aliases)
 aliases.update(shell_aliases)
 
 #-----------------------------------------------------------------------------
 # Main classes and functions
 #-----------------------------------------------------------------------------
 
 
 class LocateIPythonApp(BaseIPythonApplication):
     description = """print the path to the IPython dir"""
     subcommands = dict(
         profile=('IPython.core.profileapp.ProfileLocate',
             "print the path to an IPython profile directory",
         ),
     )
     def start(self):
         if self.subapp is not None:
             return self.subapp.start()
         else:
             print(self.ipython_dir)
 
 
 class TerminalIPythonApp(BaseIPythonApplication, InteractiveShellApp):
     name = u'ipython'
     description = usage.cl_usage
     crash_handler_class = IPAppCrashHandler
     examples = _examples
 
     flags = flags
     aliases = aliases
     classes = List()
 
     interactive_shell_class = Type(
         klass=object,   # use default_value otherwise which only allow subclasses.
         default_value=TerminalInteractiveShell,
         help="Class to use to instantiate the TerminalInteractiveShell object. Useful for custom Frontends"
     ).tag(config=True)
 
     @default('classes')
     def _classes_default(self):
         """This has to be in a method, for TerminalIPythonApp to be available."""
         return [
             InteractiveShellApp, # ShellApp comes before TerminalApp, because
             self.__class__,      # it will also affect subclasses (e.g. QtConsole)
             TerminalInteractiveShell,
             HistoryManager,
+            MagicsManager,
             ProfileDir,
             PlainTextFormatter,
             IPCompleter,
             ScriptMagics,
             LoggingMagics,
             StoreMagics,
         ]
 
     deprecated_subcommands = dict(
         qtconsole=('qtconsole.qtconsoleapp.JupyterQtConsoleApp',
             """DEPRECATED, Will be removed in IPython 6.0 : Launch the Jupyter Qt Console."""
         ),
         notebook=('notebook.notebookapp.NotebookApp',
             """DEPRECATED, Will be removed in IPython 6.0 : Launch the Jupyter HTML Notebook Server."""
         ),
         console=('jupyter_console.app.ZMQTerminalIPythonApp',
             """DEPRECATED, Will be removed in IPython 6.0 : Launch the Jupyter terminal-based Console."""
         ),
         nbconvert=('nbconvert.nbconvertapp.NbConvertApp',
             "DEPRECATED, Will be removed in IPython 6.0 : Convert notebooks to/from other formats."
         ),
         trust=('nbformat.sign.TrustNotebookApp',
             "DEPRECATED, Will be removed in IPython 6.0 : Sign notebooks to trust their potentially unsafe contents at load."
         ),
         kernelspec=('jupyter_client.kernelspecapp.KernelSpecApp',
             "DEPRECATED, Will be removed in IPython 6.0 : Manage Jupyter kernel specifications."
         ),
     )
     subcommands = dict(
         profile = ("IPython.core.profileapp.ProfileApp",
             "Create and manage IPython profiles."
         ),
         kernel = ("ipykernel.kernelapp.IPKernelApp",
             "Start a kernel without an attached frontend."
         ),
         locate=('IPython.terminal.ipapp.LocateIPythonApp',
             LocateIPythonApp.description
         ),
         history=('IPython.core.historyapp.HistoryApp',
             "Manage the IPython history database."
         ),
     )
     deprecated_subcommands['install-nbextension'] = (
         "notebook.nbextensions.InstallNBExtensionApp",
         "DEPRECATED, Will be removed in IPython 6.0 : Install Jupyter notebook extension files"
     )
     subcommands.update(deprecated_subcommands)
 
     # *do* autocreate requested profile, but don't create the config file.
     auto_create=Bool(True)
     # configurables
     quick = Bool(False,
         help="""Start IPython quickly by skipping the loading of config files."""
     ).tag(config=True)
     @observe('quick')
     def _quick_changed(self, change):
         if change['new']:
             self.load_config_file = lambda *a, **kw: None
 
     display_banner = Bool(True,
         help="Whether to display a banner upon starting IPython."
     ).tag(config=True)
 
     # if there is code of files to run from the cmd line, don't interact
     # unless the --i flag (App.force_interact) is true.
     force_interact = Bool(False,
         help="""If a command or file is given via the command-line,
         e.g. 'ipython foo.py', start an interactive shell after executing the
         file or command."""
     ).tag(config=True)
     @observe('force_interact')
     def _force_interact_changed(self, change):
         if change['new']:
             self.interact = True
 
     @observe('file_to_run', 'code_to_run', 'module_to_run')
     def _file_to_run_changed(self, change):
         new = change['new']
         if new:
             self.something_to_run = True
         if new and not self.force_interact:
                 self.interact = False
 
     # internal, not-configurable
     something_to_run=Bool(False)
 
     def parse_command_line(self, argv=None):
         """override to allow old '-pylab' flag with deprecation warning"""
 
         argv = sys.argv[1:] if argv is None else argv
 
         if '-pylab' in argv:
             # deprecated `-pylab` given,
             # warn and transform into current syntax
             argv = argv[:] # copy, don't clobber
             idx = argv.index('-pylab')
             warnings.warn("`-pylab` flag has been deprecated.\n"
             "    Use `--matplotlib <backend>` and import pylab manually.")
             argv[idx] = '--pylab'
 
         return super(TerminalIPythonApp, self).parse_command_line(argv)
     
     @catch_config_error
     def initialize(self, argv=None):
         """Do actions after construct, but before starting the app."""
         super(TerminalIPythonApp, self).initialize(argv)
         if self.subapp is not None:
             # don't bother initializing further, starting subapp
             return
         # print self.extra_args
         if self.extra_args and not self.something_to_run:
             self.file_to_run = self.extra_args[0]
         self.init_path()
         # create the shell
         self.init_shell()
         # and draw the banner
         self.init_banner()
         # Now a variety of things that happen after the banner is printed.
         self.init_gui_pylab()
         self.init_extensions()
         self.init_code()
 
     def init_shell(self):
         """initialize the InteractiveShell instance"""
         # Create an InteractiveShell instance.
         # shell.display_banner should always be False for the terminal
         # based app, because we call shell.show_banner() by hand below
         # so the banner shows *before* all extension loading stuff.
         self.shell = self.interactive_shell_class.instance(parent=self,
                         profile_dir=self.profile_dir,
                         ipython_dir=self.ipython_dir, user_ns=self.user_ns)
         self.shell.configurables.append(self)
 
     def init_banner(self):
         """optionally display the banner"""
         if self.display_banner and self.interact:
             self.shell.show_banner()
         # Make sure there is a space below the banner.
         if self.log_level <= logging.INFO: print()
 
     def _pylab_changed(self, name, old, new):
         """Replace --pylab='inline' with --pylab='auto'"""
         if new == 'inline':
             warnings.warn("'inline' not available as pylab backend, "
                       "using 'auto' instead.")
             self.pylab = 'auto'
 
     def start(self):
         if self.subapp is not None:
             return self.subapp.start()
         # perform any prexec steps:
         if self.interact:
             self.log.debug("Starting IPython's mainloop...")
             self.shell.mainloop()
         else:
             self.log.debug("IPython not interactive...")
             if not self.shell.last_execution_succeeded:
                 sys.exit(1)
 
 def load_default_config(ipython_dir=None):
     """Load the default config file from the default ipython_dir.
 
     This is useful for embedded shells.
     """
     if ipython_dir is None:
         ipython_dir = get_ipython_dir()
 
     profile_dir = os.path.join(ipython_dir, 'profile_default')
     app = TerminalIPythonApp()
     app.config_file_paths.append(profile_dir)
     app.load_config_file()
     return app.config
 
 launch_new_instance = TerminalIPythonApp.launch_instance
 
 
 if __name__ == '__main__':
     launch_new_instance()

docs/source/whatsnew/version7.rst

@@ -1,1747 +1,1765 b''
 ============
  7.x Series
 ============
 
 .. _version 7.31.1:
 
 IPython 7.31.1 (CVE-2022-21699)
 ===============================
 
 Fixed CVE-2022-21699, see IPython 8.0.1 release notes for informations.
 
 
+.. _version 7.32:
+
+IPython 7.32
+============
+
+
+The ability to configure magics to be lazily loaded has been added to IPython.
+See the ``ipython --help-all`` section on ``MagicsManager.lazy_magic``.
+One can now use::
+
+    c.MagicsManger.lazy_magics = {
+              "my_magic": "slow.to.import",
+              "my_other_magic": "also.slow",
+    }
+
+And on first use of ``%my_magic``, or corresponding cell magic, or other line magic,
+the corresponding ``load_ext`` will be called just before trying to invoke the magic.
+
 .. _version 7.31:
 
 IPython 7.31
 ============
 
 IPython 7.31 brings a couple of backports and fixes from the 8.0 branches,
 it is likely one of the last releases of the 7.x series, as 8.0 will probably be released
 between this release and what would have been 7.32.
 
 Please test 8.0 beta/rc releases in addition to this release.
 
 This Releases:
  - Backport some fixes for Python 3.10 (:ghpull:`13412`)
  - use full-alpha transparency on dvipng rendered LaTeX (:ghpull:`13372`)
 
 Many thanks to all the contributors to this release. You can find all individual
 contributions to this milestone `on github
 <https://github.com/ipython/ipython/milestone/95>`__.
 
 Thanks as well to the `D. E. Shaw group <https://deshaw.com/>`__ for sponsoring
 work on IPython and related libraries.
 
 
 .. _version 7.30:
 
 IPython 7.30
 ============
 
 IPython 7.30 fixes a couple of bugs introduce in previous releases (in
 particular with respect to path handling), and introduce a few features and
 improvements:
 
 Notably we will highlight :ghpull:`13267` "Document that ``%run`` can execute
 notebooks and ipy scripts.", which is the first commit of Fernando Pérez since
 mid 2016 (IPython 5.1). If you are new to IPython, Fernando created IPython in
 2001. The other most recent contribution of Fernando to IPython itself was
 May 2018, by reviewing and merging PRs. I want to note that Fernando is still
 active but mostly as a mentor and leader of the whole Jupyter organisation, but
 we're still happy to see him contribute code !
 
 :ghpull:`13290` "Use sphinxify (if available) in object_inspect_mime path"
 should allow richer Repr of docstrings when using jupyterlab inspector.
 
 :ghpull:`13311` make the debugger use ``ThreadPoolExecutor`` for debugger cmdloop.
 This should fix some issues/infinite loop, but let us know if you come across
 any regressions. In particular this fixes issues with `kmaork/madbg <https://github.com/kmaork/madbg>`_,
 a remote debugger for IPython.
 
 Note that this is likely the ante-penultimate release of IPython 7.x as a stable
 branch, as I hope to release IPython 8.0 as well as IPython 7.31 next
 month/early 2022.
 
 IPython 8.0 will drop support for Python 3.7, removed nose as a dependency, and
 7.x will only get critical bug fixes with 8.x becoming the new stable. This will
 not be possible without `NumFOCUS Small Development Grants
 <https://numfocus.org/programs/small-development-grants>`_ Which allowed us to
 hire `Nikita Kniazev <https://github.com/Kojoley>`_ who provide Python and C++
 help and contracting work.
 
 
 Many thanks to all the contributors to this release. You can find all individual
 contributions to this milestone `on github
 <https://github.com/ipython/ipython/milestone/94?closed=1>`__.
 
 Thanks as well to the `D. E. Shaw group <https://deshaw.com/>`__ for sponsoring
 work on IPython and related libraries.
 
 
 .. _version 7.29:
 
 IPython 7.29
 ============
 
 
 IPython 7.29 brings a couple of new functionalities to IPython and a number of bugfixes.
 It is one of the largest recent release, relatively speaking, with close to 15 Pull Requests.
 
 
  - fix an issue where base64 was returned instead of bytes when showing figures :ghpull:`13162`
  - fix compatibility with PyQt6, PySide 6 :ghpull:`13172`. This may be of
    interest if you are running on Apple Silicon as only qt6.2+ is natively
    compatible.
  - fix matplotlib qtagg eventloop :ghpull:`13179`
  - Multiple docs fixes, typos, ... etc.
  - Debugger will now exit by default on SigInt :ghpull:`13218`, this will be
    useful in notebook/lab if you forgot to exit the debugger. "Interrupt Kernel"
    will now exist the debugger.
 
 It give Pdb the ability to skip code in decorators. If functions contain a
 special value names ``__debuggerskip__ = True|False``, the function will not be
 stepped into, and Pdb will step into lower frames only if the value is set to
 ``False``. The exact behavior is still likely to have corner cases and will be
 refined in subsequent releases. Feedback welcome. See the debugger module
 documentation for more info. Thanks to the `D. E. Shaw
 group <https://deshaw.com/>`__ for funding this feature.
 
 The main branch of IPython is receiving a number of changes as we received a
 `NumFOCUS SDG <https://numfocus.org/programs/small-development-grants>`__
 ($4800), to help us finish replacing ``nose`` by ``pytest``, and make IPython
 future proof with an 8.0 release.
 
 
 Many thanks to all the contributors to this release. You can find all individual
 contributions to this milestone `on github
 <https://github.com/ipython/ipython/milestone/93>`__.
 
 Thanks as well to the `D. E. Shaw group <https://deshaw.com/>`__ for sponsoring
 work on IPython and related libraries.
 
 
 .. _version 7.28:
 
 IPython 7.28
 ============
 
 
 IPython 7.28 is again a minor release that mostly bring bugfixes, and couple of
 improvement. Many thanks to MrMino, who again did all the work this month, and
 made a number of documentation improvements.
 
 Here is a non-exhaustive list of changes,
 
 Fixes:
 
  - async with doesn't allow newlines :ghpull:`13090`
  - Dynamically changing to vi mode via %config magic) :ghpull:`13091`
 
 Virtualenv handling fixes:
 
  - init_virtualenv now uses Pathlib :ghpull:`12548`
  - Fix Improper path comparison of virtualenv directories :ghpull:`13140`
  - Fix virtual environment user warning for lower case pathes :ghpull:`13094`
  - Adapt to all sorts of drive names for cygwin :ghpull:`13153`
 
 New Features:
 
  - enable autoplay in embed YouTube player :ghpull:`13133`
 
  Documentation:
 
  - Fix formatting for the core.interactiveshell documentation :ghpull:`13118`
  - Fix broken ipyparallel's refs :ghpull:`13138`
  - Improve formatting of %time documentation :ghpull:`13125`
  - Reword the YouTubeVideo autoplay WN :ghpull:`13147`
 
 
 Highlighted features
 --------------------
 
 
 ``YouTubeVideo`` autoplay and the ability to add extra attributes to ``IFrame``
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 You can add any extra attributes to the ``<iframe>`` tag using the new
 ``extras`` argument in the ``IFrame`` class. For example::
 
     In [1]: from IPython.display import IFrame
 
     In [2]: IFrame(src="src", width=300, height=300, extras=['loading="eager"'])
 
 The above cells will result in the following HTML code being displayed in a
 notebook::
 
     <iframe
         width="300"
         height="300"
         src="src"
         frameborder="0"
         allowfullscreen
         loading="eager"
     ></iframe>
 
 Related to the above, the ``YouTubeVideo`` class now takes an
 ``allow_autoplay`` flag, which sets up the iframe of the embedded YouTube video
 such that it allows autoplay.
 
 .. note::
     Whether this works depends on the autoplay policy of the browser rendering
     the HTML allowing it. It also could get blocked by some browser extensions.
 
 Try it out!
 ::
 
     In [1]: from IPython.display import YouTubeVideo
 
     In [2]: YouTubeVideo("dQw4w9WgXcQ", allow_autoplay=True)
 
 
 
 Thanks
 ------
 
 Many thanks to all the contributors to this release. You can find all individual
 contributions to this milestone `on github
 <https://github.com/ipython/ipython/milestone/92>`__.
 
 Thanks as well to the `D. E. Shaw group <https://deshaw.com/>`__ for sponsoring
 work on IPython and related libraries.
 
 
 .. _version 7.27:
 
 IPython 7.27
 ============
 
 IPython 7.27 is a minor release that fixes a couple of issues and compatibility.
 
 - Add support for GTK4 :ghpull:`131011`
 - Add support for Qt6 :ghpull:`13085`
 - Fix an issue with pip magic on windows :ghpull:`13093`
 
 Thanks
 ------
 
 Many thanks to all the contributors to this release. You can find all individual
 contributions to this milestone `on github
 <https://github.com/ipython/ipython/milestone/91>`__.
 
 Thanks as well to the `D. E. Shaw group <https://deshaw.com/>`__ for sponsoring
 work on IPython and related libraries.
 
 .. _version 7.26:
 
 IPython 7.26
 ============
 
 IPython 7.26 is a minor release that fixes a couple of issues, updates in API
 and Copyright/Licenses issues around various part of the codebase.
 
 We'll highlight `this issue <https://github.com/ipython/ipython/issues/13039>`
 pointing out we were including and refereeing to code from Stack Overflow which
 was CC-BY-SA, hence incompatible with the BSD license of IPython. This lead us
 to a rewriting of the corresponding logic which in our case was done in a more
 efficient way (in our case we were searching string prefixes instead of full
 strings).
 
 You will notice also a number of documentation improvements and cleanup.
 
 Of particular interest are the following Pull-requests:
 
 
  - The IPython directive now uses Sphinx logging for warnings. :ghpull:`13030`.
  - Add expiry days option to pastebin magic and change http protocol to https.
    :ghpull:`13056`
  - Make Ipython.utils.timing work with jupyterlite :ghpull:`13050`.
 
 Pastebin magic expiry days option
 ---------------------------------
 
 The Pastebin magic now has ``-e`` option to determine 
 the number of days for paste expiration. For example
 the paste that created with ``%pastebin -e 20 1`` magic will
 be available for next 20 days.
 
 
 
 
 
 Thanks
 ------
 
 Many thanks to all the contributors to this release and in particular MrMino who
 is doing most of the work those days. You can find all individual contributions
 to this milestone `on github <https://github.com/ipython/ipython/milestone/90>`__.
 
 Thanks as well to the `D. E. Shaw group <https://deshaw.com/>`__ for sponsoring
 work on IPython and related libraries.
 
 
 .. _version 7.25:
 
 IPython 7.25
 ============
 
 IPython 7.25 is a minor release that contains a single bugfix, which is highly
 recommended for all users of ipdb, ipython debugger %debug magic and similar.
 
 Issuing commands like ``where`` from within the debugger would reset the
 local variables changes made by the user. It is interesting to look at the root
 cause of the issue as accessing an attribute (``frame.f_locals``) would trigger
 this side effects.
 
 Thanks in particular to the patience from the reporters at D.E. Shaw for their
 initial bug report that was due to a similar coding oversight in an extension,
 and who took time to debug and narrow down the problem.
 
 Thanks
 ------
 
 Many thanks to all the contributors to this release you can find all individual
 contributions to this milestone `on github <https://github.com/ipython/ipython/milestone/89>`__.
 
 Thanks as well to the `D. E. Shaw group <https://deshaw.com/>`__ for sponsoring
 work on IPython and related libraries.
 
 
 .. _version 7.24:
 
 IPython 7.24
 ============
 
 Third release of IPython for 2021, mostly containing bug fixes. A couple of not
 typical updates:
 
 Misc
 ----
 
 
  - Fix an issue where ``%recall`` would both succeeded and print an error message
    it failed. :ghpull:`12952`
  - Drop support for NumPy 1.16 – practically has no effect beyond indicating in
    package metadata that we do not support it. :ghpull:`12937`
 
 Debugger improvements
 ---------------------
 
 The debugger (and ``%debug`` magic) have been improved and can skip or hide frames
 originating from files that are not writable to the user, as these are less
 likely to be the source of errors, or be part of system files this can be a useful
 addition when debugging long errors.
 
 In addition to the global ``skip_hidden True|False`` command, the debugger has
 gained finer grained control of predicates as to whether to a frame should be
 considered hidden. So far 3 predicates are available :
 
   - ``tbhide``: frames containing the local variable ``__tracebackhide__`` set to
     True.
   - ``readonly``: frames originating from readonly files, set to False.
   - ``ipython_internal``: frames that are likely to be from IPython internal
     code, set to True.
 
 You can toggle individual predicates during a session with
 
 .. code-block::
 
    ipdb> skip_predicates readonly True
 
 Read-only files will now be considered hidden frames.
 
 
 You can call ``skip_predicates`` without arguments to see the states of current
 predicates:
 
 .. code-block::
 
     ipdb> skip_predicates
     current predicates:
         tbhide : True
         readonly : False
         ipython_internal : True
 
 If all predicates are set to ``False``,  ``skip_hidden`` will practically have
 no effect. We attempt to warn you when all predicates are False.
 
 Note that the ``readonly`` predicate may increase disk access as we check for
 file access permission for all frames on many command invocation, but is usually
 cached by operating systems. Let us know if you encounter any issues.
 
 As the IPython debugger does not use the traitlets infrastructure for
 configuration, by editing your ``.pdbrc`` files and appending commands you would
 like to be executed just before entering the interactive prompt. For example:
 
 
 .. code::
 
     # file : ~/.pdbrc
     skip_predicates readonly True
     skip_predicates tbhide False
 
 Will hide read only frames by default and show frames marked with
 ``__tracebackhide__``.
 
 
 
 
 Thanks
 ------
 
 Many thanks to all the contributors to this release you can find all individual
 contributions to this milestone `on github <https://github.com/ipython/ipython/milestone/87>`__.
 
 Thanks as well to the `D. E. Shaw group <https://deshaw.com/>`__ for sponsoring
 work on IPython and related libraries, in particular above mentioned
 improvements to the debugger.
 
 
 
 
 .. _version 7.23:
 
 IPython 7.23 and 7.23.1
 =======================
 
 
 Third release of IPython for 2021, mostly containing bug fixes. A couple of not
 typical updates:
 
  - We moved to GitHub actions away from Travis-CI, the transition may not be
    100% complete (not testing on nightly anymore), but as we ran out of
    Travis-Ci hours on the IPython organisation that was a necessary step.
    :ghpull:`12900`.
 
  - We have a new dependency: ``matplotlib-inline``, which try to extract
    matplotlib inline backend specific behavior. It is available on PyPI and
    conda-forge thus should not be a problem to upgrade to this version. If you
    are a package maintainer that might be an extra dependency to package first.
    :ghpull:`12817` (IPython 7.23.1 fix a typo that made this change fail)
 
 In the addition/new feature category, ``display()`` now have a ``clear=True``
 option to clear the display if any further outputs arrives, allowing users to
 avoid having to use ``clear_output()`` directly. :ghpull:`12823`.
 
 In bug fixes category, this release fix an issue when printing tracebacks
 containing Unicode characters :ghpull:`12758`.
 
 In code cleanup category :ghpull:`12932` remove usage of some deprecated
 functionality for compatibility with Python 3.10.
 
 
 
 Thanks
 ------
 
 Many thanks to all the contributors to this release you can find all individual
 contributions to this milestone `on github <https://github.com/ipython/ipython/milestone/86>`__.
 In particular MrMino for responding to almost all new issues, and triaging many
 of the old ones, as well as takluyver, minrk, willingc for reacting quikly when
 we ran out of CI Hours.
 
 Thanks as well to organisations, QuantStack (martinRenou and SylvainCorlay) for
 extracting matplotlib inline backend into its own package, and the `D. E. Shaw group
 <https://deshaw.com/>`__ for sponsoring work on IPython and related libraries.
 
 
 .. _version 7.22:
 
 IPython 7.22
 ============
 
 Second release of IPython for 2021, mostly containing bug fixes. Here is a quick
 rundown of the few changes.
 
 - Fix some ``sys.excepthook`` shenanigan when embedding with qt, recommended if
   you – for example – use `napari <https://napari.org>`__. :ghpull:`12842`.
 - Fix bug when using the new ipdb ``%context`` magic :ghpull:`12844`
 - Couples of deprecation cleanup :ghpull:`12868`
 - Update for new dpast.com api if you use the ``%pastbin`` magic. :ghpull:`12712`
 - Remove support for numpy before 1.16. :ghpull:`12836`
 
 
 Thanks
 ------
 
 We have a new team member that you should see more often on the IPython
 repository, Błażej Michalik (@MrMino) have been doing regular contributions to
 IPython, and spent time replying to many issues and guiding new users to the
 codebase; they now have triage permissions to the IPython repository and we'll
 work toward giving them more permission in the future.
 
 Many thanks to all the contributors to this release you can find all individual
 contributions to this milestone `on github <https://github.com/ipython/ipython/milestone/84>`__.
 
 Thanks as well to organisations, QuantStack for working on debugger
 compatibility for Xeus_python, and the `D. E. Shaw group
 <https://deshaw.com/>`__ for sponsoring work on IPython and related libraries.
 
 .. _version 721:
 
 IPython 7.21
 ============
 
 IPython 7.21 is the first release we have back on schedule of one release every
 month; it contains a number of minor fixes and improvements, notably, the new
 context command for ipdb
 
 
 New "context" command in ipdb
 -----------------------------
 
 It is now possible to change the number of lines shown in the backtrace
 information in ipdb using "context" command. :ghpull:`12826`
 
 (thanks @MrMino, there are other improvement from them on master).
 
 Other notable changes in IPython 7.21
 -------------------------------------
 
 - Fix some issues on new osx-arm64 :ghpull:`12804`, :ghpull:`12807`. 
 - Compatibility with Xeus-Python for debugger protocol, :ghpull:`12809`
 - Misc docs fixes for compatibility and uniformity with Numpydoc.
   :ghpull:`12824`
 
 
 Thanks
 ------
 
 Many thanks to all the contributors to this release you can find all individual
 contribution to this milestone `on github <https://github.com/ipython/ipython/milestone/83>`__.
 
 
 .. _version 720:
 
 IPython 7.20
 ============
 
 IPython 7.20 is the accumulation of 3 month of work on IPython, spacing between
 IPython release have been increased from the usual once a month for various
 reason.
 
    - Mainly as I'm too busy and the effectively sole maintainer, and
    - Second because not much changes happened before mid December.
 
 The main driver for this release was the new version of Jedi 0.18 breaking API;
 which was taken care of in the master branch early in 2020 but not in 7.x as I
 though that by now 8.0 would be out.
 
 The inclusion of a resolver in pip did not help and actually made things worse.
 If usually I would have simply pinned Jedi to ``<0.18``; this is not a solution
 anymore as now pip is free to install Jedi 0.18, and downgrade IPython.
 
 I'll do my best to keep the regular release, but as the 8.0-dev branch and 7.x
 are starting to diverge this is becoming difficult in particular with my limited
 time, so if you have any cycles to spare I'll appreciate your help to respond to
 issues and pushing 8.0 forward.
 
 Here are thus some of the changes for IPython 7.20.
 
   - Support for PyQt5 >= 5.11 :ghpull:`12715`
   - ``%reset`` remove imports more agressively :ghpull:`12718`
   - fix the ``%conda`` magic :ghpull:`12739`
   - compatibility with Jedi 0.18, and bump minimum Jedi version. :ghpull:`12793`
 
 
 .. _version 719:
 
 IPython 7.19
 ============
 
 IPython 7.19 accumulative two month of works, bug fixes and improvements, there
 was exceptionally no release last month.
 
   - Fix to restore the ability to specify more than one extension using command
     line flags when using traitlets 5.0 :ghpull:`12543`
   - Docs docs formatting that make the install commands work on zsh
     :ghpull:`12587`
   - Always display the last frame in tracebacks even if hidden with
     ``__tracebackhide__`` :ghpull:`12601`
   - Avoid an issue where a callback can be registered multiple times.
     :ghpull:`12625`
   - Avoid an issue in debugger mode where frames changes could be lost.
     :ghpull:`12627`
 
   - Never hide the frames that invoke a debugger, even if marked as hidden by
     ``__tracebackhide__`` :ghpull:`12631`
   - Fix calling the debugger in a recursive manner :ghpull:`12659`
 
 
 A number of code changes have landed on master and we are getting close to
 enough new features and codebase improvement that a 8.0 start to make sens.
 For downstream packages, please start working on migrating downstream testing
 away from iptest and using pytest, as nose will not work on Python 3.10 and we
 will likely start removing it as a dependency for testing.
 
 .. _version 718:
 
 IPython 7.18
 ============
 
 IPython 7.18 is a minor release that mostly contains bugfixes.
 
  - ``CRLF`` is now handled by magics my default; solving some issues due to copy
    pasting on windows. :ghpull:`12475`
 
  - Requiring pexpect ``>=4.3`` as we are Python 3.7+ only and earlier version of
    pexpect will be incompatible. :ghpull:`12510`
 
  - Minimum jedi version is now 0.16. :ghpull:`12488`
 
 
 
 .. _version 717:
 
 IPython 7.17
 ============
 
 IPython 7.17 brings a couple of new improvements to API and a couple of user
 facing changes to make the terminal experience more user friendly.
 
 :ghpull:`12407` introduces the ability to pass extra argument to the IPython
 debugger class; this is to help a new project from ``kmaork``
 (https://github.com/kmaork/madbg) to feature a fully remote debugger.
 
 :ghpull:`12410` finally remove support for 3.6, while the codebase is still
 technically compatible; IPython will not install on Python 3.6.
 
 lots of work on the debugger and hidden frames from ``@impact27`` in
 :ghpull:`12437`, :ghpull:`12445`, :ghpull:`12460` and in particular
 :ghpull:`12453` which make the debug magic more robust at handling spaces.
 
 Biggest API addition is code transformation which is done before code execution;
 IPython allows a number of hooks to catch non-valid Python syntax (magic, prompt
 stripping...etc). Transformers are usually called many time; typically:
 
  - When trying to figure out whether the code is complete and valid (should we
    insert a new line or execute ?)
  - During actual code execution pass before giving the code to Python's
    ``exec``.
 
 This lead to issues when transformer might have had side effects; or do external
 queries. Starting with IPython 7.17 you can expect your transformer to be called
 less time.
 
 Input transformers are now called only once in the execution path of
 `InteractiveShell`, allowing to register transformer that potentially have side
 effects (note that this is not recommended). Internal methods `should_run_async`, and
 `run_cell_async` now take a recommended optional `transformed_cell`, and
 `preprocessing_exc_tuple` parameters that will become mandatory at some point in
 the future; that is to say cells need to be explicitly transformed to be valid
 Python syntax ahead of trying to run them. :ghpull:`12440`;
 
 ``input_transformers`` can now also have an attribute ``has_side_effects`` set
 to `True`, when this attribute is present; this  will prevent the transformers
 from being ran when IPython is trying to guess whether the user input is
 complete. Note that this may means you will need to explicitly execute in some
 case where your transformations are now not ran; but will not affect users with
 no custom extensions.
 
 
 API Changes
 -----------
 
 Change of API and exposed objects automatically detected using `frappuccino
 <https://pypi.org/project/frappuccino/>`_
 
 
  The following items are new since 7.16.0::
 
      + IPython.core.interactiveshell.InteractiveShell.get_local_scope(self, stack_depth)
 
  The following signatures differ since 7.16.0::
 
      - IPython.core.interactiveshell.InteractiveShell.run_cell_async(self, raw_cell, store_history=False, silent=False, shell_futures=True)
      + IPython.core.interactiveshell.InteractiveShell.run_cell_async(self, raw_cell, store_history=False, silent=False, shell_futures=True, *, transformed_cell=None, preprocessing_exc_tuple=None)
 
      - IPython.core.interactiveshell.InteractiveShell.should_run_async(self, raw_cell)
      + IPython.core.interactiveshell.InteractiveShell.should_run_async(self, raw_cell, *, transformed_cell=None, preprocessing_exc_tuple=None)
 
      - IPython.terminal.debugger.TerminalPdb.pt_init(self)
      + IPython.terminal.debugger.TerminalPdb.pt_init(self, pt_session_options=None)
 
 This method was added::
 
      + IPython.core.interactiveshell.InteractiveShell.get_local_scope
 
 Which is now also present on subclasses::
 
      + IPython.terminal.embed.InteractiveShellEmbed.get_local_scope
      + IPython.terminal.interactiveshell.TerminalInteractiveShell.get_local_scope
 
 
 .. _version 716:
 
 IPython 7.16.1, 7.16.2
 ======================
 
 IPython 7.16.1 was release immediately after 7.16.0 to fix a conda packaging issue.
 The source is identical to 7.16.0 but the file permissions in the tar are different.
 
 IPython 7.16.2 pins jedi dependency to "<=0.17.2" which should prevent some
 issues for users still on python 3.6. This may not be sufficient as pip may
 still allow to downgrade IPython.
 
 Compatibility with Jedi > 0.17.2 was not added as this would have meant bumping
 the minimal version to >0.16.
 
 IPython 7.16
 ============
 
 
 The default traceback mode will now skip frames that are marked with
 ``__tracebackhide__ = True`` and show how many traceback frames have been
 skipped. This can be toggled by using :magic:`xmode` with the ``--show`` or
 ``--hide`` attribute. It will have no effect on non verbose traceback modes.
 
 The ipython debugger also now understands ``__tracebackhide__`` as well and will
 skip hidden frames when displaying. Movement up and down the stack will skip the
 hidden frames and will show how many frames were hidden. Internal IPython frames
 are also now hidden by default. The behavior can be changed with the
 ``skip_hidden`` while in the debugger, command and accepts "yes", "no", "true"
 and "false" case insensitive parameters.
 
 
 Misc Noticeable changes:
 ------------------------
 
 - Exceptions are now (re)raised when running notebooks via the :magic:`%run`, helping to catch issues in workflows and
   pipelines. :ghpull:`12301`
 - Fix inputhook for qt 5.15.0 :ghpull:`12355`
 - Fix wx inputhook :ghpull:`12375`
 - Add handling for malformed pathext env var (Windows) :ghpull:`12367`
 - use $SHELL in system_piped :ghpull:`12360` for uniform behavior with
   ipykernel.
 
 Reproducible Build
 ------------------
 
 IPython 7.15 reproducible build did not work, so we try again this month
 :ghpull:`12358`.
 
 
 API Changes
 -----------
 
 Change of API and exposed objects automatically detected using `frappuccino
 <https://pypi.org/project/frappuccino/>`_ (still in beta):
 
 
 The following items are new and mostly related to understanding ``__tracebackhide__``::
 
     + IPython.core.debugger.Pdb.do_down(self, arg)
     + IPython.core.debugger.Pdb.do_skip_hidden(self, arg)
     + IPython.core.debugger.Pdb.do_up(self, arg)
     + IPython.core.debugger.Pdb.hidden_frames(self, stack)
     + IPython.core.debugger.Pdb.stop_here(self, frame)
 
 
 The following items have been removed::
 
     - IPython.core.debugger.Pdb.new_do_down
     - IPython.core.debugger.Pdb.new_do_up
 
 Those were implementation details.
 
 
 .. _version 715:
 
 IPython 7.15
 ============
 
 IPython 7.15 brings a number of bug fixes and user facing improvements.
 
 Misc Noticeable changes:
 ------------------------
 
  - Long completion name have better elision in terminal :ghpull:`12284`
  - I've started to test on Python 3.9 :ghpull:`12307` and fix some errors.
  - Hi DPI scaling of figures when using qt eventloop :ghpull:`12314`
  - Document the ability to have systemwide configuration for IPython.
    :ghpull:`12328`
  - Fix issues with input autoformatting :ghpull:`12336`
  - ``IPython.core.debugger.Pdb`` is now interruptible (:ghpull:`12168`, in 7.14
    but forgotten in release notes)
  - Video HTML attributes (:ghpull:`12212`, in 7.14 but forgotten in release
    notes)
 
 Reproducible Build
 ------------------
 
 Starting with IPython 7.15, I am attempting to provide reproducible builds,
 that is to say you should be able from the source tree to generate an sdist
 and wheel that are identical byte for byte with the publish version on PyPI.
 
 I've only tested on a couple of machines so far and the process is relatively
 straightforward, so this mean that IPython not only have a deterministic build
 process, but also I have either removed, or put under control all effects of
 the build environments on the final artifact.  I encourage you to attempt the
 build process on your machine as documented in :ref:`core_developer_guide`
 and let me know if you do not obtain an identical artifact.
 
 While reproducible builds is critical to check that the supply chain of (open
 source) software has not been compromised, it can also help to speedup many
 of the build processes in large environment (conda, apt...) by allowing
 better caching of intermediate build steps.
 
 Learn more on `<https://reproducible-builds.org/>`_. `Reflections on trusting
 trust <https://dl.acm.org/doi/10.1145/358198.358210>`_ is also one of the
 cornerstone and recommended reads on this subject.
 
 .. note::
 
    The build commit from which the sdist is generated is also `signed
    <https://en.wikipedia.org/wiki/Digital_signature>`_, so you should be able to
    check it has not been compromised, and the git repository is a `merkle-tree
    <https://en.wikipedia.org/wiki/Merkle_tree>`_, you can check the consistency
    with `git-fsck <https://git-scm.com/docs/git-fsck>`_ which you likely `want
    to enable by default
    <https://gist.github.com/mbbx6spp/14b86437e794bffb4120>`_.
 
 NEP29: Last version to support Python 3.6
 -----------------------------------------
 
 IPython 7.15 will be the Last IPython version to officially support Python
 3.6, as stated by `NumPy Enhancement Proposal 29
 <https://numpy.org/neps/nep-0029-deprecation_policy.html>`_. Starting with
 next minor version of IPython I may stop testing on Python 3.6 and may stop
 publishing release artifacts that install on Python 3.6
 
 Highlighted features
 --------------------
 
 Highlighted features are not new, but seem to not be widely known, this
 section will help you discover in more narrative form what you can do with
 IPython.
 
 Increase Tab Completion Menu Height
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 In terminal IPython it is possible to increase the hight of the tab-completion
 menu. To do so set the value of
 :configtrait:`TerminalInteractiveShell.space_for_menu`, this will reserve more
 space at the bottom of the screen for various kind of menus in IPython including
 tab completion and searching in history. 
 
 Autoformat Code in the terminal
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 If you have a preferred code formatter, you can configure IPython to
 reformat your code. Set the value of
 :configtrait:`TerminalInteractiveShell.autoformatter` to for example ``'black'``
 and IPython will auto format your code when possible.
 
 
 .. _version 714:
 
 IPython 7.14
 ============
 
 IPython  7.14 is a minor release that fix a couple of bugs and prepare
 compatibility with new or future versions of some libraries. 
 
 Important changes:
 ------------------
 
  - Fix compatibility with Sphinx 3+ :ghpull:`12235`
  - Remove deprecated matplotlib parameter usage, compatibility with matplotlib
    3.3+ :`122250`
 
 Misc Changes
 ------------
 
  - set ``.py`` extension when editing current buffer in vi/emacs. :ghpull:`12167`
  - support for unicode identifiers in ``?``/``??`` :ghpull:`12208`
  - add extra options to the ``Video`` Rich objects :ghpull:`12212`
  - add pretty-printing to ``SimpleNamespace`` :ghpull:`12230`
 
 IPython.core.debugger.Pdb is now interruptible
 ----------------------------------------------
 
 A ``KeyboardInterrupt`` will now interrupt IPython's extended debugger, in order to make Jupyter able to interrupt it. (:ghpull:`12168`)
 
 Video HTML attributes
 ---------------------
 
 Add an option to `IPython.display.Video` to change the attributes of the HTML display of the video (:ghpull:`12212`)
 
 
 Pending deprecated imports
 --------------------------
 
 Many object present in ``IPython.core.display`` are there for internal use only,
 and should  already been imported from ``IPython.display`` by users and external
 libraries. Trying to import those from ``IPython.core.display`` is still possible
 but will trigger a
 deprecation warning in later versions of IPython and will become errors in the
 future. 
 
 This will simplify compatibility with other Python kernels (like Xeus-Python),
 and simplify code base. 
 
 
 
 
 .. _version 713:
 
 IPython 7.13
 ============
 
 IPython 7.13 is the final release of the 7.x branch since master is diverging
 toward an 8.0. Exiting new features have already been merged in 8.0 and will
 not be available on the 7.x branch. All the changes below have been backported
 from the master branch.
 
 
  - Fix inability to run PDB when inside an event loop :ghpull:`12141`
  - Fix ability to interrupt some processes on windows :ghpull:`12137`
  - Fix debugger shortcuts :ghpull:`12132`
  - improve tab completion when inside a string by removing irrelevant elements :ghpull:`12128`
  - Fix display of filename tab completion when the path is long :ghpull:`12122`
  - Many removal of Python 2 specific code path :ghpull:`12110`
  - displaying wav files do not require NumPy anymore, and is 5x to 30x faster :ghpull:`12113`
 
 See the list of all closed issues and pull request on `github
 <https://github.com/ipython/ipython/pulls?q=is%3Aclosed+milestone%3A7.13>`_.
 
 .. _version 712:
 
 IPython 7.12
 ============
 
 IPython 7.12 is a minor update that mostly brings code cleanup, removal of
 longtime deprecated function and a couple update to documentation cleanup as well.
 
 Notable changes are the following:
 
  - Exit non-zero when ipython is given a file path to run that doesn't exist :ghpull:`12074`
  - Test PR on ARM64 with Travis-CI :ghpull:`12073`
  - Update CI to work with latest Pytest :ghpull:`12086`
  - Add infrastructure to run ipykernel eventloop via trio :ghpull:`12097`
  - Support git blame ignore revs :ghpull:`12091`
  - Start multi-line ``__repr__`` s on their own line :ghpull:`12099`
 
 .. _version 7111:
 
 IPython 7.11.1
 ==============
 
 A couple of deprecated functions (no-op) have been reintroduces in py3compat as
 Cython was still relying on them, and will be removed in a couple of versions.
 
 .. _version 711:
 
 IPython 7.11
 ============
 
 IPython 7.11 received a couple of compatibility fixes and code cleanup.
 
 A number of function in the ``py3compat`` have been removed; a number of types
 in the IPython code base are now non-ambiguous and now always ``unicode``
 instead of ``Union[Unicode,bytes]``; many of the relevant code path have thus
 been simplified/cleaned and types annotation added.
 
 IPython support several verbosity level from exceptions. ``xmode plain`` now
 support chained exceptions. :ghpull:`11999`
 
 We are starting to remove ``shell=True`` in some usages of subprocess. While not directly
 a security issue (as IPython is made to run arbitrary code anyway) it is not good
 practice and we'd like to show the example. :ghissue:`12023`. This discussion
 was started by ``@mschwager`` thanks to a new auditing tool they are working on
 with duo-labs (`dlint <https://github.com/duo-labs/dlint>`_).
 
 Work around some bugs in Python 3.9 tokenizer :ghpull:`12057`
 
 IPython will now print its version after a crash. :ghpull:`11986`
 
 This is likely the last release from the 7.x series that will see new feature.
 The master branch will soon accept large code changes and thrilling new
 features; the 7.x branch will only start to accept critical bug fixes, and
 update dependencies.
 
 .. _version 7102:
 
 IPython 7.10.2
 ==============
 
 IPython 7.10.2 fix a couple of extra incompatibility between IPython, ipdb,
 asyncio and Prompt Toolkit 3.
 
 .. _version 7101:
 
 IPython 7.10.1
 ==============
 
 IPython 7.10.1 fix a couple of incompatibilities with Prompt toolkit 3 (please
 update Prompt toolkit to 3.0.2 at least), and fixes some interaction with
 headless IPython.
 
 .. _version 7100:
 
 IPython 7.10.0
 ==============
 
 IPython 7.10 is the first double digit minor release in the  last decade, and
 first since the release of IPython 1.0, previous double digit minor release was
 in August 2009.
 
 We've been trying to give you regular release on the last Friday of every month
 for a guaranty of rapid access to bug fixes and new features.
 
 Unlike the previous first few releases that have seen only a couple of code
 changes, 7.10 bring a number of changes, new features and bugfixes.
 
 Stop Support for Python 3.5 – Adopt NEP 29
 ------------------------------------------
 
 IPython has decided to follow the informational `NEP 29
 <https://numpy.org/neps/nep-0029-deprecation_policy.html>`_ which layout a clear
 policy as to which version of (C)Python and NumPy are supported.
 
 We thus dropped support for Python 3.5, and cleaned up a number of code path
 that were Python-version dependant. If you are on 3.5 or earlier pip should
 automatically give you the latest compatible version of IPython so you do not
 need to pin to a given version.
 
 Support for Prompt Toolkit 3.0
 ------------------------------
 
 Prompt Toolkit 3.0 was release a week before IPython 7.10 and introduces a few
 breaking changes. We believe IPython 7.10 should be compatible with both Prompt
 Toolkit 2.x and 3.x, though it has not been extensively tested with 3.x so
 please report any issues.
 
 
 Prompt Rendering Performance improvements
 -----------------------------------------
 
 Pull Request :ghpull:`11933` introduced an optimisation in the prompt rendering
 logic that should decrease the resource usage of IPython when using the
 _default_ configuration but could potentially introduce a regression of
 functionalities if you are using a custom prompt.
 
 We know assume if you haven't changed the default keybindings that the prompt
 **will not change** during the duration of your input – which is for example
 not true when using vi insert mode that switches between `[ins]` and `[nor]`
 for the current mode.
 
 If you are experiencing any issue let us know.
 
 Code autoformatting
 -------------------
 
 The IPython terminal can now auto format your code just before entering a new
 line or executing a command. To do so use the
 ``--TerminalInteractiveShell.autoformatter`` option and set it to ``'black'``;
 if black is installed IPython will use black to format your code when possible.
 
 IPython cannot always properly format your code; in particular it will
 auto formatting with *black* will only work if:
 
    - Your code does not contains magics or special python syntax.
 
    - There is no code after your cursor.
 
 The Black API is also still in motion; so this may not work with all versions of
 black.
 
 It should be possible to register custom formatter, though the API is till in
 flux.
 
 Arbitrary Mimetypes Handing in Terminal (Aka inline images in terminal)
 -----------------------------------------------------------------------
 
 When using IPython terminal it is now possible to register function to handle
 arbitrary mimetypes. While rendering non-text based representation was possible in
 many jupyter frontend; it was not possible in terminal IPython, as usually
 terminal are limited to displaying text. As many terminal these days provide
 escape sequences to display non-text; bringing this loved feature to IPython CLI
 made a lot of sens. This functionality will not only allow inline images; but
 allow opening of external program; for example ``mplayer`` to "display" sound
 files.
 
 So far only the hooks necessary for this are in place, but no default mime
 renderers added; so inline images will only be available via extensions. We will
 progressively enable these features by default in the next few releases, and
 contribution is welcomed.
 
 We welcome any feedback on the API. See :ref:`shell_mimerenderer` for more
 informations.
 
 This is originally based on work form in :ghpull:`10610` from @stephanh42
 started over two years ago, and still a lot need to be done.
 
 MISC
 ----
 
  - Completions can define their own ordering :ghpull:`11855`
  - Enable Plotting in the same cell than the one that import matplotlib
    :ghpull:`11916`
  - Allow to store and restore multiple variables at once :ghpull:`11930`
 
 You can see `all pull-requests <https://github.com/ipython/ipython/pulls?q=is%3Apr+milestone%3A7.10+is%3Aclosed>`_ for this release.
 
 API Changes
 -----------
 
 Change of API and exposed objects automatically detected using `frappuccino <https://pypi.org/project/frappuccino/>`_ (still in beta):
 
 The following items are new in IPython 7.10::
 
     + IPython.terminal.shortcuts.reformat_text_before_cursor(buffer, document, shell)
     + IPython.terminal.interactiveshell.PTK3
     + IPython.terminal.interactiveshell.black_reformat_handler(text_before_cursor)
     + IPython.terminal.prompts.RichPromptDisplayHook.write_format_data(self, format_dict, md_dict='None')
 
 The following items have been removed in 7.10::
 
     - IPython.lib.pretty.DICT_IS_ORDERED
 
 The following signatures differ between versions::
 
     - IPython.extensions.storemagic.restore_aliases(ip)
     + IPython.extensions.storemagic.restore_aliases(ip, alias='None')
 
 Special Thanks
 --------------
 
  - @stephanh42 who started the work on inline images in terminal 2 years ago
  - @augustogoulart who spent a lot of time triaging issues and responding to
    users.
  - @con-f-use who is my (@Carreau) first sponsor on GitHub, as a reminder if you
    like IPython, Jupyter and many other library of the SciPy stack you can
    donate to numfocus.org non profit
 
 .. _version 790:
 
 IPython 7.9.0
 =============
 
 IPython 7.9 is a small release with a couple of improvement and bug fixes.
 
  - Xterm terminal title should be restored on exit :ghpull:`11910`
  - special variables ``_``,``__``, ``___`` are not set anymore when cache size
    is 0 or less.  :ghpull:`11877`
  - Autoreload should have regained some speed by using a new heuristic logic to
    find all objects needing reload. This should avoid large objects traversal
    like pandas dataframes. :ghpull:`11876`
  - Get ready for Python 4. :ghpull:`11874`
  - `%env` Magic now has heuristic to hide potentially sensitive values :ghpull:`11896`
 
 This is a small release despite a number of Pull Request Pending that need to
 be reviewed/worked on. Many of the core developers have been busy outside of
 IPython/Jupyter and we thanks all contributor for their patience; we'll work on
 these as soon as we have time.
 
 
 .. _version780:
 
 IPython 7.8.0
 =============
 
 IPython 7.8.0 contain a few bugfix and 2 new APIs:
 
  - Enable changing the font color for LaTeX rendering :ghpull:`11840`
  - and Re-Expose some PDB API (see below)
 
 Expose Pdb API
 --------------
 
 Expose the built-in ``pdb.Pdb`` API. ``Pdb`` constructor arguments are generically
 exposed, regardless of python version.
 Newly exposed arguments:
 
 - ``skip`` - Python 3.1+
 - ``nosiginnt`` - Python 3.2+
 - ``readrc`` - Python 3.6+
 
 Try it out::
 
     from IPython.terminal.debugger import TerminalPdb
     pdb = TerminalPdb(skip=["skipthismodule"])
 
 
 See :ghpull:`11840`
 
 .. _version770:
 
 IPython 7.7.0
 =============
 
 IPython 7.7.0 contain multiple bug fixes and documentation updates; Here are a
 few of the outstanding issue fixed:
 
    - Fix a bug introduced in 7.6 where the ``%matplotlib`` magic would fail on
      previously acceptable arguments :ghpull:`11814`.
    - Fix the manage location on freebsd :ghpull:`11808`.
    - Fix error message about aliases after ``%reset`` call in ipykernel
      :ghpull:`11806`
    - Fix Duplication completions in emacs :ghpull:`11803`
 
 We are planning to adopt `NEP29 <https://github.com/numpy/numpy/pull/14086>`_
 (still currently in draft) which may make this minor version of IPython the
 last one to support Python 3.5 and will make the code base more aggressive
 toward removing compatibility with older versions of Python.
 
 GitHub now support to give only "Triage" permissions to users; if you'd like to
 help close stale issues and labels issues please reach to us with your GitHub
 Username and we'll add you to the triage team. It is a great way to start
 contributing and a path toward getting commit rights.
 
 .. _version761:
 
 IPython 7.6.1
 =============
 
 IPython 7.6.1 contain a critical bugfix in the ``%timeit`` magic, which would
 crash on some inputs as a side effect of :ghpull:`11716`. See :ghpull:`11812`
 
 
 .. _whatsnew760:
 
 IPython 7.6.0
 =============
 
 IPython 7.6.0 contains a couple of bug fixes and number of small features
 additions as well as some compatibility with the current development version of
 Python 3.8.
 
    - Add a ``-l`` option to :magic:`psearch` to list the available search
      types. :ghpull:`11672`
    - Support ``PathLike`` for ``DisplayObject`` and ``Image``. :ghpull:`11764`
    - Configurability of timeout in the test suite for slow platforms.
      :ghpull:`11756`
    - Accept any casing for matplotlib backend. :ghpull:`121748`
    - Properly skip test that requires numpy to be installed :ghpull:`11723`
    - More support for Python 3.8 and positional only arguments (pep570)
      :ghpull:`11720`
    - Unicode names for the completion are loaded lazily on first use which
      should decrease startup time. :ghpull:`11693`
    - Autoreload now update the types of reloaded objects; this for example allow
      pickling of reloaded objects. :ghpull:`11644`
    - Fix a bug where ``%%time`` magic would suppress cell output. :ghpull:`11716`
 
 
 Prepare migration to pytest (instead of nose) for testing
 ---------------------------------------------------------
 
 Most of the work between 7.5 and 7.6 was to prepare the migration from our
 testing framework to pytest. Most of the test suite should now work by simply
 issuing ``pytest`` from the root of the repository.
 
 The migration to pytest is just at its beginning. Many of our test still rely
 on IPython-specific plugins for nose using pytest (doctest using IPython syntax
 is one example of this where test appear as "passing", while no code has been
 ran). Many test also need to be updated like ``yield-test`` to be properly
 parametrized tests.
 
 Migration to pytest allowed me to discover a number of issues in our test
 suite; which was hiding a number of subtle issues – or not actually running
 some of the tests in our test suite – I have thus corrected many of those; like
 improperly closed resources; or used of deprecated features. I also made use of
 the ``pytest --durations=...`` to find some of our slowest test and speed them
 up (our test suite can now be up to 10% faster). Pytest as also a variety of
 plugins and flags which will make the code quality of IPython and the testing
 experience better.
 
 Misc
 ----
 
 We skipped the release of 7.6 at the end of May, but will attempt to get back
 on schedule. We are starting to think about making introducing backward
 incompatible change and start the 8.0 series.
 
 Special Thanks to Gabriel (@gpotter2 on GitHub), who among other took care many
 of the remaining task for 7.4 and 7.5, like updating the website.
 
 .. _whatsnew750:
 
 IPython 7.5.0
 =============
 
 IPython 7.5.0 consist mostly of bug-fixes, and documentation updates, with one
 minor new feature. The `Audio` display element can now be assigned an element
 id when displayed in browser. See :ghpull:`11670`
 
 The major outstanding bug fix correct a change of behavior that was introduce
 in 7.4.0 where some cell magics would not be able to access or modify global
 scope when using the ``@needs_local_scope`` decorator. This was typically
 encountered with the ``%%time`` and ``%%timeit`` magics. See :ghissue:`11659`
 and :ghpull:`11698`.
 
 .. _whatsnew740:
 
 IPython 7.4.0
 =============
 
 Unicode name completions
 ------------------------
 
 Previously, we provided completion for a unicode name with its relative symbol.
 With this, now IPython provides complete suggestions to unicode name symbols.
 
 As on the PR, if user types ``\LAT<tab>``, IPython provides a list of
 possible completions. In this case, it would be something like::
 
    'LATIN CAPITAL LETTER A',
    'LATIN CAPITAL LETTER B',
    'LATIN CAPITAL LETTER C',
    'LATIN CAPITAL LETTER D',
    ....
 
 This help to type unicode character that do not have short latex aliases, and
 have long unicode names. for example ``Ͱ``, ``\GREEK CAPITAL LETTER HETA``.
 
 This feature was contributed by Luciana Marques :ghpull:`11583`.
 
 Make audio normalization optional
 ---------------------------------
 
 Added 'normalize' argument to `IPython.display.Audio`. This argument applies
 when audio data is given as an array of samples. The default of `normalize=True`
 preserves prior behavior of normalizing the audio to the maximum possible range.
 Setting to `False` disables normalization.
 
 
 Miscellaneous
 -------------
 
  - Fix improper acceptation of ``return`` outside of functions. :ghpull:`11641`.
  - Fixed PyQt 5.11 backwards incompatibility causing sip import failure.
    :ghpull:`11613`.
  - Fix Bug where ``type?`` would crash IPython. :ghpull:`1608`.
  - Allow to apply ``@needs_local_scope`` to cell magics for convenience.
    :ghpull:`11542`.
 
 .. _whatsnew730:
 
 IPython 7.3.0
 =============
 
 .. _whatsnew720:
 
 IPython 7.3.0 bring several bug fixes and small improvements that you will
 described bellow. 
 
 The biggest change to this release is the implementation of the ``%conda`` and
 ``%pip`` magics, that will attempt to install packages in the **current
 environment**. You may still need to restart your interpreter or kernel for the
 change to be taken into account, but it should simplify installation of packages
 into remote environment. Installing using pip/conda from the command line is
 still the prefer method.
 
 The ``%pip`` magic was already present, but was only printing a warning; now it
 will actually forward commands to pip. 
 
 Misc bug fixes and improvements:
 
  - Compatibility with Python 3.8.
  - Do not expand shell variable in execution magics, and added the
    ``no_var_expand`` decorator for magic requiring a similar functionality
    :ghpull:`11516`
  - Add ``%pip`` and ``%conda`` magic :ghpull:`11524`
  - Re-initialize posix aliases after a ``%reset`` :ghpull:`11528`
  - Allow the IPython command line to run ``*.ipynb`` files :ghpull:`11529`
 
 IPython 7.2.0
 =============
 
 IPython 7.2.0 brings minor bugfixes, improvements, and new configuration options:
 
  - Fix a bug preventing PySide2 GUI integration from working :ghpull:`11464`
  - Run CI on Mac OS ! :ghpull:`11471`
  - Fix IPython "Demo" mode. :ghpull:`11498`
  - Fix ``%run`` magic  with path in name :ghpull:`11499`
  - Fix: add CWD to sys.path *after* stdlib :ghpull:`11502`
  - Better rendering of signatures, especially long ones. :ghpull:`11505`
  - Re-enable jedi by default if it's installed :ghpull:`11506`
  - Add New ``minimal`` exception reporting mode (useful for educational purpose). See :ghpull:`11509`
 
 
 Added ability to show subclasses when using pinfo and other utilities
 ---------------------------------------------------------------------
 
 When using ``?``/``??`` on a class, IPython will now list the first 10 subclasses.
 
 Special Thanks to Chris Mentzel of the Moore Foundation for this feature. Chris
 is one of the people who played a critical role in IPython/Jupyter getting
 funding.
 
 We are grateful for all the help Chris has given us over the years,
 and we're now proud to have code contributed by Chris in IPython.
 
 OSMagics.cd_force_quiet configuration option
 --------------------------------------------
 
 You can set this option to force the %cd magic to behave as if ``-q`` was passed:
 ::
 
     In [1]: cd /
     /
 
     In [2]: %config OSMagics.cd_force_quiet = True
 
     In [3]: cd /tmp
 
     In [4]:
 
 See :ghpull:`11491`
 
 In vi editing mode, whether the prompt includes the current vi mode can now be configured
 -----------------------------------------------------------------------------------------
 
 Set the ``TerminalInteractiveShell.prompt_includes_vi_mode`` to a boolean value
 (default: True) to control this feature. See :ghpull:`11492`
 
 .. _whatsnew710:
 
 IPython 7.1.0
 =============
 
 IPython 7.1.0 is the first minor release after 7.0.0 and mostly brings fixes to
 new features, internal refactoring, and fixes for regressions that happened during the 6.x->7.x
 transition. It also brings **Compatibility with Python 3.7.1**, as we're
 unwillingly relying on a bug in CPython.
 
 New Core Dev:
 
  - We welcome Jonathan Slenders to the commiters. Jonathan has done a fantastic
    work on prompt_toolkit, and we'd like to recognise his impact by giving him
    commit rights. :ghissue:`11397`
 
 Notable Changes
 
  - Major update of "latex to unicode" tab completion map (see below)
 
 Notable New Features:
 
  - Restore functionality and documentation of the **sphinx directive**, which
    is now stricter (fail on error by daefault), has new configuration options,
    has a brand new documentation page :ref:`ipython_directive` (which needs
    some cleanup). It is also now *tested* so we hope to have less regressions.
    :ghpull:`11402`
 
  - ``IPython.display.Video`` now supports ``width`` and ``height`` arguments,
    allowing a custom width and height to be set instead of using the video's
    width and height. :ghpull:`11353`
 
  - Warn when using ``HTML('<iframe>')`` instead of ``IFrame`` :ghpull:`11350`
 
  - Allow Dynamic switching of editing mode between vi/emacs and show
    normal/input mode in prompt when using vi. :ghpull:`11390`. Use ``%config
    TerminalInteractiveShell.editing_mode = 'vi'`` or ``%config
    TerminalInteractiveShell.editing_mode = 'emacs'`` to dynamically switch
    between modes.
 
 
 Notable Fixes:
 
  - Fix entering of **multi-line blocks in terminal** IPython, and various
    crashes in the new input transformation machinery :ghpull:`11354`,
    :ghpull:`11356`, :ghpull:`11358`. These also fix a **Compatibility bug
    with Python 3.7.1**.
 
  - Fix moving through generator stack in ipdb :ghpull:`11266`
 
  - %Magic command arguments now support quoting. :ghpull:`11330`
 
  - Re-add ``rprint`` and ``rprinte`` aliases. :ghpull:`11331`
 
  - Remove implicit dependency on ``ipython_genutils`` :ghpull:`11317`
 
  - Make ``nonlocal`` raise ``SyntaxError`` instead of silently failing in async
    mode. :ghpull:`11382`
 
  - Fix mishandling of magics and ``= !`` assignment just after a dedent in
    nested code blocks :ghpull:`11418`
 
  - Fix instructions for custom shortcuts :ghpull:`11426`
 
 
 Notable Internals improvements:
 
  - Use of ``os.scandir`` (Python 3 only) to speed up some file system operations.
    :ghpull:`11365`
 
  - use ``perf_counter`` instead of ``clock`` for more precise
    timing results with ``%time`` :ghpull:`11376`
 
 Many thanks to all the contributors and in particular to ``bartskowron`` and
 ``tonyfast`` who handled some pretty complicated bugs in the input machinery. We
 had a number of first time contributors and maybe hacktoberfest participants that
 made significant contributions and helped us free some time to focus on more
 complicated bugs.
 
 You
 can see all the closed issues and Merged PR, new features and fixes `here
 <https://github.com/ipython/ipython/issues?utf8=%E2%9C%93&q=+is%3Aclosed+milestone%3A7.1+>`_.
 
 Unicode Completion update
 -------------------------
 
 In IPython 7.1 the Unicode completion map has been updated and synchronized with
 the Julia language.
 
 Added and removed character characters:
 
  ``\jmath`` (``ȷ``), ``\\underleftrightarrow`` (U+034D, combining) have been
  added, while ``\\textasciicaron`` have been removed
 
 Some sequences have seen their prefix removed:
 
  - 6 characters ``\text...<tab>`` should now be inputed with ``\...<tab>`` directly,
  - 45 characters ``\Elz...<tab>`` should now be inputed with ``\...<tab>`` directly,
  - 65 characters ``\B...<tab>`` should now be inputed with ``\...<tab>`` directly,
  - 450 characters ``\m...<tab>`` should now be inputed with ``\...<tab>`` directly,
 
 Some sequences have seen their prefix shortened:
 
  - 5 characters ``\mitBbb...<tab>`` should now be inputed with ``\bbi...<tab>`` directly,
  - 52 characters ``\mit...<tab>`` should now be inputed with ``\i...<tab>`` directly,
  - 216 characters ``\mbfit...<tab>`` should now be inputed with ``\bi...<tab>`` directly,
  - 222 characters ``\mbf...<tab>`` should now be inputed with ``\b...<tab>`` directly,
 
 A couple of characters had their sequence simplified:
 
  - ``ð``, type ``\dh<tab>``, instead of ``\eth<tab>``
  - ``ħ``, type ``\hbar<tab>``, instead of ``\Elzxh<tab>``
  - ``ɸ``, type ``\ltphi<tab>``, instead of ``\textphi<tab>``
  - ``ϴ``, type ``\varTheta<tab>``, instead of ``\textTheta<tab>``
  - ``ℇ``, type ``\eulermascheroni<tab>``, instead of ``\Eulerconst<tab>``
  - ``ℎ``, type ``\planck<tab>``, instead of ``\Planckconst<tab>``
 
  - U+0336 (COMBINING LONG STROKE OVERLAY), type ``\strike<tab>``, instead of ``\Elzbar<tab>``.
 
 A couple of sequences have been updated:
 
  - ``\varepsilon`` now gives ``ɛ`` (GREEK SMALL LETTER EPSILON) instead of ``ε`` (GREEK LUNATE EPSILON SYMBOL),
  - ``\underbar`` now gives U+0331 (COMBINING MACRON BELOW) instead of U+0332 (COMBINING LOW LINE).
 
 
 .. _whatsnew700:
 
 IPython 7.0.0
 =============
 
 Released Thursday September 27th, 2018
 
 IPython 7 includes major feature improvements.
 This is also the second major version of IPython to support only
 Python 3 – starting at Python 3.4. Python 2 is still community-supported
 on the bugfix only 5.x branch, but we remind you that Python 2 "end of life"
 is on Jan 1st 2020.
 
 We were able to backport bug fixes to the 5.x branch thanks to our backport bot which
 backported more than `70 Pull-Requests
 <https://github.com/ipython/ipython/pulls?page=3&q=is%3Apr+sort%3Aupdated-desc+author%3Aapp%2Fmeeseeksdev++5.x&utf8=%E2%9C%93>`_, but there are still many PRs that required manual work. This is an area of the project where you can easily contribute by looking for `PRs that still need manual backport <https://github.com/ipython/ipython/issues?q=label%3A%22Still+Needs+Manual+Backport%22+is%3Aclosed+sort%3Aupdated-desc>`_
 
 The IPython 6.x branch will likely not see any further release unless critical
 bugs are found.
 
 Make sure you have pip > 9.0 before upgrading. You should be able to update by running:
 
 .. code::
 
     pip install ipython --upgrade
 
 .. only:: ipydev
 
   If you are trying to install or update an ``alpha``, ``beta``, or ``rc``
   version, use pip ``--pre`` flag.
 
   .. code::
 
       pip install ipython --upgrade --pre
 
 
 Or, if you have conda installed: 
 
 .. code::
    
    conda install ipython
 
 
 
 Prompt Toolkit 2.0
 ------------------
 
 IPython 7.0+ now uses ``prompt_toolkit 2.0``. If you still need to use an earlier
 ``prompt_toolkit`` version, you may need to pin IPython to ``<7.0``.
 
 Autowait: Asynchronous REPL
 ---------------------------
 
 Staring with IPython 7.0 on Python 3.6+, IPython can automatically ``await``
 top level code. You should not need to access an event loop or runner
 yourself. To learn more, read the :ref:`autoawait` section of our docs, see
 :ghpull:`11265`, or try the following code::
 
     Python 3.6.0
     Type 'copyright', 'credits' or 'license' for more information
     IPython 7.0.0 -- An enhanced Interactive Python. Type '?' for help.
 
     In [1]: import aiohttp
        ...: result = aiohttp.get('https://api.github.com')
 
     In [2]: response = await result
     <pause for a few 100s ms>
 
     In [3]: await response.json()
     Out[3]:
     {'authorizations_url': 'https://api.github.com/authorizations',
      'code_search_url': 'https://api.github.com/search/code?q={query}{&page,per_page,sort,order}',
     ...
     }
 
 .. note::
 
    Async integration is experimental code, behavior may change or be removed
    between Python and IPython versions without warnings.
 
 Integration is by default with `asyncio`, but other libraries can be configured --
 like ``curio`` or ``trio`` -- to improve concurrency in the REPL::
 
     In [1]: %autoawait trio
 
     In [2]: import trio
 
     In [3]: async def child(i):
        ...:     print("   child %s goes to sleep"%i)
        ...:     await trio.sleep(2)
        ...:     print("   child %s wakes up"%i)
 
     In [4]: print('parent start')
        ...: async with trio.open_nursery() as n:
        ...:     for i in range(3):
        ...:         n.spawn(child, i)
        ...: print('parent end')
     parent start
        child 2 goes to sleep
        child 0 goes to sleep
        child 1 goes to sleep
        <about 2 seconds pause>
        child 2 wakes up
        child 1 wakes up
        child 0 wakes up
     parent end
 
 See :ref:`autoawait` for more information.
 
 
 Asynchronous code in a Notebook interface or any other frontend using the
 Jupyter Protocol will require further updates to the IPykernel package.
 
 Non-Asynchronous code
 ~~~~~~~~~~~~~~~~~~~~~
 
 As the internal API of IPython is now asynchronous, IPython needs to run under
 an event loop. In order to allow many workflows, (like using the :magic:`%run`
 magic, or copy-pasting code that explicitly starts/stop event loop), when
 top-level code is detected as not being asynchronous, IPython code is advanced
 via a pseudo-synchronous runner, and may not advance pending tasks.
 
 Change to Nested Embed
 ~~~~~~~~~~~~~~~~~~~~~~
 
 The introduction of the ability to run async code had some effect on the
 ``IPython.embed()`` API. By default, embed will not allow you to run asynchronous
 code unless an event loop is specified.
 
 Effects on Magics
 ~~~~~~~~~~~~~~~~~
 
 Some magics will not work with async until they're updated.
 Contributions welcome.
 
 Expected Future changes
 ~~~~~~~~~~~~~~~~~~~~~~~
 
 We expect more internal but public IPython functions to become ``async``, and
 will likely end up having a persistent event loop while IPython is running.
 
 Thanks
 ~~~~~~
 
 This release took more than a year in the making.
 The code was rebased a number of
 times; leading to commit authorship that may have been lost in the final
 Pull-Request. Huge thanks to many people for contribution, discussion, code,
 documentation, use-cases: dalejung, danielballan, ellisonbg, fperez, gnestor,
 minrk, njsmith, pganssle, tacaswell, takluyver , vidartf ... And many others.
 
 
 Autoreload Improvement
 ----------------------
 
 The magic :magic:`%autoreload 2 <autoreload>` now captures new methods added to
 classes. Earlier, only methods existing as of the initial import were being
 tracked and updated.  
 
 This new feature helps dual environment development - Jupyter+IDE - where the
 code gradually moves from notebook cells to package files as it gets
 structured.
 
 **Example**: An instance of the class ``MyClass`` will be able to access the
 method ``cube()`` after it is uncommented and the file ``file1.py`` is saved on
 disk.
 
 
 .. code::
 
    # notebook
 
    from mymodule import MyClass
    first = MyClass(5)
 
 .. code::
 
    # mymodule/file1.py
 
    class MyClass:
 
        def __init__(self, a=10):
            self.a = a
 
        def square(self):
            print('compute square')
            return self.a*self.a
 
        # def cube(self):
        #     print('compute cube')
        #     return self.a*self.a*self.a
 
 
 
 
 Misc
 ----
 
 The autoindent feature that was deprecated in 5.x was re-enabled and
 un-deprecated in :ghpull:`11257`
 
 Make :magic:`%run -n -i ... <run>` work correctly. Earlier, if :magic:`%run` was
 passed both arguments, ``-n`` would be silently ignored. See :ghpull:`10308`
 
 
 The :cellmagic:`%%script` (as well as :cellmagic:`%%bash`,
 :cellmagic:`%%ruby`... ) cell magics now raise by default if the return code of
 the given code is non-zero (thus halting execution of further cells in a
 notebook). The behavior can be disable by passing the ``--no-raise-error`` flag.
 
 
 Deprecations
 ------------
 
 A couple of unused functions and methods have been deprecated and will be removed
 in future versions:
 
   - ``IPython.utils.io.raw_print_err``
   - ``IPython.utils.io.raw_print``
 
   
 Backwards incompatible changes
 ------------------------------
 
 * The API for transforming input before it is parsed as Python code has been
   completely redesigned: any custom input transformations will need to be
   rewritten. See :doc:`/config/inputtransforms` for details of the new API.