Revert "Ruffisation"

IPython/core/application.py

@@ -1,492 +1,492
 # encoding: utf-8
 """
 An application for IPython.
 
 All top-level applications should use the classes in this module for
 handling configuration and creating configurables.
 
 The job of an :class:`Application` is to create the master configuration
 object and then create the configurable objects, passing the config to them.
 """
 
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 import atexit
 from copy import deepcopy
 import logging
 import os
 import shutil
 import sys
 
 from pathlib import Path
 
 from traitlets.config.application import Application, catch_config_error
 from traitlets.config.loader import ConfigFileNotFound, PyFileConfigLoader
 from IPython.core import release, crashhandler
 from IPython.core.profiledir import ProfileDir, ProfileDirError
 from IPython.paths import get_ipython_dir, get_ipython_package_dir
 from IPython.utils.path import ensure_dir_exists
 from traitlets import (
     List, Unicode, Type, Bool, Set, Instance, Undefined,
     default, observe,
 )
 
 if os.name == "nt":
     programdata = os.environ.get("PROGRAMDATA", None)
     if programdata is not None:
         SYSTEM_CONFIG_DIRS = [str(Path(programdata) / "ipython")]
     else:  # PROGRAMDATA is not defined by default on XP.
         SYSTEM_CONFIG_DIRS = []
 else:
     SYSTEM_CONFIG_DIRS = [
         "/usr/local/etc/ipython",
         "/etc/ipython",
     ]
 
 
 ENV_CONFIG_DIRS = []
 _env_config_dir = os.path.join(sys.prefix, 'etc', 'ipython')
 if _env_config_dir not in SYSTEM_CONFIG_DIRS:
     # only add ENV_CONFIG if sys.prefix is not already included
     ENV_CONFIG_DIRS.append(_env_config_dir)
 
 
 _envvar = os.environ.get('IPYTHON_SUPPRESS_CONFIG_ERRORS')
 if _envvar in {None, ''}:
     IPYTHON_SUPPRESS_CONFIG_ERRORS = None
 else:
     if _envvar.lower() in {'1','true'}:
         IPYTHON_SUPPRESS_CONFIG_ERRORS = True
     elif _envvar.lower() in {'0','false'} :
         IPYTHON_SUPPRESS_CONFIG_ERRORS = False
     else:
         sys.exit("Unsupported value for environment variable: 'IPYTHON_SUPPRESS_CONFIG_ERRORS' is set to '%s' which is none of  {'0', '1', 'false', 'true', ''}."% _envvar )
 
 # aliases and flags
 
 base_aliases = {}
 if isinstance(Application.aliases, dict):
     # traitlets 5
     base_aliases.update(Application.aliases)
 base_aliases.update(
     {
         "profile-dir": "ProfileDir.location",
         "profile": "BaseIPythonApplication.profile",
         "ipython-dir": "BaseIPythonApplication.ipython_dir",
         "log-level": "Application.log_level",
         "config": "BaseIPythonApplication.extra_config_file",
     }
 )
 
 base_flags = dict()
 if isinstance(Application.flags, dict):
     # traitlets 5
     base_flags.update(Application.flags)
 base_flags.update(
     dict(
         debug=(
             {"Application": {"log_level": logging.DEBUG}},
             "set log level to logging.DEBUG (maximize logging output)",
         ),
         quiet=(
             {"Application": {"log_level": logging.CRITICAL}},
             "set log level to logging.CRITICAL (minimize logging output)",
         ),
         init=(
             {
                 "BaseIPythonApplication": {
                     "copy_config_files": True,
                     "auto_create": True,
                 }
             },
             """Initialize profile with default config files.  This is equivalent
             to running `ipython profile create <profile>` prior to startup.
             """,
         ),
     )
 )
 
 
 class ProfileAwareConfigLoader(PyFileConfigLoader):
     """A Python file config loader that is aware of IPython profiles."""
     def load_subconfig(self, fname, path=None, profile=None):
         if profile is not None:
             try:
                 profile_dir = ProfileDir.find_profile_dir_by_name(
                         get_ipython_dir(),
                         profile,
                 )
             except ProfileDirError:
                 return
             path = profile_dir.location
         return super(ProfileAwareConfigLoader, self).load_subconfig(fname, path=path)
 
 class BaseIPythonApplication(Application):
     name = "ipython"
     description = "IPython: an enhanced interactive Python shell."
     version = Unicode(release.version)
 
     aliases = base_aliases
     flags = base_flags
     classes = List([ProfileDir])
     
     # enable `load_subconfig('cfg.py', profile='name')`
     python_config_loader_class = ProfileAwareConfigLoader
 
     # Track whether the config_file has changed,
     # because some logic happens only if we aren't using the default.
     config_file_specified = Set()
 
     config_file_name = Unicode()
     @default('config_file_name')
     def _config_file_name_default(self):
         return self.name.replace('-','_') + u'_config.py'
     @observe('config_file_name')
     def _config_file_name_changed(self, change):
         if change['new'] != change['old']:
             self.config_file_specified.add(change['new'])
 
     # The directory that contains IPython's builtin profiles.
     builtin_profile_dir = Unicode(
         os.path.join(get_ipython_package_dir(), u'config', u'profile', u'default')
     )
     
     config_file_paths = List(Unicode())
     @default('config_file_paths')
     def _config_file_paths_default(self):
         return []
 
     extra_config_file = Unicode(
     help="""Path to an extra config file to load.
     
     If specified, load this config file in addition to any other IPython config.
     """).tag(config=True)
     @observe('extra_config_file')
     def _extra_config_file_changed(self, change):
         old = change['old']
         new = change['new']
         try:
             self.config_files.remove(old)
         except ValueError:
             pass
         self.config_file_specified.add(new)
         self.config_files.append(new)
 
     profile = Unicode(u'default',
         help="""The IPython profile to use."""
     ).tag(config=True)
     
     @observe('profile')
     def _profile_changed(self, change):
         self.builtin_profile_dir = os.path.join(
                 get_ipython_package_dir(), u'config', u'profile', change['new']
         )
 
     add_ipython_dir_to_sys_path = Bool(
         False,
         """Should the IPython profile directory be added to sys path ?
 
         This option was non-existing before IPython 8.0, and ipython_dir was added to
         sys path to allow import of extensions present there. This was historical
         baggage from when pip did not exist. This now default to false,
         but can be set to true for legacy reasons.
         """,
     ).tag(config=True)
 
     ipython_dir = Unicode(
         help="""
         The name of the IPython directory. This directory is used for logging
         configuration (through profiles), history storage, etc. The default
         is usually $HOME/.ipython. This option can also be specified through
         the environment variable IPYTHONDIR.
         """
     ).tag(config=True)
     @default('ipython_dir')
     def _ipython_dir_default(self):
         d = get_ipython_dir()
         self._ipython_dir_changed({
             'name': 'ipython_dir',
             'old': d,
             'new': d,
         })
         return d
     
     _in_init_profile_dir = False
 
     profile_dir = Instance(ProfileDir, allow_none=True)
 
     @default('profile_dir')
     def _profile_dir_default(self):
         # avoid recursion
         if self._in_init_profile_dir:
             return
         # profile_dir requested early, force initialization
         self.init_profile_dir()
         return self.profile_dir
 
     overwrite = Bool(False,
         help="""Whether to overwrite existing config files when copying"""
     ).tag(config=True)
 
     auto_create = Bool(False,
         help="""Whether to create profile dir if it doesn't exist"""
     ).tag(config=True)
 
     config_files = List(Unicode())
 
     @default('config_files')
     def _config_files_default(self):
         return [self.config_file_name]
 
     copy_config_files = Bool(False,
         help="""Whether to install the default config files into the profile dir.
         If a new profile is being created, and IPython contains config files for that
         profile, then they will be staged into the new directory.  Otherwise,
         default config files will be automatically generated.
         """).tag(config=True)
     
     verbose_crash = Bool(False,
         help="""Create a massive crash report when IPython encounters what may be an
         internal error.  The default is to append a short message to the
         usual traceback""").tag(config=True)
 
     # The class to use as the crash handler.
     crash_handler_class = Type(crashhandler.CrashHandler)
 
     @catch_config_error
     def __init__(self, **kwargs):
         super(BaseIPythonApplication, self).__init__(**kwargs)
         # ensure current working directory exists
         try:
             os.getcwd()
-        except Exception:
+        except:
             # exit if cwd doesn't exist
             self.log.error("Current working directory doesn't exist.")
             self.exit(1)
 
     #-------------------------------------------------------------------------
     # Various stages of Application creation
     #-------------------------------------------------------------------------
     
     def init_crash_handler(self):
         """Create a crash handler, typically setting sys.excepthook to it."""
         self.crash_handler = self.crash_handler_class(self)
         sys.excepthook = self.excepthook
         def unset_crashhandler():
             sys.excepthook = sys.__excepthook__
         atexit.register(unset_crashhandler)
     
     def excepthook(self, etype, evalue, tb):
         """this is sys.excepthook after init_crashhandler
 
         set self.verbose_crash=True to use our full crashhandler, instead of
         a regular traceback with a short message (crash_handler_lite)
         """
         
         if self.verbose_crash:
             return self.crash_handler(etype, evalue, tb)
         else:
             return crashhandler.crash_handler_lite(etype, evalue, tb)
 
     @observe('ipython_dir')
     def _ipython_dir_changed(self, change):
         old = change['old']
         new = change['new']
         if old is not Undefined:
             str_old = os.path.abspath(old)
             if str_old in sys.path:
                 sys.path.remove(str_old)
         if self.add_ipython_dir_to_sys_path:
             str_path = os.path.abspath(new)
             sys.path.append(str_path)
             ensure_dir_exists(new)
             readme = os.path.join(new, "README")
             readme_src = os.path.join(
                 get_ipython_package_dir(), "config", "profile", "README"
             )
             if not os.path.exists(readme) and os.path.exists(readme_src):
                 shutil.copy(readme_src, readme)
             for d in ("extensions", "nbextensions"):
                 path = os.path.join(new, d)
                 try:
                     ensure_dir_exists(path)
                 except OSError as e:
                     # this will not be EEXIST
                     self.log.error("couldn't create path %s: %s", path, e)
             self.log.debug("IPYTHONDIR set to: %s", new)
 
     def load_config_file(self, suppress_errors=IPYTHON_SUPPRESS_CONFIG_ERRORS):
         """Load the config file.
 
         By default, errors in loading config are handled, and a warning
         printed on screen. For testing, the suppress_errors option is set
         to False, so errors will make tests fail.
 
         `suppress_errors` default value is to be `None` in which case the
         behavior default to the one of `traitlets.Application`.
 
         The default value can be set :
            - to `False` by setting 'IPYTHON_SUPPRESS_CONFIG_ERRORS' environment variable to '0', or 'false' (case insensitive).
            - to `True` by setting 'IPYTHON_SUPPRESS_CONFIG_ERRORS' environment variable to '1' or 'true' (case insensitive).
            - to `None` by setting 'IPYTHON_SUPPRESS_CONFIG_ERRORS' environment variable to '' (empty string) or leaving it unset.
 
         Any other value are invalid, and will make IPython exit with a non-zero return code.
         """
 
 
         self.log.debug("Searching path %s for config files", self.config_file_paths)
         base_config = 'ipython_config.py'
         self.log.debug("Attempting to load config file: %s" %
                        base_config)
         try:
             if suppress_errors is not None:
                 old_value = Application.raise_config_file_errors
-                Application.raise_config_file_errors = not suppress_errors
+                Application.raise_config_file_errors = not suppress_errors;
             Application.load_config_file(
                 self,
                 base_config,
                 path=self.config_file_paths
             )
         except ConfigFileNotFound:
             # ignore errors loading parent
             self.log.debug("Config file %s not found", base_config)
             pass
         if suppress_errors is not None:
             Application.raise_config_file_errors = old_value
         
         for config_file_name in self.config_files:
             if not config_file_name or config_file_name == base_config:
                 continue
             self.log.debug("Attempting to load config file: %s" %
                            self.config_file_name)
             try:
                 Application.load_config_file(
                     self,
                     config_file_name,
                     path=self.config_file_paths
                 )
             except ConfigFileNotFound:
                 # Only warn if the default config file was NOT being used.
                 if config_file_name in self.config_file_specified:
                     msg = self.log.warning
                 else:
                     msg = self.log.debug
                 msg("Config file not found, skipping: %s", config_file_name)
             except Exception:
                 # For testing purposes.
                 if not suppress_errors:
                     raise
                 self.log.warning("Error loading config file: %s" %
                               self.config_file_name, exc_info=True)
 
     def init_profile_dir(self):
         """initialize the profile dir"""
         self._in_init_profile_dir = True
         if self.profile_dir is not None:
             # already ran
             return
         if 'ProfileDir.location' not in self.config:
             # location not specified, find by profile name
             try:
                 p = ProfileDir.find_profile_dir_by_name(self.ipython_dir, self.profile, self.config)
             except ProfileDirError:
                 # not found, maybe create it (always create default profile)
                 if self.auto_create or self.profile == 'default':
                     try:
                         p = ProfileDir.create_profile_dir_by_name(self.ipython_dir, self.profile, self.config)
                     except ProfileDirError:
                         self.log.fatal("Could not create profile: %r"%self.profile)
                         self.exit(1)
                     else:
                         self.log.info("Created profile dir: %r"%p.location)
                 else:
                     self.log.fatal("Profile %r not found."%self.profile)
                     self.exit(1)
             else:
                 self.log.debug("Using existing profile dir: %r", p.location)
         else:
             location = self.config.ProfileDir.location
             # location is fully specified
             try:
                 p = ProfileDir.find_profile_dir(location, self.config)
             except ProfileDirError:
                 # not found, maybe create it
                 if self.auto_create:
                     try:
                         p = ProfileDir.create_profile_dir(location, self.config)
                     except ProfileDirError:
                         self.log.fatal("Could not create profile directory: %r"%location)
                         self.exit(1)
                     else:
                         self.log.debug("Creating new profile dir: %r"%location)
                 else:
                     self.log.fatal("Profile directory %r not found."%location)
                     self.exit(1)
             else:
                 self.log.debug("Using existing profile dir: %r", p.location)
             # if profile_dir is specified explicitly, set profile name
             dir_name = os.path.basename(p.location)
             if dir_name.startswith('profile_'):
                 self.profile = dir_name[8:]
 
         self.profile_dir = p
         self.config_file_paths.append(p.location)
         self._in_init_profile_dir = False
 
     def init_config_files(self):
         """[optionally] copy default config files into profile dir."""
         self.config_file_paths.extend(ENV_CONFIG_DIRS)
         self.config_file_paths.extend(SYSTEM_CONFIG_DIRS)
         # copy config files
         path = Path(self.builtin_profile_dir)
         if self.copy_config_files:
             src = self.profile
 
             cfg = self.config_file_name
             if path and (path / cfg).exists():
                 self.log.warning(
                     "Staging %r from %s into %r [overwrite=%s]"
                     % (cfg, src, self.profile_dir.location, self.overwrite)
                 )
                 self.profile_dir.copy_config_file(cfg, path=path, overwrite=self.overwrite)
             else:
                 self.stage_default_config_file()
         else:
             # Still stage *bundled* config files, but not generated ones
             # This is necessary for `ipython profile=sympy` to load the profile
             # on the first go
             files = path.glob("*.py")
             for fullpath in files:
                 cfg = fullpath.name
                 if self.profile_dir.copy_config_file(cfg, path=path, overwrite=False):
                     # file was copied
                     self.log.warning("Staging bundled %s from %s into %r"%(
                             cfg, self.profile, self.profile_dir.location)
                     )
 
 
     def stage_default_config_file(self):
         """auto generate default config file, and stage it into the profile."""
         s = self.generate_config_file()
         config_file = Path(self.profile_dir.location) / self.config_file_name
         if self.overwrite or not config_file.exists():
             self.log.warning("Generating default config file: %r", (config_file))
             config_file.write_text(s, encoding="utf-8")
 
     @catch_config_error
     def initialize(self, argv=None):
         # don't hook up crash handler before parsing command-line
         self.parse_command_line(argv)
         self.init_crash_handler()
         if self.subapp is not None:
             # stop here if subapp is taking over
             return
         # save a copy of CLI config to re-load after config files
         # so that it has highest priority
         cl_config = deepcopy(self.config)
         self.init_profile_dir()
         self.init_config_files()
         self.load_config_file()
         # enforce cl-opts override configfile opts:
         self.update_config(cl_config)

IPython/core/builtin_trap.py

@@ -1,89 +1,86
 """
 A context manager for managing things injected into :mod:`builtins`.
 """
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 import builtins as builtin_mod
 
 from traitlets.config.configurable import Configurable
 
 from traitlets import Instance
 
 
-class __BuiltinUndefined:
-    pass
+class __BuiltinUndefined(object): pass
 BuiltinUndefined = __BuiltinUndefined()
 
-
-class __HideBuiltin:
-    pass
+class __HideBuiltin(object): pass
 HideBuiltin = __HideBuiltin()
 
 
 class BuiltinTrap(Configurable):
 
     shell = Instance('IPython.core.interactiveshell.InteractiveShellABC',
                      allow_none=True)
 
     def __init__(self, shell=None):
         super(BuiltinTrap, self).__init__(shell=shell, config=None)
         self._orig_builtins = {}
         # We define this to track if a single BuiltinTrap is nested.
         # Only turn off the trap when the outermost call to __exit__ is made.
         self._nested_level = 0
         self.shell = shell
         # builtins we always add - if set to HideBuiltin, they will just
         # be removed instead of being replaced by something else
         self.auto_builtins = {'exit': HideBuiltin,
                               'quit': HideBuiltin,
                               'get_ipython': self.shell.get_ipython,
                               }
 
     def __enter__(self):
         if self._nested_level == 0:
             self.activate()
         self._nested_level += 1
         # I return self, so callers can use add_builtin in a with clause.
         return self
 
     def __exit__(self, type, value, traceback):
         if self._nested_level == 1:
             self.deactivate()
         self._nested_level -= 1
         # Returning False will cause exceptions to propagate
         return False
 
     def add_builtin(self, key, value):
         """Add a builtin and save the original."""
         bdict = builtin_mod.__dict__
         orig = bdict.get(key, BuiltinUndefined)
         if value is HideBuiltin:
             if orig is not BuiltinUndefined: #same as 'key in bdict'
                 self._orig_builtins[key] = orig
                 del bdict[key]
         else:
             self._orig_builtins[key] = orig
             bdict[key] = value
 
     def remove_builtin(self, key, orig):
         """Remove an added builtin and re-set the original."""
         if orig is BuiltinUndefined:
             del builtin_mod.__dict__[key]
         else:
             builtin_mod.__dict__[key] = orig
 
     def activate(self):
         """Store ipython references in the __builtin__ namespace."""
 
         add_builtin = self.add_builtin
         for name, func in self.auto_builtins.items():
             add_builtin(name, func)
 
     def deactivate(self):
         """Remove any builtins which might have been added by add_builtins, or
         restore overwritten ones to their previous values."""
         remove_builtin = self.remove_builtin
         for key, val in self._orig_builtins.items():
             remove_builtin(key, val)
         self._orig_builtins.clear()
         self._builtins_added = False

IPython/core/compilerop.py

@@ -1,213 +1,214
 """Compiler tools with improved interactive support.
 
 Provides compilation machinery similar to codeop, but with caching support so
 we can provide interactive tracebacks.
 
 Authors
 -------
 * Robert Kern
 * Fernando Perez
 * Thomas Kluyver
 """
 
 # Note: though it might be more natural to name this module 'compiler', that
 # name is in the stdlib and name collisions with the stdlib tend to produce
 # weird problems (often with third-party tools).
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2010-2011 The IPython Development Team.
 #
 #  Distributed under the terms of the BSD License.
 #
 #  The full license is in the file COPYING.txt, distributed with this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 # Stdlib imports
 import __future__
 from ast import PyCF_ONLY_AST
 import codeop
 import functools
 import hashlib
 import linecache
 import operator
+import time
 from contextlib import contextmanager
 
 #-----------------------------------------------------------------------------
 # Constants
 #-----------------------------------------------------------------------------
 
 # Roughly equal to PyCF_MASK | PyCF_MASK_OBSOLETE as defined in pythonrun.h,
 # this is used as a bitmask to extract future-related code flags.
 PyCF_MASK = functools.reduce(operator.or_,
                              (getattr(__future__, fname).compiler_flag
                               for fname in __future__.all_feature_names))
 
 #-----------------------------------------------------------------------------
 # Local utilities
 #-----------------------------------------------------------------------------
 
 def code_name(code, number=0):
     """ Compute a (probably) unique name for code for caching.
 
     This now expects code to be unicode.
     """
     hash_digest = hashlib.sha1(code.encode("utf-8")).hexdigest()
     # Include the number and 12 characters of the hash in the name.  It's
     # pretty much impossible that in a single session we'll have collisions
     # even with truncated hashes, and the full one makes tracebacks too long
     return '<ipython-input-{0}-{1}>'.format(number, hash_digest[:12])
 
 #-----------------------------------------------------------------------------
 # Classes and functions
 #-----------------------------------------------------------------------------
 
 class CachingCompiler(codeop.Compile):
     """A compiler that caches code compiled from interactive statements.
     """
 
     def __init__(self):
         codeop.Compile.__init__(self)
 
         # Caching a dictionary { filename: execution_count } for nicely
         # rendered tracebacks. The filename corresponds to the filename
         # argument used for the builtins.compile function.
         self._filename_map = {}
 
     def ast_parse(self, source, filename='<unknown>', symbol='exec'):
         """Parse code to an AST with the current compiler flags active.
 
         Arguments are exactly the same as ast.parse (in the standard library),
         and are passed to the built-in compile function."""
         return compile(source, filename, symbol, self.flags | PyCF_ONLY_AST, 1)
 
     def reset_compiler_flags(self):
         """Reset compiler flags to default state."""
         # This value is copied from codeop.Compile.__init__, so if that ever
         # changes, it will need to be updated.
         self.flags = codeop.PyCF_DONT_IMPLY_DEDENT
 
     @property
     def compiler_flags(self):
         """Flags currently active in the compilation process.
         """
         return self.flags
 
     def get_code_name(self, raw_code, transformed_code, number):
         """Compute filename given the code, and the cell number.
 
         Parameters
         ----------
         raw_code : str
             The raw cell code.
         transformed_code : str
             The executable Python source code to cache and compile.
         number : int
             A number which forms part of the code's name. Used for the execution
             counter.
 
         Returns
         -------
         The computed filename.
         """
         return code_name(transformed_code, number)
 
     def format_code_name(self, name):
         """Return a user-friendly label and name for a code block.
 
         Parameters
         ----------
         name : str
             The name for the code block returned from get_code_name
 
         Returns
         -------
         A (label, name) pair that can be used in tracebacks, or None if the default formatting should be used.
         """
         if name in self._filename_map:
             return "Cell", "In[%s]" % self._filename_map[name]
 
     def cache(self, transformed_code, number=0, raw_code=None):
         """Make a name for a block of code, and cache the code.
 
         Parameters
         ----------
         transformed_code : str
             The executable Python source code to cache and compile.
         number : int
             A number which forms part of the code's name. Used for the execution
             counter.
         raw_code : str
             The raw code before transformation, if None, set to `transformed_code`.
 
         Returns
         -------
         The name of the cached code (as a string). Pass this as the filename
         argument to compilation, so that tracebacks are correctly hooked up.
         """
         if raw_code is None:
             raw_code = transformed_code
 
         name = self.get_code_name(raw_code, transformed_code, number)
 
         # Save the execution count
         self._filename_map[name] = number
 
         # Since Python 2.5, setting mtime to `None` means the lines will
         # never be removed by `linecache.checkcache`.  This means all the
         # monkeypatching has *never* been necessary, since this code was
         # only added in 2010, at which point IPython had already stopped
         # supporting Python 2.4.
         #
         # Note that `linecache.clearcache` and `linecache.updatecache` may
         # still remove our code from the cache, but those show explicit
         # intent, and we should not try to interfere.  Normally the former
         # is never called except when out of memory, and the latter is only
         # called for lines *not* in the cache.
         entry = (
             len(transformed_code),
             None,
             [line + "\n" for line in transformed_code.splitlines()],
             name,
         )
         linecache.cache[name] = entry
         return name
 
     @contextmanager
     def extra_flags(self, flags):
         ## bits that we'll set to 1
         turn_on_bits = ~self.flags & flags
 
 
         self.flags = self.flags | flags
         try:
             yield
         finally:
             # turn off only the bits we turned on so that something like
             # __future__ that set flags stays.
             self.flags &= ~turn_on_bits
 
 
 def check_linecache_ipython(*args):
     """Deprecated since IPython 8.6.  Call linecache.checkcache() directly.
 
     It was already not necessary to call this function directly.  If no
     CachingCompiler had been created, this function would fail badly.  If
     an instance had been created, this function would've been monkeypatched
     into place.
 
     As of IPython 8.6, the monkeypatching has gone away entirely.  But there
     were still internal callers of this function, so maybe external callers
     also existed?
     """
     import warnings
 
     warnings.warn(
         "Deprecated Since IPython 8.6, Just call linecache.checkcache() directly.",
         DeprecationWarning,
         stacklevel=2,
     )
     linecache.checkcache()

IPython/core/completer.py

@@ -1,3409 +1,3410
 """Completion for IPython.
 
 This module started as fork of the rlcompleter module in the Python standard
 library.  The original enhancements made to rlcompleter have been sent
 upstream and were accepted as of Python 2.3,
 
 This module now support a wide variety of completion mechanism both available
 for normal classic Python code, as well as completer for IPython specific
 Syntax like magics.
 
 Latex and Unicode completion
 ============================
 
 IPython and compatible frontends not only can complete your code, but can help
 you to input a wide range of characters. In particular we allow you to insert
 a unicode character using the tab completion mechanism.
 
 Forward latex/unicode completion
 --------------------------------
 
 Forward completion allows you to easily type a unicode character using its latex
 name, or unicode long description. To do so type a backslash follow by the
 relevant name and press tab:
 
 
 Using latex completion:
 
 .. code::
 
     \\alpha<tab>
     α
 
 or using unicode completion:
 
 
 .. code::
 
     \\GREEK SMALL LETTER ALPHA<tab>
     α
 
 
 Only valid Python identifiers will complete. Combining characters (like arrow or
 dots) are also available, unlike latex they need to be put after the their
 counterpart that is to say, ``F\\\\vec<tab>`` is correct, not ``\\\\vec<tab>F``.
 
 Some browsers are known to display combining characters incorrectly.
 
 Backward latex completion
 -------------------------
 
 It is sometime challenging to know how to type a character, if you are using
 IPython, or any compatible frontend you can prepend backslash to the character
 and press :kbd:`Tab` to expand it to its latex form.
 
 .. code::
 
     \\α<tab>
     \\alpha
 
 
 Both forward and backward completions can be deactivated by setting the
 :std:configtrait:`Completer.backslash_combining_completions` option to
 ``False``.
 
 
 Experimental
 ============
 
 Starting with IPython 6.0, this module can make use of the Jedi library to
 generate completions both using static analysis of the code, and dynamically
 inspecting multiple namespaces. Jedi is an autocompletion and static analysis
 for Python. The APIs attached to this new mechanism is unstable and will
 raise unless use in an :any:`provisionalcompleter` context manager.
 
 You will find that the following are experimental:
 
     - :any:`provisionalcompleter`
     - :any:`IPCompleter.completions`
     - :any:`Completion`
     - :any:`rectify_completions`
 
 .. note::
 
     better name for :any:`rectify_completions` ?
 
 We welcome any feedback on these new API, and we also encourage you to try this
 module in debug mode (start IPython with ``--Completer.debug=True``) in order
 to have extra logging information if :any:`jedi` is crashing, or if current
 IPython completer pending deprecations are returning results not yet handled
 by :any:`jedi`
 
 Using Jedi for tab completion allow snippets like the following to work without
 having to execute any code:
 
    >>> myvar = ['hello', 42]
    ... myvar[1].bi<tab>
 
 Tab completion will be able to infer that ``myvar[1]`` is a real number without
 executing almost any code unlike the deprecated :any:`IPCompleter.greedy`
 option.
 
 Be sure to update :any:`jedi` to the latest stable version or to try the
 current development version to get better completions.
 
 Matchers
 ========
 
 All completions routines are implemented using unified *Matchers* API.
 The matchers API is provisional and subject to change without notice.
 
 The built-in matchers include:
 
 - :any:`IPCompleter.dict_key_matcher`:  dictionary key completions,
 - :any:`IPCompleter.magic_matcher`: completions for magics,
 - :any:`IPCompleter.unicode_name_matcher`,
   :any:`IPCompleter.fwd_unicode_matcher`
   and :any:`IPCompleter.latex_name_matcher`: see `Forward latex/unicode completion`_,
 - :any:`back_unicode_name_matcher` and :any:`back_latex_name_matcher`: see `Backward latex completion`_,
 - :any:`IPCompleter.file_matcher`: paths to files and directories,
 - :any:`IPCompleter.python_func_kw_matcher` - function keywords,
 - :any:`IPCompleter.python_matches` - globals and attributes (v1 API),
 - ``IPCompleter.jedi_matcher`` - static analysis with Jedi,
 - :any:`IPCompleter.custom_completer_matcher` - pluggable completer with a default
   implementation in :any:`InteractiveShell` which uses IPython hooks system
   (`complete_command`) with string dispatch (including regular expressions).
   Differently to other matchers, ``custom_completer_matcher`` will not suppress
   Jedi results to match behaviour in earlier IPython versions.
 
 Custom matchers can be added by appending to ``IPCompleter.custom_matchers`` list.
 
 Matcher API
 -----------
 
 Simplifying some details, the ``Matcher`` interface can described as
 
 .. code-block::
 
     MatcherAPIv1 = Callable[[str], list[str]]
     MatcherAPIv2 = Callable[[CompletionContext], SimpleMatcherResult]
 
     Matcher = MatcherAPIv1 | MatcherAPIv2
 
 The ``MatcherAPIv1`` reflects the matcher API as available prior to IPython 8.6.0
 and remains supported as a simplest way for generating completions. This is also
 currently the only API supported by the IPython hooks system `complete_command`.
 
 To distinguish between matcher versions ``matcher_api_version`` attribute is used.
 More precisely, the API allows to omit ``matcher_api_version`` for v1 Matchers,
 and requires a literal ``2`` for v2 Matchers.
 
 Once the API stabilises future versions may relax the requirement for specifying
 ``matcher_api_version`` by switching to :any:`functools.singledispatch`, therefore
 please do not rely on the presence of ``matcher_api_version`` for any purposes.
 
 Suppression of competing matchers
 ---------------------------------
 
 By default results from all matchers are combined, in the order determined by
 their priority. Matchers can request to suppress results from subsequent
 matchers by setting ``suppress`` to ``True`` in the ``MatcherResult``.
 
 When multiple matchers simultaneously request suppression, the results from of
 the matcher with higher priority will be returned.
 
 Sometimes it is desirable to suppress most but not all other matchers;
 this can be achieved by adding a set of identifiers of matchers which
 should not be suppressed to ``MatcherResult`` under ``do_not_suppress`` key.
 
 The suppression behaviour can is user-configurable via
 :std:configtrait:`IPCompleter.suppress_competing_matchers`.
 """
 
 
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 #
 # Some of this code originated from rlcompleter in the Python standard library
 # Copyright (C) 2001 Python Software Foundation, www.python.org
 
 from __future__ import annotations
 import builtins as builtin_mod
 import enum
 import glob
 import inspect
 import itertools
 import keyword
 import ast
 import os
 import re
 import string
 import sys
 import tokenize
 import time
 import unicodedata
 import uuid
 import warnings
 from ast import literal_eval
 from collections import defaultdict
 from contextlib import contextmanager
 from dataclasses import dataclass
 from functools import cached_property, partial
 from types import SimpleNamespace
 from typing import (
     Iterable,
     Iterator,
     List,
     Tuple,
     Union,
     Any,
     Sequence,
     Dict,
     Optional,
     TYPE_CHECKING,
     Set,
     Sized,
     TypeVar,
     Literal,
 )
 
 from IPython.core.guarded_eval import guarded_eval, EvaluationContext
 from IPython.core.error import TryNext
 from IPython.core.inputtransformer2 import ESC_MAGIC
 from IPython.core.latex_symbols import latex_symbols, reverse_latex_symbol
 from IPython.core.oinspect import InspectColors
 from IPython.testing.skipdoctest import skip_doctest
 from IPython.utils import generics
 from IPython.utils.decorators import sphinx_options
 from IPython.utils.dir2 import dir2, get_real_method
+from IPython.utils.docs import GENERATING_DOCUMENTATION
 from IPython.utils.path import ensure_dir_exists
 from IPython.utils.process import arg_split
 from traitlets import (
     Bool,
     Enum,
     Int,
     List as ListTrait,
     Unicode,
     Dict as DictTrait,
     Union as UnionTrait,
     observe,
 )
 from traitlets.config.configurable import Configurable
 
 import __main__
 
 from typing import cast
 
 if sys.version_info < (3, 12):
     from typing_extensions import TypedDict, NotRequired, Protocol, TypeAlias, TypeGuard
 else:
     from typing import TypedDict, NotRequired, Protocol, TypeAlias, TypeGuard
 
 
 # skip module docstests
 __skip_doctest__ = True
 
 
 try:
     import jedi
     jedi.settings.case_insensitive_completion = False
     import jedi.api.helpers
     import jedi.api.classes
     JEDI_INSTALLED = True
 except ImportError:
     JEDI_INSTALLED = False
 
 
 # -----------------------------------------------------------------------------
 # Globals
 #-----------------------------------------------------------------------------
 
 # ranges where we have most of the valid unicode names. We could be more finer
 # grained but is it worth it for performance  While unicode have character in the
 # range 0, 0x110000, we seem to have name for about 10% of those. (131808 as I
 # write this). With below range we cover them all, with a density of ~67%
 # biggest next gap we consider only adds up about 1% density and there are 600
 # gaps that would need hard coding.
 _UNICODE_RANGES = [(32, 0x323B0), (0xE0001, 0xE01F0)]
 
 # Public API
 __all__ = ["Completer", "IPCompleter"]
 
 if sys.platform == 'win32':
     PROTECTABLES = ' '
 else:
     PROTECTABLES = ' ()[]{}?=\\|;:\'#*"^&'
 
 # Protect against returning an enormous number of completions which the frontend
 # may have trouble processing.
 MATCHES_LIMIT = 500
 
 # Completion type reported when no type can be inferred.
 _UNKNOWN_TYPE = "<unknown>"
 
 # sentinel value to signal lack of a match
 not_found = object()
 
 class ProvisionalCompleterWarning(FutureWarning):
     """
     Exception raise by an experimental feature in this module.
 
     Wrap code in :any:`provisionalcompleter` context manager if you
     are certain you want to use an unstable feature.
     """
     pass
 
 warnings.filterwarnings('error', category=ProvisionalCompleterWarning)
 
 
 @skip_doctest
 @contextmanager
 def provisionalcompleter(action='ignore'):
     """
     This context manager has to be used in any place where unstable completer
     behavior and API may be called.
 
     >>> with provisionalcompleter():
     ...     completer.do_experimental_things() # works
 
     >>> completer.do_experimental_things() # raises.
 
     .. note::
 
         Unstable
 
         By using this context manager you agree that the API in use may change
         without warning, and that you won't complain if they do so.
 
         You also understand that, if the API is not to your liking, you should report
         a bug to explain your use case upstream.
 
         We'll be happy to get your feedback, feature requests, and improvements on
         any of the unstable APIs!
     """
     with warnings.catch_warnings():
         warnings.filterwarnings(action, category=ProvisionalCompleterWarning)
         yield
 
 
 def has_open_quotes(s: str) -> Union[str, bool]:
     """Return whether a string has open quotes.
 
     This simply counts whether the number of quote characters of either type in
     the string is odd.
 
     Returns
     -------
     If there is an open quote, the quote character is returned.  Else, return
     False.
     """
     # We check " first, then ', so complex cases with nested quotes will get
     # the " to take precedence.
     if s.count('"') % 2:
         return '"'
     elif s.count("'") % 2:
         return "'"
     else:
         return False
 
 
 def protect_filename(s: str, protectables: str = PROTECTABLES) -> str:
     """Escape a string to protect certain characters."""
     if set(s) & set(protectables):
         if sys.platform == "win32":
             return '"' + s + '"'
         else:
             return "".join(("\\" + c if c in protectables else c) for c in s)
     else:
         return s
 
 
 def expand_user(path:str) -> Tuple[str, bool, str]:
     """Expand ``~``-style usernames in strings.
 
     This is similar to :func:`os.path.expanduser`, but it computes and returns
     extra information that will be useful if the input was being used in
     computing completions, and you wish to return the completions with the
     original '~' instead of its expanded value.
 
     Parameters
     ----------
     path : str
         String to be expanded.  If no ~ is present, the output is the same as the
         input.
 
     Returns
     -------
     newpath : str
         Result of ~ expansion in the input path.
     tilde_expand : bool
         Whether any expansion was performed or not.
     tilde_val : str
         The value that ~ was replaced with.
     """
     # Default values
     tilde_expand = False
     tilde_val = ''
     newpath = path
 
     if path.startswith('~'):
         tilde_expand = True
         rest = len(path)-1
         newpath = os.path.expanduser(path)
         if rest:
             tilde_val = newpath[:-rest]
         else:
             tilde_val = newpath
 
     return newpath, tilde_expand, tilde_val
 
 
 def compress_user(path:str, tilde_expand:bool, tilde_val:str) -> str:
     """Does the opposite of expand_user, with its outputs.
     """
     if tilde_expand:
         return path.replace(tilde_val, '~')
     else:
         return path
 
 
 def completions_sorting_key(word):
     """key for sorting completions
 
     This does several things:
 
     - Demote any completions starting with underscores to the end
     - Insert any %magic and %%cellmagic completions in the alphabetical order
       by their name
     """
     prio1, prio2 = 0, 0
 
     if word.startswith('__'):
         prio1 = 2
     elif word.startswith('_'):
         prio1 = 1
 
     if word.endswith('='):
         prio1 = -1
 
     if word.startswith('%%'):
         # If there's another % in there, this is something else, so leave it alone
         if "%" not in word[2:]:
             word = word[2:]
             prio2 = 2
     elif word.startswith('%'):
         if "%" not in word[1:]:
             word = word[1:]
             prio2 = 1
 
     return prio1, word, prio2
 
 
 class _FakeJediCompletion:
     """
     This is a workaround to communicate to the UI that Jedi has crashed and to
     report a bug. Will be used only id :any:`IPCompleter.debug` is set to true.
 
     Added in IPython 6.0 so should likely be removed for 7.0
 
     """
 
     def __init__(self, name):
 
         self.name = name
         self.complete = name
         self.type = 'crashed'
         self.name_with_symbols = name
         self.signature = ""
         self._origin = "fake"
         self.text = "crashed"
 
     def __repr__(self):
         return '<Fake completion object jedi has crashed>'
 
 
 _JediCompletionLike = Union["jedi.api.Completion", _FakeJediCompletion]
 
 
 class Completion:
     """
     Completion object used and returned by IPython completers.
 
     .. warning::
 
         Unstable
 
         This function is unstable, API may change without warning.
         It will also raise unless use in proper context manager.
 
     This act as a middle ground :any:`Completion` object between the
     :any:`jedi.api.classes.Completion` object and the Prompt Toolkit completion
     object. While Jedi need a lot of information about evaluator and how the
     code should be ran/inspected, PromptToolkit (and other frontend) mostly
     need user facing information.
 
     - Which range should be replaced replaced by what.
     - Some metadata (like completion type), or meta information to displayed to
       the use user.
 
     For debugging purpose we can also store the origin of the completion (``jedi``,
     ``IPython.python_matches``, ``IPython.magics_matches``...).
     """
 
     __slots__ = ['start', 'end', 'text', 'type', 'signature', '_origin']
 
     def __init__(
         self,
         start: int,
         end: int,
         text: str,
         *,
         type: Optional[str] = None,
         _origin="",
         signature="",
     ) -> None:
         warnings.warn(
             "``Completion`` is a provisional API (as of IPython 6.0). "
             "It may change without warnings. "
             "Use in corresponding context manager.",
             category=ProvisionalCompleterWarning,
             stacklevel=2,
         )
 
         self.start = start
         self.end = end
         self.text = text
         self.type = type
         self.signature = signature
         self._origin = _origin
 
     def __repr__(self):
         return '<Completion start=%s end=%s text=%r type=%r, signature=%r,>' % \
                 (self.start, self.end, self.text, self.type or '?', self.signature or '?')
 
     def __eq__(self, other) -> bool:
         """
         Equality and hash do not hash the type (as some completer may not be
         able to infer the type), but are use to (partially) de-duplicate
         completion.
 
         Completely de-duplicating completion is a bit tricker that just
         comparing as it depends on surrounding text, which Completions are not
         aware of.
         """
         return self.start == other.start and \
             self.end == other.end and \
             self.text == other.text
 
     def __hash__(self):
         return hash((self.start, self.end, self.text))
 
 
 class SimpleCompletion:
     """Completion item to be included in the dictionary returned by new-style Matcher (API v2).
 
     .. warning::
 
         Provisional
 
         This class is used to describe the currently supported attributes of
         simple completion items, and any additional implementation details
         should not be relied on. Additional attributes may be included in
         future versions, and meaning of text disambiguated from the current
         dual meaning of "text to insert" and "text to used as a label".
     """
 
     __slots__ = ["text", "type"]
 
     def __init__(self, text: str, *, type: Optional[str] = None):
         self.text = text
         self.type = type
 
     def __repr__(self):
         return f"<SimpleCompletion text={self.text!r} type={self.type!r}>"
 
 
 class _MatcherResultBase(TypedDict):
     """Definition of dictionary to be returned by new-style Matcher (API v2)."""
 
     #: Suffix of the provided ``CompletionContext.token``, if not given defaults to full token.
     matched_fragment: NotRequired[str]
 
     #: Whether to suppress results from all other matchers (True), some
     #: matchers (set of identifiers) or none (False); default is False.
     suppress: NotRequired[Union[bool, Set[str]]]
 
     #: Identifiers of matchers which should NOT be suppressed when this matcher
     #: requests to suppress all other matchers; defaults to an empty set.
     do_not_suppress: NotRequired[Set[str]]
 
     #: Are completions already ordered and should be left as-is? default is False.
     ordered: NotRequired[bool]
 
 
 @sphinx_options(show_inherited_members=True, exclude_inherited_from=["dict"])
 class SimpleMatcherResult(_MatcherResultBase, TypedDict):
     """Result of new-style completion matcher."""
 
     # note: TypedDict is added again to the inheritance chain
     # in order to get __orig_bases__ for documentation
 
     #: List of candidate completions
     completions: Sequence[SimpleCompletion] | Iterator[SimpleCompletion]
 
 
 class _JediMatcherResult(_MatcherResultBase):
     """Matching result returned by Jedi (will be processed differently)"""
 
     #: list of candidate completions
     completions: Iterator[_JediCompletionLike]
 
 
 AnyMatcherCompletion = Union[_JediCompletionLike, SimpleCompletion]
 AnyCompletion = TypeVar("AnyCompletion", AnyMatcherCompletion, Completion)
 
 
 @dataclass
 class CompletionContext:
     """Completion context provided as an argument to matchers in the Matcher API v2."""
 
     # rationale: many legacy matchers relied on completer state (`self.text_until_cursor`)
     # which was not explicitly visible as an argument of the matcher, making any refactor
     # prone to errors; by explicitly passing `cursor_position` we can decouple the matchers
     # from the completer, and make substituting them in sub-classes easier.
 
     #: Relevant fragment of code directly preceding the cursor.
     #: The extraction of token is implemented via splitter heuristic
     #: (following readline behaviour for legacy reasons), which is user configurable
     #: (by switching the greedy mode).
     token: str
 
     #: The full available content of the editor or buffer
     full_text: str
 
     #: Cursor position in the line (the same for ``full_text`` and ``text``).
     cursor_position: int
 
     #: Cursor line in ``full_text``.
     cursor_line: int
 
     #: The maximum number of completions that will be used downstream.
     #: Matchers can use this information to abort early.
     #: The built-in Jedi matcher is currently excepted from this limit.
     # If not given, return all possible completions.
     limit: Optional[int]
 
     @cached_property
     def text_until_cursor(self) -> str:
         return self.line_with_cursor[: self.cursor_position]
 
     @cached_property
     def line_with_cursor(self) -> str:
         return self.full_text.split("\n")[self.cursor_line]
 
 
 #: Matcher results for API v2.
 MatcherResult = Union[SimpleMatcherResult, _JediMatcherResult]
 
 
 class _MatcherAPIv1Base(Protocol):
     def __call__(self, text: str) -> List[str]:
         """Call signature."""
         ...
 
     #: Used to construct the default matcher identifier
     __qualname__: str
 
 
 class _MatcherAPIv1Total(_MatcherAPIv1Base, Protocol):
     #: API version
     matcher_api_version: Optional[Literal[1]]
 
     def __call__(self, text: str) -> List[str]:
         """Call signature."""
         ...
 
 
 #: Protocol describing Matcher API v1.
 MatcherAPIv1: TypeAlias = Union[_MatcherAPIv1Base, _MatcherAPIv1Total]
 
 
 class MatcherAPIv2(Protocol):
     """Protocol describing Matcher API v2."""
 
     #: API version
     matcher_api_version: Literal[2] = 2
 
     def __call__(self, context: CompletionContext) -> MatcherResult:
         """Call signature."""
         ...
 
     #: Used to construct the default matcher identifier
     __qualname__: str
 
 
 Matcher: TypeAlias = Union[MatcherAPIv1, MatcherAPIv2]
 
 
 def _is_matcher_v1(matcher: Matcher) -> TypeGuard[MatcherAPIv1]:
     api_version = _get_matcher_api_version(matcher)
     return api_version == 1
 
 
 def _is_matcher_v2(matcher: Matcher) -> TypeGuard[MatcherAPIv2]:
     api_version = _get_matcher_api_version(matcher)
     return api_version == 2
 
 
 def _is_sizable(value: Any) -> TypeGuard[Sized]:
     """Determines whether objects is sizable"""
     return hasattr(value, "__len__")
 
 
 def _is_iterator(value: Any) -> TypeGuard[Iterator]:
     """Determines whether objects is sizable"""
     return hasattr(value, "__next__")
 
 
 def has_any_completions(result: MatcherResult) -> bool:
     """Check if any result includes any completions."""
     completions = result["completions"]
     if _is_sizable(completions):
         return len(completions) != 0
     if _is_iterator(completions):
         try:
             old_iterator = completions
             first = next(old_iterator)
             result["completions"] = cast(
                 Iterator[SimpleCompletion],
                 itertools.chain([first], old_iterator),
             )
             return True
         except StopIteration:
             return False
     raise ValueError(
         "Completions returned by matcher need to be an Iterator or a Sizable"
     )
 
 
 def completion_matcher(
     *,
     priority: Optional[float] = None,
     identifier: Optional[str] = None,
     api_version: int = 1,
 ) -> Callable[[Matcher], Matcher]:
     """Adds attributes describing the matcher.
 
     Parameters
     ----------
     priority : Optional[float]
         The priority of the matcher, determines the order of execution of matchers.
         Higher priority means that the matcher will be executed first. Defaults to 0.
     identifier : Optional[str]
         identifier of the matcher allowing users to modify the behaviour via traitlets,
         and also used to for debugging (will be passed as ``origin`` with the completions).
 
         Defaults to matcher function's ``__qualname__`` (for example,
         ``IPCompleter.file_matcher`` for the built-in matched defined
         as a ``file_matcher`` method of the ``IPCompleter`` class).
     api_version: Optional[int]
         version of the Matcher API used by this matcher.
         Currently supported values are 1 and 2.
         Defaults to 1.
     """
 
     def wrapper(func: Matcher):
         func.matcher_priority = priority or 0  # type: ignore
         func.matcher_identifier = identifier or func.__qualname__  # type: ignore
         func.matcher_api_version = api_version  # type: ignore
         if TYPE_CHECKING:
             if api_version == 1:
                 func = cast(MatcherAPIv1, func)
             elif api_version == 2:
                 func = cast(MatcherAPIv2, func)
         return func
 
     return wrapper
 
 
 def _get_matcher_priority(matcher: Matcher):
     return getattr(matcher, "matcher_priority", 0)
 
 
 def _get_matcher_id(matcher: Matcher):
     return getattr(matcher, "matcher_identifier", matcher.__qualname__)
 
 
 def _get_matcher_api_version(matcher):
     return getattr(matcher, "matcher_api_version", 1)
 
 
 context_matcher = partial(completion_matcher, api_version=2)
 
 
 _IC = Iterable[Completion]
 
 
 def _deduplicate_completions(text: str, completions: _IC)-> _IC:
     """
     Deduplicate a set of completions.
 
     .. warning::
 
         Unstable
 
         This function is unstable, API may change without warning.
 
     Parameters
     ----------
     text : str
         text that should be completed.
     completions : Iterator[Completion]
         iterator over the completions to deduplicate
 
     Yields
     ------
     `Completions` objects
     Completions coming from multiple sources, may be different but end up having
     the same effect when applied to ``text``. If this is the case, this will
     consider completions as equal and only emit the first encountered.
     Not folded in `completions()` yet for debugging purpose, and to detect when
     the IPython completer does return things that Jedi does not, but should be
     at some point.
     """
     completions = list(completions)
     if not completions:
         return
 
     new_start = min(c.start for c in completions)
     new_end = max(c.end for c in completions)
 
     seen = set()
     for c in completions:
         new_text = text[new_start:c.start] + c.text + text[c.end:new_end]
         if new_text not in seen:
             yield c
             seen.add(new_text)
 
 
 def rectify_completions(text: str, completions: _IC, *, _debug: bool = False) -> _IC:
     """
     Rectify a set of completions to all have the same ``start`` and ``end``
 
     .. warning::
 
         Unstable
 
         This function is unstable, API may change without warning.
         It will also raise unless use in proper context manager.
 
     Parameters
     ----------
     text : str
         text that should be completed.
     completions : Iterator[Completion]
         iterator over the completions to rectify
     _debug : bool
         Log failed completion
 
     Notes
     -----
     :any:`jedi.api.classes.Completion` s returned by Jedi may not have the same start and end, though
     the Jupyter Protocol requires them to behave like so. This will readjust
     the completion to have the same ``start`` and ``end`` by padding both
     extremities with surrounding text.
 
     During stabilisation should support a ``_debug`` option to log which
     completion are return by the IPython completer and not found in Jedi in
     order to make upstream bug report.
     """
     warnings.warn("`rectify_completions` is a provisional API (as of IPython 6.0). "
                  "It may change without warnings. "
                  "Use in corresponding context manager.",
                   category=ProvisionalCompleterWarning, stacklevel=2)
 
     completions = list(completions)
     if not completions:
         return
     starts = (c.start for c in completions)
     ends = (c.end for c in completions)
 
     new_start = min(starts)
     new_end = max(ends)
 
     seen_jedi = set()
     seen_python_matches = set()
     for c in completions:
         new_text = text[new_start:c.start] + c.text + text[c.end:new_end]
         if c._origin == 'jedi':
             seen_jedi.add(new_text)
         elif c._origin == "IPCompleter.python_matcher":
             seen_python_matches.add(new_text)
         yield Completion(new_start, new_end, new_text, type=c.type, _origin=c._origin, signature=c.signature)
     diff = seen_python_matches.difference(seen_jedi)
     if diff and _debug:
         print('IPython.python matches have extras:', diff)
 
 
 if sys.platform == 'win32':
     DELIMS = ' \t\n`!@#$^&*()=+[{]}|;\'",<>?'
 else:
     DELIMS = ' \t\n`!@#$^&*()=+[{]}\\|;:\'",<>?'
 
 GREEDY_DELIMS = ' =\r\n'
 
 
 class CompletionSplitter(object):
     """An object to split an input line in a manner similar to readline.
 
     By having our own implementation, we can expose readline-like completion in
     a uniform manner to all frontends.  This object only needs to be given the
     line of text to be split and the cursor position on said line, and it
     returns the 'word' to be completed on at the cursor after splitting the
     entire line.
 
     What characters are used as splitting delimiters can be controlled by
     setting the ``delims`` attribute (this is a property that internally
     automatically builds the necessary regular expression)"""
 
     # Private interface
 
     # A string of delimiter characters.  The default value makes sense for
     # IPython's most typical usage patterns.
     _delims = DELIMS
 
     # The expression (a normal string) to be compiled into a regular expression
     # for actual splitting.  We store it as an attribute mostly for ease of
     # debugging, since this type of code can be so tricky to debug.
     _delim_expr = None
 
     # The regular expression that does the actual splitting
     _delim_re = None
 
     def __init__(self, delims=None):
         delims = CompletionSplitter._delims if delims is None else delims
         self.delims = delims
 
     @property
     def delims(self):
         """Return the string of delimiter characters."""
         return self._delims
 
     @delims.setter
     def delims(self, delims):
         """Set the delimiters for line splitting."""
         expr = '[' + ''.join('\\'+ c for c in delims) + ']'
         self._delim_re = re.compile(expr)
         self._delims = delims
         self._delim_expr = expr
 
     def split_line(self, line, cursor_pos=None):
         """Split a line of text with a cursor at the given position.
         """
         cut_line = line if cursor_pos is None else line[:cursor_pos]
         return self._delim_re.split(cut_line)[-1]
 
 
 
 class Completer(Configurable):
 
     greedy = Bool(
         False,
         help="""Activate greedy completion.
 
         .. deprecated:: 8.8
             Use :std:configtrait:`Completer.evaluation` and :std:configtrait:`Completer.auto_close_dict_keys` instead.
 
         When enabled in IPython 8.8 or newer, changes configuration as follows:
 
         - ``Completer.evaluation = 'unsafe'``
         - ``Completer.auto_close_dict_keys = True``
         """,
     ).tag(config=True)
 
     evaluation = Enum(
         ("forbidden", "minimal", "limited", "unsafe", "dangerous"),
         default_value="limited",
         help="""Policy for code evaluation under completion.
 
         Successive options allow to enable more eager evaluation for better
         completion suggestions, including for nested dictionaries, nested lists,
         or even results of function calls.
         Setting ``unsafe`` or higher can lead to evaluation of arbitrary user
         code on :kbd:`Tab` with potentially unwanted or dangerous side effects.
 
         Allowed values are:
 
         - ``forbidden``: no evaluation of code is permitted,
         - ``minimal``: evaluation of literals and access to built-in namespace;
           no item/attribute evaluationm no access to locals/globals,
           no evaluation of any operations or comparisons.
         - ``limited``: access to all namespaces, evaluation of hard-coded methods
           (for example: :any:`dict.keys`, :any:`object.__getattr__`,
           :any:`object.__getitem__`) on allow-listed objects (for example:
           :any:`dict`, :any:`list`, :any:`tuple`, ``pandas.Series``),
         - ``unsafe``: evaluation of all methods and function calls but not of
           syntax with side-effects like `del x`,
         - ``dangerous``: completely arbitrary evaluation.
         """,
     ).tag(config=True)
 
     use_jedi = Bool(default_value=JEDI_INSTALLED,
                     help="Experimental: Use Jedi to generate autocompletions. "
                     "Default to True if jedi is installed.").tag(config=True)
 
     jedi_compute_type_timeout = Int(default_value=400,
         help="""Experimental: restrict time (in milliseconds) during which Jedi can compute types.
         Set to 0 to stop computing types. Non-zero value lower than 100ms may hurt
         performance by preventing jedi to build its cache.
         """).tag(config=True)
 
     debug = Bool(default_value=False,
                  help='Enable debug for the Completer. Mostly print extra '
                       'information for experimental jedi integration.')\
                       .tag(config=True)
 
     backslash_combining_completions = Bool(True,
         help="Enable unicode completions, e.g. \\alpha<tab> . "
              "Includes completion of latex commands, unicode names, and expanding "
              "unicode characters back to latex commands.").tag(config=True)
 
     auto_close_dict_keys = Bool(
         False,
         help="""
         Enable auto-closing dictionary keys.
 
         When enabled string keys will be suffixed with a final quote
         (matching the opening quote), tuple keys will also receive a
         separating comma if needed, and keys which are final will
         receive a closing bracket (``]``).
         """,
     ).tag(config=True)
 
     def __init__(self, namespace=None, global_namespace=None, **kwargs):
         """Create a new completer for the command line.
 
         Completer(namespace=ns, global_namespace=ns2) -> completer instance.
 
         If unspecified, the default namespace where completions are performed
         is __main__ (technically, __main__.__dict__). Namespaces should be
         given as dictionaries.
 
         An optional second namespace can be given.  This allows the completer
         to handle cases where both the local and global scopes need to be
         distinguished.
         """
 
         # Don't bind to namespace quite yet, but flag whether the user wants a
         # specific namespace or to use __main__.__dict__. This will allow us
         # to bind to __main__.__dict__ at completion time, not now.
         if namespace is None:
             self.use_main_ns = True
         else:
             self.use_main_ns = False
             self.namespace = namespace
 
         # The global namespace, if given, can be bound directly
         if global_namespace is None:
             self.global_namespace = {}
         else:
             self.global_namespace = global_namespace
 
         self.custom_matchers = []
 
         super(Completer, self).__init__(**kwargs)
 
     def complete(self, text, state):
         """Return the next possible completion for 'text'.
 
         This is called successively with state == 0, 1, 2, ... until it
         returns None.  The completion should begin with 'text'.
 
         """
         if self.use_main_ns:
             self.namespace = __main__.__dict__
 
         if state == 0:
             if "." in text:
                 self.matches = self.attr_matches(text)
             else:
                 self.matches = self.global_matches(text)
         try:
             return self.matches[state]
         except IndexError:
             return None
 
     def global_matches(self, text):
         """Compute matches when text is a simple name.
 
         Return a list of all keywords, built-in functions and names currently
         defined in self.namespace or self.global_namespace that match.
 
         """
         matches = []
         match_append = matches.append
         n = len(text)
         for lst in [
             keyword.kwlist,
             builtin_mod.__dict__.keys(),
             list(self.namespace.keys()),
             list(self.global_namespace.keys()),
         ]:
             for word in lst:
                 if word[:n] == text and word != "__builtins__":
                     match_append(word)
 
         snake_case_re = re.compile(r"[^_]+(_[^_]+)+?\Z")
         for lst in [list(self.namespace.keys()), list(self.global_namespace.keys())]:
             shortened = {
                 "_".join([sub[0] for sub in word.split("_")]): word
                 for word in lst
                 if snake_case_re.match(word)
             }
             for word in shortened.keys():
                 if word[:n] == text and word != "__builtins__":
                     match_append(shortened[word])
         return matches
 
     def attr_matches(self, text):
         """Compute matches when text contains a dot.
 
         Assuming the text is of the form NAME.NAME....[NAME], and is
         evaluatable in self.namespace or self.global_namespace, it will be
         evaluated and its attributes (as revealed by dir()) are used as
         possible completions.  (For class instances, class members are
         also considered.)
 
         WARNING: this can still invoke arbitrary C code, if an object
         with a __getattr__ hook is evaluated.
 
         """
         return self._attr_matches(text)[0]
 
     # we simple attribute matching with normal identifiers.
     _ATTR_MATCH_RE = re.compile(r"(.+)\.(\w*)$")
 
     def _attr_matches(
         self, text: str, include_prefix: bool = True
     ) -> Tuple[Sequence[str], str]:
         m2 = self._ATTR_MATCH_RE.match(self.line_buffer)
         if not m2:
             return [], ""
         expr, attr = m2.group(1, 2)
 
         obj = self._evaluate_expr(expr)
 
         if obj is not_found:
             return [], ""
 
         if self.limit_to__all__ and hasattr(obj, '__all__'):
             words = get__all__entries(obj)
         else:
             words = dir2(obj)
 
         try:
             words = generics.complete_object(obj, words)
         except TryNext:
             pass
         except AssertionError:
             raise
         except Exception:
             # Silence errors from completion function
             pass
         # Build match list to return
         n = len(attr)
 
         # Note: ideally we would just return words here and the prefix
         # reconciliator would know that we intend to append to rather than
         # replace the input text; this requires refactoring to return range
         # which ought to be replaced (as does jedi).
         if include_prefix:
             tokens = _parse_tokens(expr)
             rev_tokens = reversed(tokens)
             skip_over = {tokenize.ENDMARKER, tokenize.NEWLINE}
             name_turn = True
 
             parts = []
             for token in rev_tokens:
                 if token.type in skip_over:
                     continue
                 if token.type == tokenize.NAME and name_turn:
                     parts.append(token.string)
                     name_turn = False
                 elif (
                     token.type == tokenize.OP and token.string == "." and not name_turn
                 ):
                     parts.append(token.string)
                     name_turn = True
                 else:
                     # short-circuit if not empty nor name token
                     break
 
             prefix_after_space = "".join(reversed(parts))
         else:
             prefix_after_space = ""
 
         return (
             ["%s.%s" % (prefix_after_space, w) for w in words if w[:n] == attr],
             "." + attr,
         )
 
     def _trim_expr(self, code: str) -> str:
         """
         Trim the code until it is a valid expression and not a tuple;
 
         return the trimmed expression for guarded_eval.
         """
         while code:
             code = code[1:]
             try:
                 res = ast.parse(code)
             except SyntaxError:
                 continue
 
             assert res is not None
             if len(res.body) != 1:
                 continue
             expr = res.body[0].value
             if isinstance(expr, ast.Tuple) and not code[-1] == ")":
                 # we skip implicit tuple, like when trimming `fun(a,b`<completion>
                 # as `a,b` would be a tuple, and we actually expect to get only `b`
                 continue
             return code
         return ""
 
     def _evaluate_expr(self, expr):
         obj = not_found
         done = False
         while not done and expr:
             try:
                 obj = guarded_eval(
                     expr,
                     EvaluationContext(
                         globals=self.global_namespace,
                         locals=self.namespace,
                         evaluation=self.evaluation,
                     ),
                 )
                 done = True
             except Exception as e:
                 if self.debug:
                     print("Evaluation exception", e)
                 # trim the expression to remove any invalid prefix
                 # e.g. user starts `(d[`, so we get `expr = '(d'`,
                 # where parenthesis is not closed.
                 # TODO: make this faster by reusing parts of the computation?
                 expr = self._trim_expr(expr)
         return obj
 
 def get__all__entries(obj):
     """returns the strings in the __all__ attribute"""
     try:
         words = getattr(obj, '__all__')
     except Exception:
         return []
 
     return [w for w in words if isinstance(w, str)]
 
 
 class _DictKeyState(enum.Flag):
     """Represent state of the key match in context of other possible matches.
 
     - given `d1 = {'a': 1}` completion on `d1['<tab>` will yield `{'a': END_OF_ITEM}` as there is no tuple.
     - given `d2 = {('a', 'b'): 1}`: `d2['a', '<tab>` will yield `{'b': END_OF_TUPLE}` as there is no tuple members to add beyond `'b'`.
     - given `d3 = {('a', 'b'): 1}`: `d3['<tab>` will yield `{'a': IN_TUPLE}` as `'a'` can be added.
     - given `d4 = {'a': 1, ('a', 'b'): 2}`: `d4['<tab>` will yield `{'a': END_OF_ITEM & END_OF_TUPLE}`
     """
 
     BASELINE = 0
     END_OF_ITEM = enum.auto()
     END_OF_TUPLE = enum.auto()
     IN_TUPLE = enum.auto()
 
 
 def _parse_tokens(c):
     """Parse tokens even if there is an error."""
     tokens = []
     token_generator = tokenize.generate_tokens(iter(c.splitlines()).__next__)
     while True:
         try:
             tokens.append(next(token_generator))
         except tokenize.TokenError:
             return tokens
         except StopIteration:
             return tokens
 
 
 def _match_number_in_dict_key_prefix(prefix: str) -> Union[str, None]:
     """Match any valid Python numeric literal in a prefix of dictionary keys.
 
     References:
     - https://docs.python.org/3/reference/lexical_analysis.html#numeric-literals
     - https://docs.python.org/3/library/tokenize.html
     """
     if prefix[-1].isspace():
         # if user typed a space we do not have anything to complete
         # even if there was a valid number token before
         return None
     tokens = _parse_tokens(prefix)
     rev_tokens = reversed(tokens)
     skip_over = {tokenize.ENDMARKER, tokenize.NEWLINE}
     number = None
     for token in rev_tokens:
         if token.type in skip_over:
             continue
         if number is None:
             if token.type == tokenize.NUMBER:
                 number = token.string
                 continue
             else:
                 # we did not match a number
                 return None
         if token.type == tokenize.OP:
             if token.string == ",":
                 break
             if token.string in {"+", "-"}:
                 number = token.string + number
         else:
             return None
     return number
 
 
 _INT_FORMATS = {
     "0b": bin,
     "0o": oct,
     "0x": hex,
 }
 
 
 def match_dict_keys(
     keys: List[Union[str, bytes, Tuple[Union[str, bytes], ...]]],
     prefix: str,
     delims: str,
     extra_prefix: Optional[Tuple[Union[str, bytes], ...]] = None,
 ) -> Tuple[str, int, Dict[str, _DictKeyState]]:
     """Used by dict_key_matches, matching the prefix to a list of keys
 
     Parameters
     ----------
     keys
         list of keys in dictionary currently being completed.
     prefix
         Part of the text already typed by the user. E.g. `mydict[b'fo`
     delims
         String of delimiters to consider when finding the current key.
     extra_prefix : optional
         Part of the text already typed in multi-key index cases. E.g. for
         `mydict['foo', "bar", 'b`, this would be `('foo', 'bar')`.
 
     Returns
     -------
     A tuple of three elements: ``quote``, ``token_start``, ``matched``, with
     ``quote`` being the quote that need to be used to close current string.
     ``token_start`` the position where the replacement should start occurring,
     ``matches`` a dictionary of replacement/completion keys on keys and values
         indicating whether the state.
     """
     prefix_tuple = extra_prefix if extra_prefix else ()
 
     prefix_tuple_size = sum(
         [
             # for pandas, do not count slices as taking space
             not isinstance(k, slice)
             for k in prefix_tuple
         ]
     )
     text_serializable_types = (str, bytes, int, float, slice)
 
     def filter_prefix_tuple(key):
         # Reject too short keys
         if len(key) <= prefix_tuple_size:
             return False
         # Reject keys which cannot be serialised to text
         for k in key:
             if not isinstance(k, text_serializable_types):
                 return False
         # Reject keys that do not match the prefix
         for k, pt in zip(key, prefix_tuple):
             if k != pt and not isinstance(pt, slice):
                 return False
         # All checks passed!
         return True
 
     filtered_key_is_final: Dict[Union[str, bytes, int, float], _DictKeyState] = (
         defaultdict(lambda: _DictKeyState.BASELINE)
     )
 
     for k in keys:
         # If at least one of the matches is not final, mark as undetermined.
         # This can happen with `d = {111: 'b', (111, 222): 'a'}` where
         # `111` appears final on first match but is not final on the second.
 
         if isinstance(k, tuple):
             if filter_prefix_tuple(k):
                 key_fragment = k[prefix_tuple_size]
                 filtered_key_is_final[key_fragment] |= (
                     _DictKeyState.END_OF_TUPLE
                     if len(k) == prefix_tuple_size + 1
                     else _DictKeyState.IN_TUPLE
                 )
         elif prefix_tuple_size > 0:
             # we are completing a tuple but this key is not a tuple,
             # so we should ignore it
             pass
         else:
             if isinstance(k, text_serializable_types):
                 filtered_key_is_final[k] |= _DictKeyState.END_OF_ITEM
 
     filtered_keys = filtered_key_is_final.keys()
 
     if not prefix:
         return "", 0, {repr(k): v for k, v in filtered_key_is_final.items()}
 
     quote_match = re.search("(?:\"|')", prefix)
     is_user_prefix_numeric = False
 
     if quote_match:
         quote = quote_match.group()
         valid_prefix = prefix + quote
         try:
             prefix_str = literal_eval(valid_prefix)
         except Exception:
             return "", 0, {}
     else:
         # If it does not look like a string, let's assume
         # we are dealing with a number or variable.
         number_match = _match_number_in_dict_key_prefix(prefix)
 
         # We do not want the key matcher to suggest variable names so we yield:
         if number_match is None:
             # The alternative would be to assume that user forgort the quote
             # and if the substring matches, suggest adding it at the start.
             return "", 0, {}
 
         prefix_str = number_match
         is_user_prefix_numeric = True
         quote = ""
 
     pattern = '[^' + ''.join('\\' + c for c in delims) + ']*$'
     token_match = re.search(pattern, prefix, re.UNICODE)
     assert token_match is not None # silence mypy
     token_start = token_match.start()
     token_prefix = token_match.group()
 
     matched: Dict[str, _DictKeyState] = {}
 
     str_key: Union[str, bytes]
 
     for key in filtered_keys:
         if isinstance(key, (int, float)):
             # User typed a number but this key is not a number.
             if not is_user_prefix_numeric:
                 continue
             str_key = str(key)
             if isinstance(key, int):
                 int_base = prefix_str[:2].lower()
                 # if user typed integer using binary/oct/hex notation:
                 if int_base in _INT_FORMATS:
                     int_format = _INT_FORMATS[int_base]
                     str_key = int_format(key)
         else:
             # User typed a string but this key is a number.
             if is_user_prefix_numeric:
                 continue
             str_key = key
         try:
             if not str_key.startswith(prefix_str):
                 continue
         except (AttributeError, TypeError, UnicodeError):
             # Python 3+ TypeError on b'a'.startswith('a') or vice-versa
             continue
 
         # reformat remainder of key to begin with prefix
         rem = str_key[len(prefix_str) :]
         # force repr wrapped in '
         rem_repr = repr(rem + '"') if isinstance(rem, str) else repr(rem + b'"')
         rem_repr = rem_repr[1 + rem_repr.index("'"):-2]
         if quote == '"':
             # The entered prefix is quoted with ",
             # but the match is quoted with '.
             # A contained " hence needs escaping for comparison:
             rem_repr = rem_repr.replace('"', '\\"')
 
         # then reinsert prefix from start of token
         match = "%s%s" % (token_prefix, rem_repr)
 
         matched[match] = filtered_key_is_final[key]
     return quote, token_start, matched
 
 
 def cursor_to_position(text:str, line:int, column:int)->int:
     """
     Convert the (line,column) position of the cursor in text to an offset in a
     string.
 
     Parameters
     ----------
     text : str
         The text in which to calculate the cursor offset
     line : int
         Line of the cursor; 0-indexed
     column : int
         Column of the cursor 0-indexed
 
     Returns
     -------
     Position of the cursor in ``text``, 0-indexed.
 
     See Also
     --------
     position_to_cursor : reciprocal of this function
 
     """
     lines = text.split('\n')
     assert line <= len(lines), '{} <= {}'.format(str(line), str(len(lines)))
 
     return sum(len(line) + 1 for line in lines[:line]) + column
 
 def position_to_cursor(text:str, offset:int)->Tuple[int, int]:
     """
     Convert the position of the cursor in text (0 indexed) to a line
     number(0-indexed) and a column number (0-indexed) pair
 
     Position should be a valid position in ``text``.
 
     Parameters
     ----------
     text : str
         The text in which to calculate the cursor offset
     offset : int
         Position of the cursor in ``text``, 0-indexed.
 
     Returns
     -------
     (line, column) : (int, int)
         Line of the cursor; 0-indexed, column of the cursor 0-indexed
 
     See Also
     --------
     cursor_to_position : reciprocal of this function
 
     """
 
     assert 0 <= offset <= len(text) , "0 <= %s <= %s" % (offset , len(text))
 
     before = text[:offset]
     blines = before.split('\n')  # ! splitnes trim trailing \n
     line = before.count('\n')
     col = len(blines[-1])
     return line, col
 
 
 def _safe_isinstance(obj, module, class_name, *attrs):
     """Checks if obj is an instance of module.class_name if loaded
     """
     if module in sys.modules:
         m = sys.modules[module]
         for attr in [class_name, *attrs]:
             m = getattr(m, attr)
         return isinstance(obj, m)
 
 
 @context_matcher()
 def back_unicode_name_matcher(context: CompletionContext):
     """Match Unicode characters back to Unicode name
 
     Same as :any:`back_unicode_name_matches`, but adopted to new Matcher API.
     """
     fragment, matches = back_unicode_name_matches(context.text_until_cursor)
     return _convert_matcher_v1_result_to_v2(
         matches, type="unicode", fragment=fragment, suppress_if_matches=True
     )
 
 
 def back_unicode_name_matches(text: str) -> Tuple[str, Sequence[str]]:
     """Match Unicode characters back to Unicode name
 
     This does  ``☃`` -> ``\\snowman``
 
     Note that snowman is not a valid python3 combining character but will be expanded.
     Though it will not recombine back to the snowman character by the completion machinery.
 
     This will not either back-complete standard sequences like \\n, \\b ...
 
     .. deprecated:: 8.6
         You can use :meth:`back_unicode_name_matcher` instead.
 
     Returns
     =======
 
     Return a tuple with two elements:
 
     - The Unicode character that was matched (preceded with a backslash), or
         empty string,
     - a sequence (of 1), name for the match Unicode character, preceded by
         backslash, or empty if no match.
     """
     if len(text)<2:
         return '', ()
     maybe_slash = text[-2]
     if maybe_slash != '\\':
         return '', ()
 
     char = text[-1]
     # no expand on quote for completion in strings.
     # nor backcomplete standard ascii keys
     if char in string.ascii_letters or char in ('"',"'"):
         return '', ()
     try :
         unic = unicodedata.name(char)
         return '\\'+char,('\\'+unic,)
     except KeyError:
         pass
     return '', ()
 
 
 @context_matcher()
 def back_latex_name_matcher(context: CompletionContext):
     """Match latex characters back to unicode name
 
     Same as :any:`back_latex_name_matches`, but adopted to new Matcher API.
     """
     fragment, matches = back_latex_name_matches(context.text_until_cursor)
     return _convert_matcher_v1_result_to_v2(
         matches, type="latex", fragment=fragment, suppress_if_matches=True
     )
 
 
 def back_latex_name_matches(text: str) -> Tuple[str, Sequence[str]]:
     """Match latex characters back to unicode name
 
     This does ``\\ℵ`` -> ``\\aleph``
 
     .. deprecated:: 8.6
         You can use :meth:`back_latex_name_matcher` instead.
     """
     if len(text)<2:
         return '', ()
     maybe_slash = text[-2]
     if maybe_slash != '\\':
         return '', ()
 
 
     char = text[-1]
     # no expand on quote for completion in strings.
     # nor backcomplete standard ascii keys
     if char in string.ascii_letters or char in ('"',"'"):
         return '', ()
     try :
         latex = reverse_latex_symbol[char]
         # '\\' replace the \ as well
         return '\\'+char,[latex]
     except KeyError:
         pass
     return '', ()
 
 
 def _formatparamchildren(parameter) -> str:
     """
     Get parameter name and value from Jedi Private API
 
     Jedi does not expose a simple way to get `param=value` from its API.
 
     Parameters
     ----------
     parameter
         Jedi's function `Param`
 
     Returns
     -------
     A string like 'a', 'b=1', '*args', '**kwargs'
 
     """
     description = parameter.description
     if not description.startswith('param '):
         raise ValueError('Jedi function parameter description have change format.'
                          'Expected "param ...", found %r".' % description)
     return description[6:]
 
 def _make_signature(completion)-> str:
     """
     Make the signature from a jedi completion
 
     Parameters
     ----------
     completion : jedi.Completion
         object does not complete a function type
 
     Returns
     -------
     a string consisting of the function signature, with the parenthesis but
     without the function name. example:
     `(a, *args, b=1, **kwargs)`
 
     """
 
     # it looks like this might work on jedi 0.17
     if hasattr(completion, 'get_signatures'):
         signatures = completion.get_signatures()
         if not signatures:
             return  '(?)'
 
         c0 = completion.get_signatures()[0]
         return '('+c0.to_string().split('(', maxsplit=1)[1]
 
     return '(%s)'% ', '.join([f for f in (_formatparamchildren(p) for signature in completion.get_signatures()
                                           for p in signature.defined_names()) if f])
 
 
 _CompleteResult = Dict[str, MatcherResult]
 
 
 DICT_MATCHER_REGEX = re.compile(
     r"""(?x)
 (  # match dict-referring - or any get item object - expression
     .+
 )
 \[   # open bracket
 \s*  # and optional whitespace
 # Capture any number of serializable objects (e.g. "a", "b", 'c')
 # and slices
 ((?:(?:
     (?: # closed string
         [uUbB]?  # string prefix (r not handled)
         (?:
             '(?:[^']|(?<!\\)\\')*'
         |
             "(?:[^"]|(?<!\\)\\")*"
         )
     )
     |
         # capture integers and slices
         (?:[-+]?\d+)?(?::(?:[-+]?\d+)?){0,2}
     |
         # integer in bin/hex/oct notation
         0[bBxXoO]_?(?:\w|\d)+
     )
     \s*,\s*
 )*)
 ((?:
     (?: # unclosed string
         [uUbB]?  # string prefix (r not handled)
         (?:
             '(?:[^']|(?<!\\)\\')*
             |
             "(?:[^"]|(?<!\\)\\")*
         )
     )
     |
         # unfinished integer
         (?:[-+]?\d+)
     |
         # integer in bin/hex/oct notation
         0[bBxXoO]_?(?:\w|\d)+
     )
 )?
 $
 """
 )
 
 
 def _convert_matcher_v1_result_to_v2(
     matches: Sequence[str],
     type: str,
     fragment: Optional[str] = None,
     suppress_if_matches: bool = False,
 ) -> SimpleMatcherResult:
     """Utility to help with transition"""
     result = {
         "completions": [SimpleCompletion(text=match, type=type) for match in matches],
         "suppress": (True if matches else False) if suppress_if_matches else False,
     }
     if fragment is not None:
         result["matched_fragment"] = fragment
     return cast(SimpleMatcherResult, result)
 
 
 class IPCompleter(Completer):
     """Extension of the completer class with IPython-specific features"""
 
     @observe('greedy')
     def _greedy_changed(self, change):
         """update the splitter and readline delims when greedy is changed"""
         if change["new"]:
             self.evaluation = "unsafe"
             self.auto_close_dict_keys = True
             self.splitter.delims = GREEDY_DELIMS
         else:
             self.evaluation = "limited"
             self.auto_close_dict_keys = False
             self.splitter.delims = DELIMS
 
     dict_keys_only = Bool(
         False,
         help="""
         Whether to show dict key matches only.
 
         (disables all matchers except for `IPCompleter.dict_key_matcher`).
         """,
     )
 
     suppress_competing_matchers = UnionTrait(
         [Bool(allow_none=True), DictTrait(Bool(None, allow_none=True))],
         default_value=None,
         help="""
         Whether to suppress completions from other *Matchers*.
 
         When set to ``None`` (default) the matchers will attempt to auto-detect
         whether suppression of other matchers is desirable. For example, at
         the beginning of a line followed by `%` we expect a magic completion
         to be the only applicable option, and after ``my_dict['`` we usually
         expect a completion with an existing dictionary key.
 
         If you want to disable this heuristic and see completions from all matchers,
         set ``IPCompleter.suppress_competing_matchers = False``.
         To disable the heuristic for specific matchers provide a dictionary mapping:
         ``IPCompleter.suppress_competing_matchers = {'IPCompleter.dict_key_matcher': False}``.
 
         Set ``IPCompleter.suppress_competing_matchers = True`` to limit
         completions to the set of matchers with the highest priority;
         this is equivalent to ``IPCompleter.merge_completions`` and
         can be beneficial for performance, but will sometimes omit relevant
         candidates from matchers further down the priority list.
         """,
     ).tag(config=True)
 
     merge_completions = Bool(
         True,
         help="""Whether to merge completion results into a single list
 
         If False, only the completion results from the first non-empty
         completer will be returned.
 
         As of version 8.6.0, setting the value to ``False`` is an alias for:
         ``IPCompleter.suppress_competing_matchers = True.``.
         """,
     ).tag(config=True)
 
     disable_matchers = ListTrait(
         Unicode(),
         help="""List of matchers to disable.
 
         The list should contain matcher identifiers (see :any:`completion_matcher`).
         """,
     ).tag(config=True)
 
     omit__names = Enum(
         (0, 1, 2),
         default_value=2,
         help="""Instruct the completer to omit private method names
 
         Specifically, when completing on ``object.<tab>``.
 
         When 2 [default]: all names that start with '_' will be excluded.
 
         When 1: all 'magic' names (``__foo__``) will be excluded.
 
         When 0: nothing will be excluded.
         """
     ).tag(config=True)
     limit_to__all__ = Bool(False,
         help="""
         DEPRECATED as of version 5.0.
 
         Instruct the completer to use __all__ for the completion
 
         Specifically, when completing on ``object.<tab>``.
 
         When True: only those names in obj.__all__ will be included.
 
         When False [default]: the __all__ attribute is ignored
         """,
     ).tag(config=True)
 
     profile_completions = Bool(
         default_value=False,
         help="If True, emit profiling data for completion subsystem using cProfile."
     ).tag(config=True)
 
     profiler_output_dir = Unicode(
         default_value=".completion_profiles",
         help="Template for path at which to output profile data for completions."
     ).tag(config=True)
 
     @observe('limit_to__all__')
     def _limit_to_all_changed(self, change):
         warnings.warn('`IPython.core.IPCompleter.limit_to__all__` configuration '
             'value has been deprecated since IPython 5.0, will be made to have '
             'no effects and then removed in future version of IPython.',
             UserWarning)
 
     def __init__(
         self, shell=None, namespace=None, global_namespace=None, config=None, **kwargs
     ):
         """IPCompleter() -> completer
 
         Return a completer object.
 
         Parameters
         ----------
         shell
             a pointer to the ipython shell itself.  This is needed
             because this completer knows about magic functions, and those can
             only be accessed via the ipython instance.
         namespace : dict, optional
             an optional dict where completions are performed.
         global_namespace : dict, optional
             secondary optional dict for completions, to
             handle cases (such as IPython embedded inside functions) where
             both Python scopes are visible.
         config : Config
             traitlet's config object
         **kwargs
             passed to super class unmodified.
         """
 
         self.magic_escape = ESC_MAGIC
         self.splitter = CompletionSplitter()
 
         # _greedy_changed() depends on splitter and readline being defined:
         super().__init__(
             namespace=namespace,
             global_namespace=global_namespace,
             config=config,
             **kwargs,
         )
 
         # List where completion matches will be stored
         self.matches = []
         self.shell = shell
         # Regexp to split filenames with spaces in them
         self.space_name_re = re.compile(r'([^\\] )')
         # Hold a local ref. to glob.glob for speed
         self.glob = glob.glob
 
         # Determine if we are running on 'dumb' terminals, like (X)Emacs
         # buffers, to avoid completion problems.
         term = os.environ.get('TERM','xterm')
         self.dumb_terminal = term in ['dumb','emacs']
 
         # Special handling of backslashes needed in win32 platforms
         if sys.platform == "win32":
             self.clean_glob = self._clean_glob_win32
         else:
             self.clean_glob = self._clean_glob
 
         #regexp to parse docstring for function signature
         self.docstring_sig_re = re.compile(r'^[\w|\s.]+\(([^)]*)\).*')
         self.docstring_kwd_re = re.compile(r'[\s|\[]*(\w+)(?:\s*=\s*.*)')
         #use this if positional argument name is also needed
         #= re.compile(r'[\s|\[]*(\w+)(?:\s*=?\s*.*)')
 
         self.magic_arg_matchers = [
             self.magic_config_matcher,
             self.magic_color_matcher,
         ]
 
         # This is set externally by InteractiveShell
         self.custom_completers = None
 
         # This is a list of names of unicode characters that can be completed
         # into their corresponding unicode value. The list is large, so we
         # lazily initialize it on first use. Consuming code should access this
         # attribute through the `@unicode_names` property.
         self._unicode_names = None
 
         self._backslash_combining_matchers = [
             self.latex_name_matcher,
             self.unicode_name_matcher,
             back_latex_name_matcher,
             back_unicode_name_matcher,
             self.fwd_unicode_matcher,
         ]
 
         if not self.backslash_combining_completions:
             for matcher in self._backslash_combining_matchers:
                 self.disable_matchers.append(_get_matcher_id(matcher))
 
         if not self.merge_completions:
             self.suppress_competing_matchers = True
 
     @property
     def matchers(self) -> List[Matcher]:
         """All active matcher routines for completion"""
         if self.dict_keys_only:
             return [self.dict_key_matcher]
 
         if self.use_jedi:
             return [
                 *self.custom_matchers,
                 *self._backslash_combining_matchers,
                 *self.magic_arg_matchers,
                 self.custom_completer_matcher,
                 self.magic_matcher,
                 self._jedi_matcher,
                 self.dict_key_matcher,
                 self.file_matcher,
             ]
         else:
             return [
                 *self.custom_matchers,
                 *self._backslash_combining_matchers,
                 *self.magic_arg_matchers,
                 self.custom_completer_matcher,
                 self.dict_key_matcher,
                 self.magic_matcher,
                 self.python_matcher,
                 self.file_matcher,
                 self.python_func_kw_matcher,
             ]
 
     def all_completions(self, text:str) -> List[str]:
         """
         Wrapper around the completion methods for the benefit of emacs.
         """
         prefix = text.rpartition('.')[0]
         with provisionalcompleter():
             return ['.'.join([prefix, c.text]) if prefix and self.use_jedi else c.text
                     for c in self.completions(text, len(text))]
 
         return self.complete(text)[1]
 
     def _clean_glob(self, text:str):
         return self.glob("%s*" % text)
 
     def _clean_glob_win32(self, text:str):
         return [f.replace("\\","/")
                 for f in self.glob("%s*" % text)]
 
     @context_matcher()
     def file_matcher(self, context: CompletionContext) -> SimpleMatcherResult:
         """Same as :any:`file_matches`, but adopted to new Matcher API."""
         matches = self.file_matches(context.token)
         # TODO: add a heuristic for suppressing (e.g. if it has OS-specific delimiter,
         #  starts with `/home/`, `C:\`, etc)
         return _convert_matcher_v1_result_to_v2(matches, type="path")
 
     def file_matches(self, text: str) -> List[str]:
         """Match filenames, expanding ~USER type strings.
 
         Most of the seemingly convoluted logic in this completer is an
         attempt to handle filenames with spaces in them.  And yet it's not
         quite perfect, because Python's readline doesn't expose all of the
         GNU readline details needed for this to be done correctly.
 
         For a filename with a space in it, the printed completions will be
         only the parts after what's already been typed (instead of the
         full completions, as is normally done).  I don't think with the
         current (as of Python 2.3) Python readline it's possible to do
         better.
 
         .. deprecated:: 8.6
             You can use :meth:`file_matcher` instead.
         """
 
         # chars that require escaping with backslash - i.e. chars
         # that readline treats incorrectly as delimiters, but we
         # don't want to treat as delimiters in filename matching
         # when escaped with backslash
         if text.startswith('!'):
             text = text[1:]
             text_prefix = u'!'
         else:
             text_prefix = u''
 
         text_until_cursor = self.text_until_cursor
         # track strings with open quotes
         open_quotes = has_open_quotes(text_until_cursor)
 
         if '(' in text_until_cursor or '[' in text_until_cursor:
             lsplit = text
         else:
             try:
                 # arg_split ~ shlex.split, but with unicode bugs fixed by us
                 lsplit = arg_split(text_until_cursor)[-1]
             except ValueError:
                 # typically an unmatched ", or backslash without escaped char.
                 if open_quotes:
                     lsplit = text_until_cursor.split(open_quotes)[-1]
                 else:
                     return []
             except IndexError:
                 # tab pressed on empty line
                 lsplit = ""
 
         if not open_quotes and lsplit != protect_filename(lsplit):
             # if protectables are found, do matching on the whole escaped name
             has_protectables = True
             text0,text = text,lsplit
         else:
             has_protectables = False
             text = os.path.expanduser(text)
 
         if text == "":
             return [text_prefix + protect_filename(f) for f in self.glob("*")]
 
         # Compute the matches from the filesystem
         if sys.platform == 'win32':
             m0 = self.clean_glob(text)
         else:
             m0 = self.clean_glob(text.replace('\\', ''))
 
         if has_protectables:
             # If we had protectables, we need to revert our changes to the
             # beginning of filename so that we don't double-write the part
             # of the filename we have so far
             len_lsplit = len(lsplit)
             matches = [text_prefix + text0 +
                        protect_filename(f[len_lsplit:]) for f in m0]
         else:
             if open_quotes:
                 # if we have a string with an open quote, we don't need to
                 # protect the names beyond the quote (and we _shouldn't_, as
                 # it would cause bugs when the filesystem call is made).
                 matches = m0 if sys.platform == "win32" else\
                     [protect_filename(f, open_quotes) for f in m0]
             else:
                 matches = [text_prefix +
                            protect_filename(f) for f in m0]
 
         # Mark directories in input list by appending '/' to their names.
         return [x+'/' if os.path.isdir(x) else x for x in matches]
 
     @context_matcher()
     def magic_matcher(self, context: CompletionContext) -> SimpleMatcherResult:
         """Match magics."""
         text = context.token
         matches = self.magic_matches(text)
         result = _convert_matcher_v1_result_to_v2(matches, type="magic")
         is_magic_prefix = len(text) > 0 and text[0] == "%"
         result["suppress"] = is_magic_prefix and bool(result["completions"])
         return result
 
     def magic_matches(self, text: str) -> List[str]:
         """Match magics.
 
         .. deprecated:: 8.6
             You can use :meth:`magic_matcher` instead.
         """
         # Get all shell magics now rather than statically, so magics loaded at
         # runtime show up too.
         lsm = self.shell.magics_manager.lsmagic()
         line_magics = lsm['line']
         cell_magics = lsm['cell']
         pre = self.magic_escape
         pre2 = pre+pre
 
         explicit_magic = text.startswith(pre)
 
         # Completion logic:
         # - user gives %%: only do cell magics
         # - user gives %: do both line and cell magics
         # - no prefix: do both
         # In other words, line magics are skipped if the user gives %% explicitly
         #
         # We also exclude magics that match any currently visible names:
         # https://github.com/ipython/ipython/issues/4877, unless the user has
         # typed a %:
         # https://github.com/ipython/ipython/issues/10754
         bare_text = text.lstrip(pre)
         global_matches = self.global_matches(bare_text)
         if not explicit_magic:
             def matches(magic):
                 """
                 Filter magics, in particular remove magics that match
                 a name present in global namespace.
                 """
                 return ( magic.startswith(bare_text) and
                          magic not in global_matches )
         else:
             def matches(magic):
                 return magic.startswith(bare_text)
 
         comp = [ pre2+m for m in cell_magics if matches(m)]
         if not text.startswith(pre2):
             comp += [ pre+m for m in line_magics if matches(m)]
 
         return comp
 
     @context_matcher()
     def magic_config_matcher(self, context: CompletionContext) -> SimpleMatcherResult:
         """Match class names and attributes for %config magic."""
         # NOTE: uses `line_buffer` equivalent for compatibility
         matches = self.magic_config_matches(context.line_with_cursor)
         return _convert_matcher_v1_result_to_v2(matches, type="param")
 
     def magic_config_matches(self, text: str) -> List[str]:
         """Match class names and attributes for %config magic.
 
         .. deprecated:: 8.6
             You can use :meth:`magic_config_matcher` instead.
         """
         texts = text.strip().split()
 
         if len(texts) > 0 and (texts[0] == 'config' or texts[0] == '%config'):
             # get all configuration classes
             classes = sorted(set([ c for c in self.shell.configurables
                                    if c.__class__.class_traits(config=True)
                                    ]), key=lambda x: x.__class__.__name__)
             classnames = [ c.__class__.__name__ for c in classes ]
 
             # return all classnames if config or %config is given
             if len(texts) == 1:
                 return classnames
 
             # match classname
             classname_texts = texts[1].split('.')
             classname = classname_texts[0]
             classname_matches = [ c for c in classnames
                                   if c.startswith(classname) ]
 
             # return matched classes or the matched class with attributes
             if texts[1].find('.') < 0:
                 return classname_matches
             elif len(classname_matches) == 1 and \
                             classname_matches[0] == classname:
                 cls = classes[classnames.index(classname)].__class__
                 help = cls.class_get_help()
                 # strip leading '--' from cl-args:
                 help = re.sub(re.compile(r'^--', re.MULTILINE), '', help)
                 return [ attr.split('=')[0]
                          for attr in help.strip().splitlines()
                          if attr.startswith(texts[1]) ]
         return []
 
     @context_matcher()
     def magic_color_matcher(self, context: CompletionContext) -> SimpleMatcherResult:
         """Match color schemes for %colors magic."""
         # NOTE: uses `line_buffer` equivalent for compatibility
         matches = self.magic_color_matches(context.line_with_cursor)
         return _convert_matcher_v1_result_to_v2(matches, type="param")
 
     def magic_color_matches(self, text: str) -> List[str]:
         """Match color schemes for %colors magic.
 
         .. deprecated:: 8.6
             You can use :meth:`magic_color_matcher` instead.
         """
         texts = text.split()
         if text.endswith(' '):
             # .split() strips off the trailing whitespace. Add '' back
             # so that: '%colors ' -> ['%colors', '']
             texts.append('')
 
         if len(texts) == 2 and (texts[0] == 'colors' or texts[0] == '%colors'):
             prefix = texts[1]
             return [ color for color in InspectColors.keys()
                      if color.startswith(prefix) ]
         return []
 
     @context_matcher(identifier="IPCompleter.jedi_matcher")
     def _jedi_matcher(self, context: CompletionContext) -> _JediMatcherResult:
         matches = self._jedi_matches(
             cursor_column=context.cursor_position,
             cursor_line=context.cursor_line,
             text=context.full_text,
         )
         return {
             "completions": matches,
             # static analysis should not suppress other matchers
             "suppress": False,
         }
 
     def _jedi_matches(
         self, cursor_column: int, cursor_line: int, text: str
     ) -> Iterator[_JediCompletionLike]:
         """
         Return a list of :any:`jedi.api.Completion`\\s object from a ``text`` and
         cursor position.
 
         Parameters
         ----------
         cursor_column : int
             column position of the cursor in ``text``, 0-indexed.
         cursor_line : int
             line position of the cursor in ``text``, 0-indexed
         text : str
             text to complete
 
         Notes
         -----
         If ``IPCompleter.debug`` is ``True`` may return a :any:`_FakeJediCompletion`
         object containing a string with the Jedi debug information attached.
 
         .. deprecated:: 8.6
             You can use :meth:`_jedi_matcher` instead.
         """
         namespaces = [self.namespace]
         if self.global_namespace is not None:
             namespaces.append(self.global_namespace)
 
         completion_filter = lambda x:x
         offset = cursor_to_position(text, cursor_line, cursor_column)
         # filter output if we are completing for object members
         if offset:
             pre = text[offset-1]
             if pre == '.':
                 if self.omit__names == 2:
                     completion_filter = lambda c:not c.name.startswith('_')
                 elif self.omit__names == 1:
                     completion_filter = lambda c:not (c.name.startswith('__') and c.name.endswith('__'))
                 elif self.omit__names == 0:
                     completion_filter = lambda x:x
                 else:
                     raise ValueError("Don't understand self.omit__names == {}".format(self.omit__names))
 
         interpreter = jedi.Interpreter(text[:offset], namespaces)
         try_jedi = True
 
         try:
             # find the first token in the current tree -- if it is a ' or " then we are in a string
             completing_string = False
             try:
                 first_child = next(c for c in interpreter._get_module().tree_node.children if hasattr(c, 'value'))
             except StopIteration:
                 pass
             else:
                 # note the value may be ', ", or it may also be ''' or """, or
                 # in some cases, """what/you/typed..., but all of these are
                 # strings.
                 completing_string = len(first_child.value) > 0 and first_child.value[0] in {"'", '"'}
 
             # if we are in a string jedi is likely not the right candidate for
             # now. Skip it.
             try_jedi = not completing_string
         except Exception as e:
             # many of things can go wrong, we are using private API just don't crash.
             if self.debug:
                 print("Error detecting if completing a non-finished string :", e, '|')
 
         if not try_jedi:
             return iter([])
         try:
             return filter(completion_filter, interpreter.complete(column=cursor_column, line=cursor_line + 1))
         except Exception as e:
             if self.debug:
                 return iter(
                     [
                         _FakeJediCompletion(
                             'Oops Jedi has crashed, please report a bug with the following:\n"""\n%s\ns"""'
                             % (e)
                         )
                     ]
                 )
             else:
                 return iter([])
 
     @context_matcher()
     def python_matcher(self, context: CompletionContext) -> SimpleMatcherResult:
         """Match attributes or global python names"""
         text = context.line_with_cursor
         if "." in text:
             try:
                 matches, fragment = self._attr_matches(text, include_prefix=False)
                 if text.endswith(".") and self.omit__names:
                     if self.omit__names == 1:
                         # true if txt is _not_ a __ name, false otherwise:
                         no__name = lambda txt: re.match(r".*\.__.*?__", txt) is None
                     else:
                         # true if txt is _not_ a _ name, false otherwise:
                         no__name = (
                             lambda txt: re.match(r"\._.*?", txt[txt.rindex(".") :])
                             is None
                         )
                     matches = filter(no__name, matches)
                 return _convert_matcher_v1_result_to_v2(
                     matches, type="attribute", fragment=fragment
                 )
             except NameError:
                 # catches <undefined attributes>.<tab>
                 matches = []
                 return _convert_matcher_v1_result_to_v2(matches, type="attribute")
         else:
             matches = self.global_matches(context.token)
             # TODO: maybe distinguish between functions, modules and just "variables"
             return _convert_matcher_v1_result_to_v2(matches, type="variable")
 
     @completion_matcher(api_version=1)
     def python_matches(self, text: str) -> Iterable[str]:
         """Match attributes or global python names.
 
         .. deprecated:: 8.27
             You can use :meth:`python_matcher` instead."""
         if "." in text:
             try:
                 matches = self.attr_matches(text)
                 if text.endswith('.') and self.omit__names:
                     if self.omit__names == 1:
                         # true if txt is _not_ a __ name, false otherwise:
                         no__name = (lambda txt:
                                     re.match(r'.*\.__.*?__',txt) is None)
                     else:
                         # true if txt is _not_ a _ name, false otherwise:
                         no__name = (lambda txt:
                                     re.match(r'\._.*?',txt[txt.rindex('.'):]) is None)
                     matches = filter(no__name, matches)
             except NameError:
                 # catches <undefined attributes>.<tab>
                 matches = []
         else:
             matches = self.global_matches(text)
         return matches
 
     def _default_arguments_from_docstring(self, doc):
         """Parse the first line of docstring for call signature.
 
         Docstring should be of the form 'min(iterable[, key=func])\n'.
         It can also parse cython docstring of the form
         'Minuit.migrad(self, int ncall=10000, resume=True, int nsplit=1)'.
         """
         if doc is None:
             return []
 
         #care only the firstline
         line = doc.lstrip().splitlines()[0]
 
         #p = re.compile(r'^[\w|\s.]+\(([^)]*)\).*')
         #'min(iterable[, key=func])\n' -> 'iterable[, key=func]'
         sig = self.docstring_sig_re.search(line)
         if sig is None:
             return []
         # iterable[, key=func]' -> ['iterable[' ,' key=func]']
         sig = sig.groups()[0].split(',')
         ret = []
         for s in sig:
             #re.compile(r'[\s|\[]*(\w+)(?:\s*=\s*.*)')
             ret += self.docstring_kwd_re.findall(s)
         return ret
 
     def _default_arguments(self, obj):
         """Return the list of default arguments of obj if it is callable,
         or empty list otherwise."""
         call_obj = obj
         ret = []
         if inspect.isbuiltin(obj):
             pass
         elif not (inspect.isfunction(obj) or inspect.ismethod(obj)):
             if inspect.isclass(obj):
                 #for cython embedsignature=True the constructor docstring
                 #belongs to the object itself not __init__
                 ret += self._default_arguments_from_docstring(
                             getattr(obj, '__doc__', ''))
                 # for classes, check for __init__,__new__
                 call_obj = (getattr(obj, '__init__', None) or
                        getattr(obj, '__new__', None))
             # for all others, check if they are __call__able
             elif hasattr(obj, '__call__'):
                 call_obj = obj.__call__
         ret += self._default_arguments_from_docstring(
                  getattr(call_obj, '__doc__', ''))
 
         _keeps = (inspect.Parameter.KEYWORD_ONLY,
                   inspect.Parameter.POSITIONAL_OR_KEYWORD)
 
         try:
             sig = inspect.signature(obj)
             ret.extend(k for k, v in sig.parameters.items() if
                        v.kind in _keeps)
         except ValueError:
             pass
 
         return list(set(ret))
 
     @context_matcher()
     def python_func_kw_matcher(self, context: CompletionContext) -> SimpleMatcherResult:
         """Match named parameters (kwargs) of the last open function."""
         matches = self.python_func_kw_matches(context.token)
         return _convert_matcher_v1_result_to_v2(matches, type="param")
 
     def python_func_kw_matches(self, text):
         """Match named parameters (kwargs) of the last open function.
 
         .. deprecated:: 8.6
             You can use :meth:`python_func_kw_matcher` instead.
         """
 
         if "." in text: # a parameter cannot be dotted
             return []
         try: regexp = self.__funcParamsRegex
         except AttributeError:
             regexp = self.__funcParamsRegex = re.compile(r'''
                 '.*?(?<!\\)' |    # single quoted strings or
                 ".*?(?<!\\)" |    # double quoted strings or
                 \w+          |    # identifier
                 \S                # other characters
                 ''', re.VERBOSE | re.DOTALL)
         # 1. find the nearest identifier that comes before an unclosed
         # parenthesis before the cursor
         # e.g. for "foo (1+bar(x), pa<cursor>,a=1)", the candidate is "foo"
         tokens = regexp.findall(self.text_until_cursor)
         iterTokens = reversed(tokens)
         openPar = 0
 
         for token in iterTokens:
             if token == ')':
                 openPar -= 1
             elif token == '(':
                 openPar += 1
                 if openPar > 0:
                     # found the last unclosed parenthesis
                     break
         else:
             return []
         # 2. Concatenate dotted names ("foo.bar" for "foo.bar(x, pa" )
         ids = []
         isId = re.compile(r'\w+$').match
 
         while True:
             try:
                 ids.append(next(iterTokens))
                 if not isId(ids[-1]):
                     ids.pop()
                     break
                 if not next(iterTokens) == '.':
                     break
             except StopIteration:
                 break
 
         # Find all named arguments already assigned to, as to avoid suggesting
         # them again
         usedNamedArgs = set()
         par_level = -1
         for token, next_token in zip(tokens, tokens[1:]):
             if token == '(':
                 par_level += 1
             elif token == ')':
                 par_level -= 1
 
             if par_level != 0:
                 continue
 
             if next_token != '=':
                 continue
 
             usedNamedArgs.add(token)
 
         argMatches = []
         try:
             callableObj = '.'.join(ids[::-1])
             namedArgs = self._default_arguments(eval(callableObj,
                                                     self.namespace))
 
             # Remove used named arguments from the list, no need to show twice
             for namedArg in set(namedArgs) - usedNamedArgs:
                 if namedArg.startswith(text):
                     argMatches.append("%s=" %namedArg)
         except:
             pass
 
         return argMatches
 
     @staticmethod
     def _get_keys(obj: Any) -> List[Any]:
         # Objects can define their own completions by defining an
         # _ipy_key_completions_() method.
         method = get_real_method(obj, '_ipython_key_completions_')
         if method is not None:
             return method()
 
         # Special case some common in-memory dict-like types
         if isinstance(obj, dict) or _safe_isinstance(obj, "pandas", "DataFrame"):
             try:
                 return list(obj.keys())
             except Exception:
                 return []
         elif _safe_isinstance(obj, "pandas", "core", "indexing", "_LocIndexer"):
             try:
                 return list(obj.obj.keys())
             except Exception:
                 return []
         elif _safe_isinstance(obj, 'numpy', 'ndarray') or\
              _safe_isinstance(obj, 'numpy', 'void'):
             return obj.dtype.names or []
         return []
 
     @context_matcher()
     def dict_key_matcher(self, context: CompletionContext) -> SimpleMatcherResult:
         """Match string keys in a dictionary, after e.g. ``foo[``."""
         matches = self.dict_key_matches(context.token)
         return _convert_matcher_v1_result_to_v2(
             matches, type="dict key", suppress_if_matches=True
         )
 
     def dict_key_matches(self, text: str) -> List[str]:
         """Match string keys in a dictionary, after e.g. ``foo[``.
 
         .. deprecated:: 8.6
             You can use :meth:`dict_key_matcher` instead.
         """
 
         # Short-circuit on closed dictionary (regular expression would
         # not match anyway, but would take quite a while).
         if self.text_until_cursor.strip().endswith("]"):
             return []
 
         match = DICT_MATCHER_REGEX.search(self.text_until_cursor)
 
         if match is None:
             return []
 
         expr, prior_tuple_keys, key_prefix = match.groups()
 
         obj = self._evaluate_expr(expr)
 
         if obj is not_found:
             return []
 
         keys = self._get_keys(obj)
         if not keys:
             return keys
 
         tuple_prefix = guarded_eval(
             prior_tuple_keys,
             EvaluationContext(
                 globals=self.global_namespace,
                 locals=self.namespace,
                 evaluation=self.evaluation,  # type: ignore
                 in_subscript=True,
             ),
         )
 
         closing_quote, token_offset, matches = match_dict_keys(
             keys, key_prefix, self.splitter.delims, extra_prefix=tuple_prefix
         )
         if not matches:
             return []
 
         # get the cursor position of
         # - the text being completed
         # - the start of the key text
         # - the start of the completion
         text_start = len(self.text_until_cursor) - len(text)
         if key_prefix:
             key_start = match.start(3)
             completion_start = key_start + token_offset
         else:
             key_start = completion_start = match.end()
 
         # grab the leading prefix, to make sure all completions start with `text`
         if text_start > key_start:
             leading = ''
         else:
             leading = text[text_start:completion_start]
 
         # append closing quote and bracket as appropriate
         # this is *not* appropriate if the opening quote or bracket is outside
         # the text given to this method, e.g. `d["""a\nt
         can_close_quote = False
         can_close_bracket = False
 
         continuation = self.line_buffer[len(self.text_until_cursor) :].strip()
 
         if continuation.startswith(closing_quote):
             # do not close if already closed, e.g. `d['a<tab>'`
             continuation = continuation[len(closing_quote) :]
         else:
             can_close_quote = True
 
         continuation = continuation.strip()
 
         # e.g. `pandas.DataFrame` has different tuple indexer behaviour,
         # handling it is out of scope, so let's avoid appending suffixes.
         has_known_tuple_handling = isinstance(obj, dict)
 
         can_close_bracket = (
             not continuation.startswith("]") and self.auto_close_dict_keys
         )
         can_close_tuple_item = (
             not continuation.startswith(",")
             and has_known_tuple_handling
             and self.auto_close_dict_keys
         )
         can_close_quote = can_close_quote and self.auto_close_dict_keys
 
         # fast path if closing quote should be appended but not suffix is allowed
         if not can_close_quote and not can_close_bracket and closing_quote:
             return [leading + k for k in matches]
 
         results = []
 
         end_of_tuple_or_item = _DictKeyState.END_OF_TUPLE | _DictKeyState.END_OF_ITEM
 
         for k, state_flag in matches.items():
             result = leading + k
             if can_close_quote and closing_quote:
                 result += closing_quote
 
             if state_flag == end_of_tuple_or_item:
                 # We do not know which suffix to add,
                 # e.g. both tuple item and string
                 # match this item.
                 pass
 
             if state_flag in end_of_tuple_or_item and can_close_bracket:
                 result += "]"
             if state_flag == _DictKeyState.IN_TUPLE and can_close_tuple_item:
                 result += ", "
             results.append(result)
         return results
 
     @context_matcher()
     def unicode_name_matcher(self, context: CompletionContext):
         """Same as :any:`unicode_name_matches`, but adopted to new Matcher API."""
         fragment, matches = self.unicode_name_matches(context.text_until_cursor)
         return _convert_matcher_v1_result_to_v2(
             matches, type="unicode", fragment=fragment, suppress_if_matches=True
         )
 
     @staticmethod
     def unicode_name_matches(text: str) -> Tuple[str, List[str]]:
         """Match Latex-like syntax for unicode characters base
         on the name of the character.
 
         This does  ``\\GREEK SMALL LETTER ETA`` -> ``η``
 
         Works only on valid python 3 identifier, or on combining characters that
         will combine to form a valid identifier.
         """
         slashpos = text.rfind('\\')
         if slashpos > -1:
             s = text[slashpos+1:]
             try :
                 unic = unicodedata.lookup(s)
                 # allow combining chars
                 if ('a'+unic).isidentifier():
                     return '\\'+s,[unic]
             except KeyError:
                 pass
         return '', []
 
     @context_matcher()
     def latex_name_matcher(self, context: CompletionContext):
         """Match Latex syntax for unicode characters.
 
         This does both ``\\alp`` -> ``\\alpha`` and ``\\alpha`` -> ``α``
         """
         fragment, matches = self.latex_matches(context.text_until_cursor)
         return _convert_matcher_v1_result_to_v2(
             matches, type="latex", fragment=fragment, suppress_if_matches=True
         )
 
     def latex_matches(self, text: str) -> Tuple[str, Sequence[str]]:
         """Match Latex syntax for unicode characters.
 
         This does both ``\\alp`` -> ``\\alpha`` and ``\\alpha`` -> ``α``
 
         .. deprecated:: 8.6
             You can use :meth:`latex_name_matcher` instead.
         """
         slashpos = text.rfind('\\')
         if slashpos > -1:
             s = text[slashpos:]
             if s in latex_symbols:
                 # Try to complete a full latex symbol to unicode
                 # \\alpha -> α
                 return s, [latex_symbols[s]]
             else:
                 # If a user has partially typed a latex symbol, give them
                 # a full list of options \al -> [\aleph, \alpha]
                 matches = [k for k in latex_symbols if k.startswith(s)]
                 if matches:
                     return s, matches
         return '', ()
 
     @context_matcher()
     def custom_completer_matcher(self, context):
         """Dispatch custom completer.
 
         If a match is found, suppresses all other matchers except for Jedi.
         """
         matches = self.dispatch_custom_completer(context.token) or []
         result = _convert_matcher_v1_result_to_v2(
             matches, type=_UNKNOWN_TYPE, suppress_if_matches=True
         )
         result["ordered"] = True
         result["do_not_suppress"] = {_get_matcher_id(self._jedi_matcher)}
         return result
 
     def dispatch_custom_completer(self, text):
         """
         .. deprecated:: 8.6
             You can use :meth:`custom_completer_matcher` instead.
         """
         if not self.custom_completers:
             return
 
         line = self.line_buffer
         if not line.strip():
             return None
 
         # Create a little structure to pass all the relevant information about
         # the current completion to any custom completer.
         event = SimpleNamespace()
         event.line = line
         event.symbol = text
         cmd = line.split(None,1)[0]
         event.command = cmd
         event.text_until_cursor = self.text_until_cursor
 
         # for foo etc, try also to find completer for %foo
         if not cmd.startswith(self.magic_escape):
             try_magic = self.custom_completers.s_matches(
                 self.magic_escape + cmd)
         else:
             try_magic = []
 
         for c in itertools.chain(self.custom_completers.s_matches(cmd),
                  try_magic,
                  self.custom_completers.flat_matches(self.text_until_cursor)):
             try:
                 res = c(event)
                 if res:
                     # first, try case sensitive match
                     withcase = [r for r in res if r.startswith(text)]
                     if withcase:
                         return withcase
                     # if none, then case insensitive ones are ok too
                     text_low = text.lower()
                     return [r for r in res if r.lower().startswith(text_low)]
             except TryNext:
                 pass
             except KeyboardInterrupt:
                 """
                 If custom completer take too long,
                 let keyboard interrupt abort and return nothing.
                 """
                 break
 
         return None
 
     def completions(self, text: str, offset: int)->Iterator[Completion]:
         """
         Returns an iterator over the possible completions
 
         .. warning::
 
             Unstable
 
             This function is unstable, API may change without warning.
             It will also raise unless use in proper context manager.
 
         Parameters
         ----------
         text : str
             Full text of the current input, multi line string.
         offset : int
             Integer representing the position of the cursor in ``text``. Offset
             is 0-based indexed.
 
         Yields
         ------
         Completion
 
         Notes
         -----
         The cursor on a text can either be seen as being "in between"
         characters or "On" a character depending on the interface visible to
         the user. For consistency the cursor being on "in between" characters X
         and Y is equivalent to the cursor being "on" character Y, that is to say
         the character the cursor is on is considered as being after the cursor.
 
         Combining characters may span more that one position in the
         text.
 
         .. note::
 
             If ``IPCompleter.debug`` is :any:`True` will yield a ``--jedi/ipython--``
             fake Completion token to distinguish completion returned by Jedi
             and usual IPython completion.
 
         .. note::
 
             Completions are not completely deduplicated yet. If identical
             completions are coming from different sources this function does not
             ensure that each completion object will only be present once.
         """
         warnings.warn("_complete is a provisional API (as of IPython 6.0). "
                       "It may change without warnings. "
                       "Use in corresponding context manager.",
                       category=ProvisionalCompleterWarning, stacklevel=2)
 
         seen = set()
         profiler:Optional[cProfile.Profile]
         try:
             if self.profile_completions:
                 import cProfile
                 profiler = cProfile.Profile()
                 profiler.enable()
             else:
                 profiler = None
 
             for c in self._completions(text, offset, _timeout=self.jedi_compute_type_timeout/1000):
                 if c and (c in seen):
                     continue
                 yield c
                 seen.add(c)
         except KeyboardInterrupt:
             """if completions take too long and users send keyboard interrupt,
             do not crash and return ASAP. """
             pass
         finally:
             if profiler is not None:
                 profiler.disable()
                 ensure_dir_exists(self.profiler_output_dir)
                 output_path = os.path.join(self.profiler_output_dir, str(uuid.uuid4()))
                 print("Writing profiler output to", output_path)
                 profiler.dump_stats(output_path)
 
     def _completions(self, full_text: str, offset: int, *, _timeout) -> Iterator[Completion]:
         """
         Core completion module.Same signature as :any:`completions`, with the
         extra `timeout` parameter (in seconds).
 
         Computing jedi's completion ``.type`` can be quite expensive (it is a
         lazy property) and can require some warm-up, more warm up than just
         computing the ``name`` of a completion. The warm-up can be :
 
             - Long warm-up the first time a module is encountered after
             install/update: actually build parse/inference tree.
 
             - first time the module is encountered in a session: load tree from
             disk.
 
         We don't want to block completions for tens of seconds so we give the
         completer a "budget" of ``_timeout`` seconds per invocation to compute
         completions types, the completions that have not yet been computed will
         be marked as "unknown" an will have a chance to be computed next round
         are things get cached.
 
         Keep in mind that Jedi is not the only thing treating the completion so
         keep the timeout short-ish as if we take more than 0.3 second we still
         have lots of processing to do.
 
         """
         deadline = time.monotonic() + _timeout
 
         before = full_text[:offset]
         cursor_line, cursor_column = position_to_cursor(full_text, offset)
 
         jedi_matcher_id = _get_matcher_id(self._jedi_matcher)
 
         def is_non_jedi_result(
             result: MatcherResult, identifier: str
         ) -> TypeGuard[SimpleMatcherResult]:
             return identifier != jedi_matcher_id
 
         results = self._complete(
             full_text=full_text, cursor_line=cursor_line, cursor_pos=cursor_column
         )
 
         non_jedi_results: Dict[str, SimpleMatcherResult] = {
             identifier: result
             for identifier, result in results.items()
             if is_non_jedi_result(result, identifier)
         }
 
         jedi_matches = (
             cast(_JediMatcherResult, results[jedi_matcher_id])["completions"]
             if jedi_matcher_id in results
             else ()
         )
 
         iter_jm = iter(jedi_matches)
         if _timeout:
             for jm in iter_jm:
                 try:
                     type_ = jm.type
                 except Exception:
                     if self.debug:
                         print("Error in Jedi getting type of ", jm)
                     type_ = None
                 delta = len(jm.name_with_symbols) - len(jm.complete)
                 if type_ == 'function':
                     signature = _make_signature(jm)
                 else:
                     signature = ''
                 yield Completion(start=offset - delta,
                                  end=offset,
                                  text=jm.name_with_symbols,
                                  type=type_,
                                  signature=signature,
                                  _origin='jedi')
 
                 if time.monotonic() > deadline:
                     break
 
         for jm in iter_jm:
             delta = len(jm.name_with_symbols) - len(jm.complete)
             yield Completion(
                 start=offset - delta,
                 end=offset,
                 text=jm.name_with_symbols,
                 type=_UNKNOWN_TYPE,  # don't compute type for speed
                 _origin="jedi",
                 signature="",
             )
 
         # TODO:
         # Suppress this, right now just for debug.
         if jedi_matches and non_jedi_results and self.debug:
             some_start_offset = before.rfind(
                 next(iter(non_jedi_results.values()))["matched_fragment"]
             )
             yield Completion(
                 start=some_start_offset,
                 end=offset,
                 text="--jedi/ipython--",
                 _origin="debug",
                 type="none",
                 signature="",
             )
 
         ordered: List[Completion] = []
         sortable: List[Completion] = []
 
         for origin, result in non_jedi_results.items():
             matched_text = result["matched_fragment"]
             start_offset = before.rfind(matched_text)
             is_ordered = result.get("ordered", False)
             container = ordered if is_ordered else sortable
 
             # I'm unsure if this is always true, so let's assert and see if it
             # crash
             assert before.endswith(matched_text)
 
             for simple_completion in result["completions"]:
                 completion = Completion(
                     start=start_offset,
                     end=offset,
                     text=simple_completion.text,
                     _origin=origin,
                     signature="",
                     type=simple_completion.type or _UNKNOWN_TYPE,
                 )
                 container.append(completion)
 
         yield from list(self._deduplicate(ordered + self._sort(sortable)))[
             :MATCHES_LIMIT
         ]
 
     def complete(self, text=None, line_buffer=None, cursor_pos=None) -> Tuple[str, Sequence[str]]:
         """Find completions for the given text and line context.
 
         Note that both the text and the line_buffer are optional, but at least
         one of them must be given.
 
         Parameters
         ----------
         text : string, optional
             Text to perform the completion on.  If not given, the line buffer
             is split using the instance's CompletionSplitter object.
         line_buffer : string, optional
             If not given, the completer attempts to obtain the current line
             buffer via readline.  This keyword allows clients which are
             requesting for text completions in non-readline contexts to inform
             the completer of the entire text.
         cursor_pos : int, optional
             Index of the cursor in the full line buffer.  Should be provided by
             remote frontends where kernel has no access to frontend state.
 
         Returns
         -------
         Tuple of two items:
         text : str
             Text that was actually used in the completion.
         matches : list
             A list of completion matches.
 
         Notes
         -----
             This API is likely to be deprecated and replaced by
             :any:`IPCompleter.completions` in the future.
 
         """
         warnings.warn('`Completer.complete` is pending deprecation since '
                 'IPython 6.0 and will be replaced by `Completer.completions`.',
                       PendingDeprecationWarning)
         # potential todo, FOLD the 3rd throw away argument of _complete
         # into the first 2 one.
         # TODO: Q: does the above refer to jedi completions (i.e. 0-indexed?)
         # TODO: should we deprecate now, or does it stay?
 
         results = self._complete(
             line_buffer=line_buffer, cursor_pos=cursor_pos, text=text, cursor_line=0
         )
 
         jedi_matcher_id = _get_matcher_id(self._jedi_matcher)
 
         return self._arrange_and_extract(
             results,
             # TODO: can we confirm that excluding Jedi here was a deliberate choice in previous version?
             skip_matchers={jedi_matcher_id},
             # this API does not support different start/end positions (fragments of token).
             abort_if_offset_changes=True,
         )
 
     def _arrange_and_extract(
         self,
         results: Dict[str, MatcherResult],
         skip_matchers: Set[str],
         abort_if_offset_changes: bool,
     ):
         sortable: List[AnyMatcherCompletion] = []
         ordered: List[AnyMatcherCompletion] = []
         most_recent_fragment = None
         for identifier, result in results.items():
             if identifier in skip_matchers:
                 continue
             if not result["completions"]:
                 continue
             if not most_recent_fragment:
                 most_recent_fragment = result["matched_fragment"]
             if (
                 abort_if_offset_changes
                 and result["matched_fragment"] != most_recent_fragment
             ):
                 break
             if result.get("ordered", False):
                 ordered.extend(result["completions"])
             else:
                 sortable.extend(result["completions"])
 
         if not most_recent_fragment:
             most_recent_fragment = ""  # to satisfy typechecker (and just in case)
 
         return most_recent_fragment, [
             m.text for m in self._deduplicate(ordered + self._sort(sortable))
         ]
 
     def _complete(self, *, cursor_line, cursor_pos, line_buffer=None, text=None,
                   full_text=None) -> _CompleteResult:
         """
         Like complete but can also returns raw jedi completions as well as the
         origin of the completion text. This could (and should) be made much
         cleaner but that will be simpler once we drop the old (and stateful)
         :any:`complete` API.
 
         With current provisional API, cursor_pos act both (depending on the
         caller) as the offset in the ``text`` or ``line_buffer``, or as the
         ``column`` when passing multiline strings this could/should be renamed
         but would add extra noise.
 
         Parameters
         ----------
         cursor_line
             Index of the line the cursor is on. 0 indexed.
         cursor_pos
             Position of the cursor in the current line/line_buffer/text. 0
             indexed.
         line_buffer : optional, str
             The current line the cursor is in, this is mostly due to legacy
             reason that readline could only give a us the single current line.
             Prefer `full_text`.
         text : str
             The current "token" the cursor is in, mostly also for historical
             reasons. as the completer would trigger only after the current line
             was parsed.
         full_text : str
             Full text of the current cell.
 
         Returns
         -------
         An ordered dictionary where keys are identifiers of completion
         matchers and values are ``MatcherResult``s.
         """
 
         # if the cursor position isn't given, the only sane assumption we can
         # make is that it's at the end of the line (the common case)
         if cursor_pos is None:
             cursor_pos = len(line_buffer) if text is None else len(text)
 
         if self.use_main_ns:
             self.namespace = __main__.__dict__
 
         # if text is either None or an empty string, rely on the line buffer
         if (not line_buffer) and full_text:
             line_buffer = full_text.split('\n')[cursor_line]
         if not text:  # issue #11508: check line_buffer before calling split_line
             text = (
                 self.splitter.split_line(line_buffer, cursor_pos) if line_buffer else ""
             )
 
         # If no line buffer is given, assume the input text is all there was
         if line_buffer is None:
             line_buffer = text
 
         # deprecated - do not use `line_buffer` in new code.
         self.line_buffer = line_buffer
         self.text_until_cursor = self.line_buffer[:cursor_pos]
 
         if not full_text:
             full_text = line_buffer
 
         context = CompletionContext(
             full_text=full_text,
             cursor_position=cursor_pos,
             cursor_line=cursor_line,
             token=text,
             limit=MATCHES_LIMIT,
         )
 
         # Start with a clean slate of completions
         results: Dict[str, MatcherResult] = {}
 
         jedi_matcher_id = _get_matcher_id(self._jedi_matcher)
 
         suppressed_matchers: Set[str] = set()
 
         matchers = {
             _get_matcher_id(matcher): matcher
             for matcher in sorted(
                 self.matchers, key=_get_matcher_priority, reverse=True
             )
         }
 
         for matcher_id, matcher in matchers.items():
             matcher_id = _get_matcher_id(matcher)
 
             if matcher_id in self.disable_matchers:
                 continue
 
             if matcher_id in results:
                 warnings.warn(f"Duplicate matcher ID: {matcher_id}.")
 
             if matcher_id in suppressed_matchers:
                 continue
 
             result: MatcherResult
             try:
                 if _is_matcher_v1(matcher):
                     result = _convert_matcher_v1_result_to_v2(
                         matcher(text), type=_UNKNOWN_TYPE
                     )
                 elif _is_matcher_v2(matcher):
                     result = matcher(context)
                 else:
                     api_version = _get_matcher_api_version(matcher)
                     raise ValueError(f"Unsupported API version {api_version}")
             except BaseException:
                 # Show the ugly traceback if the matcher causes an
                 # exception, but do NOT crash the kernel!
                 sys.excepthook(*sys.exc_info())
                 continue
 
             # set default value for matched fragment if suffix was not selected.
             result["matched_fragment"] = result.get("matched_fragment", context.token)
 
             if not suppressed_matchers:
                 suppression_recommended: Union[bool, Set[str]] = result.get(
                     "suppress", False
                 )
 
                 suppression_config = (
                     self.suppress_competing_matchers.get(matcher_id, None)
                     if isinstance(self.suppress_competing_matchers, dict)
                     else self.suppress_competing_matchers
                 )
                 should_suppress = (
                     (suppression_config is True)
                     or (suppression_recommended and (suppression_config is not False))
                 ) and has_any_completions(result)
 
                 if should_suppress:
                     suppression_exceptions: Set[str] = result.get(
                         "do_not_suppress", set()
                     )
                     if isinstance(suppression_recommended, Iterable):
                         to_suppress = set(suppression_recommended)
                     else:
                         to_suppress = set(matchers)
                     suppressed_matchers = to_suppress - suppression_exceptions
 
                     new_results = {}
                     for previous_matcher_id, previous_result in results.items():
                         if previous_matcher_id not in suppressed_matchers:
                             new_results[previous_matcher_id] = previous_result
                     results = new_results
 
             results[matcher_id] = result
 
         _, matches = self._arrange_and_extract(
             results,
             # TODO Jedi completions non included in legacy stateful API; was this deliberate or omission?
             #  if it was omission, we can remove the filtering step, otherwise remove this comment.
             skip_matchers={jedi_matcher_id},
             abort_if_offset_changes=False,
         )
 
         # populate legacy stateful API
         self.matches = matches
 
         return results
 
     @staticmethod
     def _deduplicate(
         matches: Sequence[AnyCompletion],
     ) -> Iterable[AnyCompletion]:
         filtered_matches: Dict[str, AnyCompletion] = {}
         for match in matches:
             text = match.text
             if (
                 text not in filtered_matches
                 or filtered_matches[text].type == _UNKNOWN_TYPE
             ):
                 filtered_matches[text] = match
 
         return filtered_matches.values()
 
     @staticmethod
     def _sort(matches: Sequence[AnyCompletion]):
         return sorted(matches, key=lambda x: completions_sorting_key(x.text))
 
     @context_matcher()
     def fwd_unicode_matcher(self, context: CompletionContext):
         """Same as :any:`fwd_unicode_match`, but adopted to new Matcher API."""
         # TODO: use `context.limit` to terminate early once we matched the maximum
         #  number that will be used downstream; can be added as an optional to
         #  `fwd_unicode_match(text: str, limit: int = None)` or we could re-implement here.
         fragment, matches = self.fwd_unicode_match(context.text_until_cursor)
         return _convert_matcher_v1_result_to_v2(
             matches, type="unicode", fragment=fragment, suppress_if_matches=True
         )
 
     def fwd_unicode_match(self, text: str) -> Tuple[str, Sequence[str]]:
         """
         Forward match a string starting with a backslash with a list of
         potential Unicode completions.
 
         Will compute list of Unicode character names on first call and cache it.
 
         .. deprecated:: 8.6
             You can use :meth:`fwd_unicode_matcher` instead.
 
         Returns
         -------
         At tuple with:
             - matched text (empty if no matches)
             - list of potential completions, empty tuple  otherwise)
         """
         # TODO: self.unicode_names is here a list we traverse each time with ~100k elements.
         # We could do a faster match using a Trie.
 
         # Using pygtrie the following seem to work:
 
         #     s = PrefixSet()
 
         #     for c in range(0,0x10FFFF + 1):
         #         try:
         #             s.add(unicodedata.name(chr(c)))
         #         except ValueError:
         #             pass
         #     [''.join(k) for k in s.iter(prefix)]
 
         # But need to be timed and adds an extra dependency.
 
         slashpos = text.rfind('\\')
         # if text starts with slash
         if slashpos > -1:
             # PERF: It's important that we don't access self._unicode_names
             # until we're inside this if-block. _unicode_names is lazily
             # initialized, and it takes a user-noticeable amount of time to
             # initialize it, so we don't want to initialize it unless we're
             # actually going to use it.
             s = text[slashpos + 1 :]
             sup = s.upper()
             candidates = [x for x in self.unicode_names if x.startswith(sup)]
             if candidates:
                 return s, candidates
             candidates = [x for x in self.unicode_names if sup in x]
             if candidates:
                 return s, candidates
             splitsup = sup.split(" ")
             candidates = [
                 x for x in self.unicode_names if all(u in x for u in splitsup)
             ]
             if candidates:
                 return s, candidates
 
             return "", ()
 
         # if text does not start with slash
         else:
             return '', ()
 
     @property
     def unicode_names(self) -> List[str]:
         """List of names of unicode code points that can be completed.
 
         The list is lazily initialized on first access.
         """
         if self._unicode_names is None:
             names = []
             for c in range(0,0x10FFFF + 1):
                 try:
                     names.append(unicodedata.name(chr(c)))
                 except ValueError:
                     pass
             self._unicode_names = _unicode_name_compute(_UNICODE_RANGES)
 
         return self._unicode_names
 
 def _unicode_name_compute(ranges:List[Tuple[int,int]]) -> List[str]:
     names = []
     for start,stop in ranges:
         for c in range(start, stop) :
             try:
                 names.append(unicodedata.name(chr(c)))
             except ValueError:
                 pass
     return names

IPython/core/debugger.py

@@ -1,1131 +1,1136
 """
 Pdb debugger class.
 
 
 This is an extension to PDB which adds a number of new features.
 Note that there is also the `IPython.terminal.debugger` class which provides UI
 improvements.
 
 We also strongly recommend to use this via the `ipdb` package, which provides
 extra configuration options.
 
 Among other things, this subclass of PDB:
  - supports many IPython magics like pdef/psource
  - hide frames in tracebacks based on `__tracebackhide__`
  - allows to skip frames based on `__debuggerskip__`
 
 
 Global Configuration
 --------------------
 
 The IPython debugger will by read the global ``~/.pdbrc`` file.
 That is to say you can list all commands supported by ipdb in your `~/.pdbrc`
 configuration file, to globally configure pdb.
 
 Example::
 
    # ~/.pdbrc
    skip_predicates debuggerskip false
    skip_hidden false
    context 25
 
 Features
 --------
 
 The IPython debugger can hide and skip frames when printing or moving through
 the stack. This can have a performance impact, so can be configures.
 
 The skipping and hiding frames are configurable via the `skip_predicates`
 command.
 
 By default, frames from readonly files will be hidden, frames containing
 ``__tracebackhide__ = True`` will be hidden.
 
 Frames containing ``__debuggerskip__`` will be stepped over, frames whose parent
 frames value of ``__debuggerskip__`` is ``True`` will also be skipped.
 
     >>> def helpers_helper():
     ...     pass
     ...
     ... def helper_1():
     ...     print("don't step in me")
     ...     helpers_helpers() # will be stepped over unless breakpoint set.
     ...
     ...
     ... def helper_2():
     ...     print("in me neither")
     ...
 
 One can define a decorator that wraps a function between the two helpers:
 
     >>> def pdb_skipped_decorator(function):
     ...
     ...
     ...     def wrapped_fn(*args, **kwargs):
     ...         __debuggerskip__ = True
     ...         helper_1()
     ...         __debuggerskip__ = False
     ...         result = function(*args, **kwargs)
     ...         __debuggerskip__ = True
     ...         helper_2()
     ...         # setting __debuggerskip__ to False again is not necessary
     ...         return result
     ...
     ...     return wrapped_fn
 
 When decorating a function, ipdb will directly step into ``bar()`` by
 default:
 
     >>> @foo_decorator
     ... def bar(x, y):
     ...     return x * y
 
 
 You can toggle the behavior with
 
     ipdb> skip_predicates debuggerskip false
 
 or configure it in your ``.pdbrc``
 
 
 
 License
 -------
 
 Modified from the standard pdb.Pdb class to avoid including readline, so that
 the command line completion of other programs which include this isn't
 damaged.
 
 In the future, this class will be expanded with improvements over the standard
 pdb.
 
 The original code in this file is mainly lifted out of cmd.py in Python 2.2,
 with minor changes. Licensing should therefore be under the standard Python
 terms.  For details on the PSF (Python Software Foundation) standard license,
 see:
 
 https://docs.python.org/2/license.html
 
 
 All the changes since then are under the same license as IPython.
 
 """
 
 #*****************************************************************************
 #
 #       This file is licensed under the PSF license.
 #
 #       Copyright (C) 2001 Python Software Foundation, www.python.org
 #       Copyright (C) 2005-2006 Fernando Perez. <[e-mail]>
 #
 #
 #*****************************************************************************
 
 from __future__ import annotations
 
 import inspect
 import linecache
 import os
 import re
 import sys
 from contextlib import contextmanager
 from functools import lru_cache
 
 from IPython import get_ipython
 from IPython.core.excolors import exception_colors
-from IPython.utils import PyColorize, py3compat
+from IPython.utils import PyColorize, coloransi, py3compat
 
 from typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
     # otherwise circular import
     from IPython.core.interactiveshell import InteractiveShell
 
 # skip module docstests
 __skip_doctest__ = True
 
 prompt = 'ipdb> '
 
 # We have to check this directly from sys.argv, config struct not yet available
 from pdb import Pdb as OldPdb
 
 # Allow the set_trace code to operate outside of an ipython instance, even if
 # it does so with some limitations.  The rest of this support is implemented in
 # the Tracer constructor.
 
 DEBUGGERSKIP = "__debuggerskip__"
 
 
 # this has been implemented in Pdb in Python 3.13 (https://github.com/python/cpython/pull/106676
 # on lower python versions, we backported the feature.
 CHAIN_EXCEPTIONS = sys.version_info < (3, 13)
 
 
 def make_arrow(pad):
     """generate the leading arrow in front of traceback or debugger"""
     if pad >= 2:
         return '-'*(pad-2) + '> '
     elif pad == 1:
         return '>'
     return ''
 
 
 def BdbQuit_excepthook(et, ev, tb, excepthook=None):
     """Exception hook which handles `BdbQuit` exceptions.
 
     All other exceptions are processed using the `excepthook`
     parameter.
     """
     raise ValueError(
         "`BdbQuit_excepthook` is deprecated since version 5.1. It is still around only because it is still imported by ipdb.",
     )
 
 
 RGX_EXTRA_INDENT = re.compile(r'(?<=\n)\s+')
 
 
 def strip_indentation(multiline_string):
     return RGX_EXTRA_INDENT.sub('', multiline_string)
 
 
 def decorate_fn_with_doc(new_fn, old_fn, additional_text=""):
     """Make new_fn have old_fn's doc string. This is particularly useful
     for the ``do_...`` commands that hook into the help system.
     Adapted from from a comp.lang.python posting
     by Duncan Booth."""
     def wrapper(*args, **kw):
         return new_fn(*args, **kw)
     if old_fn.__doc__:
         wrapper.__doc__ = strip_indentation(old_fn.__doc__) + additional_text
     return wrapper
 
 
 class Pdb(OldPdb):
     """Modified Pdb class, does not load readline.
 
     for a standalone version that uses prompt_toolkit, see
     `IPython.terminal.debugger.TerminalPdb` and
     `IPython.terminal.debugger.set_trace()`
 
 
     This debugger can hide and skip frames that are tagged according to some predicates.
     See the `skip_predicates` commands.
 
     """
 
     shell: InteractiveShell
 
     if CHAIN_EXCEPTIONS:
         MAX_CHAINED_EXCEPTION_DEPTH = 999
 
     default_predicates = {
         "tbhide": True,
         "readonly": False,
         "ipython_internal": True,
         "debuggerskip": True,
     }
 
     def __init__(self, completekey=None, stdin=None, stdout=None, context=5, **kwargs):
         """Create a new IPython debugger.
 
         Parameters
         ----------
         completekey : default None
             Passed to pdb.Pdb.
         stdin : default None
             Passed to pdb.Pdb.
         stdout : default None
             Passed to pdb.Pdb.
         context : int
             Number of lines of source code context to show when
             displaying stacktrace information.
         **kwargs
             Passed to pdb.Pdb.
 
         Notes
         -----
         The possibilities are python version dependent, see the python
         docs for more info.
         """
 
         # Parent constructor:
         try:
             self.context = int(context)
             if self.context <= 0:
                 raise ValueError("Context must be a positive integer")
         except (TypeError, ValueError) as e:
                 raise ValueError("Context must be a positive integer") from e
 
         # `kwargs` ensures full compatibility with stdlib's `pdb.Pdb`.
         OldPdb.__init__(self, completekey, stdin, stdout, **kwargs)
 
         # IPython changes...
         self.shell = get_ipython()
 
         if self.shell is None:
             save_main = sys.modules['__main__']
             # No IPython instance running, we must create one
             from IPython.terminal.interactiveshell import \
                 TerminalInteractiveShell
             self.shell = TerminalInteractiveShell.instance()
             # needed by any code which calls __import__("__main__") after
             # the debugger was entered. See also #9941.
             sys.modules["__main__"] = save_main
 
 
         color_scheme = self.shell.colors
 
         self.aliases = {}
 
         # Create color table: we copy the default one from the traceback
         # module and add a few attributes needed for debugging
         self.color_scheme_table = exception_colors()
 
+        # shorthands
+        C = coloransi.TermColors
+        cst = self.color_scheme_table
+
+
         # Add a python parser so we can syntax highlight source while
         # debugging.
         self.parser = PyColorize.Parser(style=color_scheme)
         self.set_colors(color_scheme)
 
         # Set the prompt - the default prompt is '(Pdb)'
         self.prompt = prompt
         self.skip_hidden = True
         self.report_skipped = True
 
         # list of predicates we use to skip frames
         self._predicates = self.default_predicates
 
         if CHAIN_EXCEPTIONS:
             self._chained_exceptions = tuple()
             self._chained_exception_index = 0
 
     #
     def set_colors(self, scheme):
         """Shorthand access to the color table scheme selector method."""
         self.color_scheme_table.set_active_scheme(scheme)
         self.parser.style = scheme
 
     def set_trace(self, frame=None):
         if frame is None:
             frame = sys._getframe().f_back
         self.initial_frame = frame
         return super().set_trace(frame)
 
     def _hidden_predicate(self, frame):
         """
         Given a frame return whether it it should be hidden or not by IPython.
         """
 
         if self._predicates["readonly"]:
             fname = frame.f_code.co_filename
             # we need to check for file existence and interactively define
             # function would otherwise appear as RO.
             if os.path.isfile(fname) and not os.access(fname, os.W_OK):
                 return True
 
         if self._predicates["tbhide"]:
             if frame in (self.curframe, getattr(self, "initial_frame", None)):
                 return False
             frame_locals = self._get_frame_locals(frame)
             if "__tracebackhide__" not in frame_locals:
                 return False
             return frame_locals["__tracebackhide__"]
         return False
 
     def hidden_frames(self, stack):
         """
         Given an index in the stack return whether it should be skipped.
 
         This is used in up/down and where to skip frames.
         """
         # The f_locals dictionary is updated from the actual frame
         # locals whenever the .f_locals accessor is called, so we
         # avoid calling it here to preserve self.curframe_locals.
         # Furthermore, there is no good reason to hide the current frame.
         ip_hide = [self._hidden_predicate(s[0]) for s in stack]
         ip_start = [i for i, s in enumerate(ip_hide) if s == "__ipython_bottom__"]
         if ip_start and self._predicates["ipython_internal"]:
             ip_hide = [h if i > ip_start[0] else True for (i, h) in enumerate(ip_hide)]
         return ip_hide
 
     if CHAIN_EXCEPTIONS:
 
         def _get_tb_and_exceptions(self, tb_or_exc):
             """
             Given a tracecack or an exception, return a tuple of chained exceptions
             and current traceback to inspect.
             This will deal with selecting the right ``__cause__`` or ``__context__``
             as well as handling cycles, and return a flattened list of exceptions we
             can jump to with do_exceptions.
             """
             _exceptions = []
             if isinstance(tb_or_exc, BaseException):
                 traceback, current = tb_or_exc.__traceback__, tb_or_exc
 
                 while current is not None:
                     if current in _exceptions:
                         break
                     _exceptions.append(current)
                     if current.__cause__ is not None:
                         current = current.__cause__
                     elif (
                         current.__context__ is not None
                         and not current.__suppress_context__
                     ):
                         current = current.__context__
 
                     if len(_exceptions) >= self.MAX_CHAINED_EXCEPTION_DEPTH:
                         self.message(
                             f"More than {self.MAX_CHAINED_EXCEPTION_DEPTH}"
                             " chained exceptions found, not all exceptions"
                             "will be browsable with `exceptions`."
                         )
                         break
             else:
                 traceback = tb_or_exc
             return tuple(reversed(_exceptions)), traceback
 
         @contextmanager
         def _hold_exceptions(self, exceptions):
             """
             Context manager to ensure proper cleaning of exceptions references
             When given a chained exception instead of a traceback,
             pdb may hold references to many objects which may leak memory.
             We use this context manager to make sure everything is properly cleaned
             """
             try:
                 self._chained_exceptions = exceptions
                 self._chained_exception_index = len(exceptions) - 1
                 yield
             finally:
                 # we can't put those in forget as otherwise they would
                 # be cleared on exception change
                 self._chained_exceptions = tuple()
                 self._chained_exception_index = 0
 
         def do_exceptions(self, arg):
             """exceptions [number]
             List or change current exception in an exception chain.
             Without arguments, list all the current exception in the exception
             chain. Exceptions will be numbered, with the current exception indicated
             with an arrow.
             If given an integer as argument, switch to the exception at that index.
             """
             if not self._chained_exceptions:
                 self.message(
                     "Did not find chained exceptions. To move between"
                     " exceptions, pdb/post_mortem must be given an exception"
                     " object rather than a traceback."
                 )
                 return
             if not arg:
                 for ix, exc in enumerate(self._chained_exceptions):
                     prompt = ">" if ix == self._chained_exception_index else " "
                     rep = repr(exc)
                     if len(rep) > 80:
                         rep = rep[:77] + "..."
                     indicator = (
                         "  -"
                         if self._chained_exceptions[ix].__traceback__ is None
                         else f"{ix:>3}"
                     )
                     self.message(f"{prompt} {indicator} {rep}")
             else:
                 try:
                     number = int(arg)
                 except ValueError:
                     self.error("Argument must be an integer")
                     return
                 if 0 <= number < len(self._chained_exceptions):
                     if self._chained_exceptions[number].__traceback__ is None:
                         self.error(
                             "This exception does not have a traceback, cannot jump to it"
                         )
                         return
 
                     self._chained_exception_index = number
                     self.setup(None, self._chained_exceptions[number].__traceback__)
                     self.print_stack_entry(self.stack[self.curindex])
                 else:
                     self.error("No exception with that number")
 
     def interaction(self, frame, tb_or_exc):
         try:
             if CHAIN_EXCEPTIONS:
                 # this context manager is part of interaction in 3.13
                 _chained_exceptions, tb = self._get_tb_and_exceptions(tb_or_exc)
                 if isinstance(tb_or_exc, BaseException):
                     assert tb is not None, "main exception must have a traceback"
                 with self._hold_exceptions(_chained_exceptions):
                     OldPdb.interaction(self, frame, tb)
             else:
                 OldPdb.interaction(self, frame, tb_or_exc)
 
         except KeyboardInterrupt:
             self.stdout.write("\n" + self.shell.get_exception_only())
 
     def precmd(self, line):
         """Perform useful escapes on the command before it is executed."""
 
         if line.endswith("??"):
             line = "pinfo2 " + line[:-2]
         elif line.endswith("?"):
             line = "pinfo " + line[:-1]
 
         line = super().precmd(line)
 
         return line
 
     def new_do_quit(self, arg):
         return OldPdb.do_quit(self, arg)
 
     do_q = do_quit = decorate_fn_with_doc(new_do_quit, OldPdb.do_quit)
 
     def print_stack_trace(self, context=None):
         Colors = self.color_scheme_table.active_colors
         ColorsNormal = Colors.Normal
         if context is None:
             context = self.context
         try:
             context = int(context)
             if context <= 0:
                 raise ValueError("Context must be a positive integer")
         except (TypeError, ValueError) as e:
                 raise ValueError("Context must be a positive integer") from e
         try:
             skipped = 0
             for hidden, frame_lineno in zip(self.hidden_frames(self.stack), self.stack):
                 if hidden and self.skip_hidden:
                     skipped += 1
                     continue
                 if skipped:
                     print(
                         f"{Colors.excName}    [... skipping {skipped} hidden frame(s)]{ColorsNormal}\n"
                     )
                     skipped = 0
                 self.print_stack_entry(frame_lineno, context=context)
             if skipped:
                 print(
                     f"{Colors.excName}    [... skipping {skipped} hidden frame(s)]{ColorsNormal}\n"
                 )
         except KeyboardInterrupt:
             pass
 
     def print_stack_entry(self, frame_lineno, prompt_prefix='\n-> ',
                           context=None):
         if context is None:
             context = self.context
         try:
             context = int(context)
             if context <= 0:
                 raise ValueError("Context must be a positive integer")
         except (TypeError, ValueError) as e:
                 raise ValueError("Context must be a positive integer") from e
         print(self.format_stack_entry(frame_lineno, '', context), file=self.stdout)
 
         # vds: >>
         frame, lineno = frame_lineno
         filename = frame.f_code.co_filename
         self.shell.hooks.synchronize_with_editor(filename, lineno, 0)
         # vds: <<
 
     def _get_frame_locals(self, frame):
         """ "
         Accessing f_local of current frame reset the namespace, so we want to avoid
         that or the following can happen
 
         ipdb> foo
         "old"
         ipdb> foo = "new"
         ipdb> foo
         "new"
         ipdb> where
         ipdb> foo
         "old"
 
         So if frame is self.current_frame we instead return self.curframe_locals
 
         """
         if frame is getattr(self, "curframe", None):
             return self.curframe_locals
         else:
             return frame.f_locals
 
     def format_stack_entry(self, frame_lineno, lprefix=': ', context=None):
         if context is None:
             context = self.context
         try:
             context = int(context)
             if context <= 0:
                 print("Context must be a positive integer", file=self.stdout)
         except (TypeError, ValueError):
                 print("Context must be a positive integer", file=self.stdout)
 
         import reprlib
 
         ret = []
 
         Colors = self.color_scheme_table.active_colors
         ColorsNormal = Colors.Normal
         tpl_link = "%s%%s%s" % (Colors.filenameEm, ColorsNormal)
         tpl_call = "%s%%s%s%%s%s" % (Colors.vName, Colors.valEm, ColorsNormal)
         tpl_line = "%%s%s%%s %s%%s" % (Colors.lineno, ColorsNormal)
         tpl_line_em = "%%s%s%%s %s%%s%s" % (Colors.linenoEm, Colors.line, ColorsNormal)
 
         frame, lineno = frame_lineno
 
         return_value = ''
         loc_frame = self._get_frame_locals(frame)
         if "__return__" in loc_frame:
             rv = loc_frame["__return__"]
             # return_value += '->'
             return_value += reprlib.repr(rv) + "\n"
         ret.append(return_value)
 
         #s = filename + '(' + `lineno` + ')'
         filename = self.canonic(frame.f_code.co_filename)
         link = tpl_link % py3compat.cast_unicode(filename)
 
         if frame.f_code.co_name:
             func = frame.f_code.co_name
         else:
             func = "<lambda>"
 
         call = ""
         if func != "?":
             if "__args__" in loc_frame:
                 args = reprlib.repr(loc_frame["__args__"])
             else:
                 args = '()'
             call = tpl_call % (func, args)
 
         # The level info should be generated in the same format pdb uses, to
         # avoid breaking the pdbtrack functionality of python-mode in *emacs.
         if frame is self.curframe:
             ret.append('> ')
         else:
             ret.append("  ")
         ret.append("%s(%s)%s\n" % (link, lineno, call))
 
         start = lineno - 1 - context//2
         lines = linecache.getlines(filename)
         start = min(start, len(lines) - context)
         start = max(start, 0)
         lines = lines[start : start + context]
 
         for i, line in enumerate(lines):
             show_arrow = start + 1 + i == lineno
             linetpl = (frame is self.curframe or show_arrow) and tpl_line_em or tpl_line
             ret.append(
                 self.__format_line(
                     linetpl, filename, start + 1 + i, line, arrow=show_arrow
                 )
             )
         return "".join(ret)
 
     def __format_line(self, tpl_line, filename, lineno, line, arrow=False):
         bp_mark = ""
         bp_mark_color = ""
 
         new_line, err = self.parser.format2(line, 'str')
         if not err:
             line = new_line
 
         bp = None
         if lineno in self.get_file_breaks(filename):
             bps = self.get_breaks(filename, lineno)
             bp = bps[-1]
 
         if bp:
             Colors = self.color_scheme_table.active_colors
             bp_mark = str(bp.number)
             bp_mark_color = Colors.breakpoint_enabled
             if not bp.enabled:
                 bp_mark_color = Colors.breakpoint_disabled
 
         numbers_width = 7
         if arrow:
             # This is the line with the error
             pad = numbers_width - len(str(lineno)) - len(bp_mark)
             num = '%s%s' % (make_arrow(pad), str(lineno))
         else:
             num = '%*s' % (numbers_width - len(bp_mark), str(lineno))
 
         return tpl_line % (bp_mark_color + bp_mark, num, line)
 
     def print_list_lines(self, filename, first, last):
         """The printing (as opposed to the parsing part of a 'list'
         command."""
         try:
             Colors = self.color_scheme_table.active_colors
             ColorsNormal = Colors.Normal
             tpl_line = '%%s%s%%s %s%%s' % (Colors.lineno, ColorsNormal)
             tpl_line_em = '%%s%s%%s %s%%s%s' % (Colors.linenoEm, Colors.line, ColorsNormal)
             src = []
             if filename == "<string>" and hasattr(self, "_exec_filename"):
                 filename = self._exec_filename
 
             for lineno in range(first, last+1):
                 line = linecache.getline(filename, lineno)
                 if not line:
                     break
 
                 if lineno == self.curframe.f_lineno:
                     line = self.__format_line(
                         tpl_line_em, filename, lineno, line, arrow=True
                     )
                 else:
                     line = self.__format_line(
                         tpl_line, filename, lineno, line, arrow=False
                     )
 
                 src.append(line)
                 self.lineno = lineno
 
             print(''.join(src), file=self.stdout)
 
         except KeyboardInterrupt:
             pass
 
     def do_skip_predicates(self, args):
         """
         Turn on/off individual predicates as to whether a frame should be hidden/skip.
 
         The global option to skip (or not) hidden frames is set with skip_hidden
 
         To change the value of a predicate
 
             skip_predicates key [true|false]
 
         Call without arguments to see the current values.
 
         To permanently change the value of an option add the corresponding
         command to your ``~/.pdbrc`` file. If you are programmatically using the
         Pdb instance you can also change the ``default_predicates`` class
         attribute.
         """
         if not args.strip():
             print("current predicates:")
             for p, v in self._predicates.items():
                 print("   ", p, ":", v)
             return
         type_value = args.strip().split(" ")
         if len(type_value) != 2:
             print(
                 f"Usage: skip_predicates <type> <value>, with <type> one of {set(self._predicates.keys())}"
             )
             return
 
         type_, value = type_value
         if type_ not in self._predicates:
             print(f"{type_!r} not in {set(self._predicates.keys())}")
             return
         if value.lower() not in ("true", "yes", "1", "no", "false", "0"):
             print(
                 f"{value!r} is invalid - use one of ('true', 'yes', '1', 'no', 'false', '0')"
             )
             return
 
         self._predicates[type_] = value.lower() in ("true", "yes", "1")
         if not any(self._predicates.values()):
             print(
                 "Warning, all predicates set to False, skip_hidden may not have any effects."
             )
 
     def do_skip_hidden(self, arg):
         """
         Change whether or not we should skip frames with the
         __tracebackhide__ attribute.
         """
         if not arg.strip():
             print(
                 f"skip_hidden = {self.skip_hidden}, use 'yes','no', 'true', or 'false' to change."
             )
         elif arg.strip().lower() in ("true", "yes"):
             self.skip_hidden = True
         elif arg.strip().lower() in ("false", "no"):
             self.skip_hidden = False
         if not any(self._predicates.values()):
             print(
                 "Warning, all predicates set to False, skip_hidden may not have any effects."
             )
 
     def do_list(self, arg):
         """Print lines of code from the current stack frame
         """
         self.lastcmd = 'list'
         last = None
         if arg and arg != ".":
             try:
                 x = eval(arg, {}, {})
                 if type(x) == type(()):
                     first, last = x
                     first = int(first)
                     last = int(last)
                     if last < first:
                         # Assume it's a count
                         last = first + last
                 else:
                     first = max(1, int(x) - 5)
             except:
                 print('*** Error in argument:', repr(arg), file=self.stdout)
                 return
         elif self.lineno is None or arg == ".":
             first = max(1, self.curframe.f_lineno - 5)
         else:
             first = self.lineno + 1
         if last is None:
             last = first + 10
         self.print_list_lines(self.curframe.f_code.co_filename, first, last)
 
         # vds: >>
         lineno = first
         filename = self.curframe.f_code.co_filename
         self.shell.hooks.synchronize_with_editor(filename, lineno, 0)
         # vds: <<
 
     do_l = do_list
 
     def getsourcelines(self, obj):
         lines, lineno = inspect.findsource(obj)
         if inspect.isframe(obj) and obj.f_globals is self._get_frame_locals(obj):
             # must be a module frame: do not try to cut a block out of it
             return lines, 1
         elif inspect.ismodule(obj):
             return lines, 1
         return inspect.getblock(lines[lineno:]), lineno+1
 
     def do_longlist(self, arg):
         """Print lines of code from the current stack frame.
 
         Shows more lines than 'list' does.
         """
         self.lastcmd = 'longlist'
         try:
             lines, lineno = self.getsourcelines(self.curframe)
         except OSError as err:
             self.error(err)
             return
         last = lineno + len(lines)
         self.print_list_lines(self.curframe.f_code.co_filename, lineno, last)
     do_ll = do_longlist
 
     def do_debug(self, arg):
         """debug code
         Enter a recursive debugger that steps through the code
         argument (which is an arbitrary expression or statement to be
         executed in the current environment).
         """
         trace_function = sys.gettrace()
         sys.settrace(None)
         globals = self.curframe.f_globals
         locals = self.curframe_locals
         p = self.__class__(completekey=self.completekey,
                            stdin=self.stdin, stdout=self.stdout)
         p.use_rawinput = self.use_rawinput
         p.prompt = "(%s) " % self.prompt.strip()
         self.message("ENTERING RECURSIVE DEBUGGER")
         sys.call_tracing(p.run, (arg, globals, locals))
         self.message("LEAVING RECURSIVE DEBUGGER")
         sys.settrace(trace_function)
         self.lastcmd = p.lastcmd
 
     def do_pdef(self, arg):
         """Print the call signature for any callable object.
 
         The debugger interface to %pdef"""
         namespaces = [
             ("Locals", self.curframe_locals),
             ("Globals", self.curframe.f_globals),
         ]
         self.shell.find_line_magic("pdef")(arg, namespaces=namespaces)
 
     def do_pdoc(self, arg):
         """Print the docstring for an object.
 
         The debugger interface to %pdoc."""
         namespaces = [
             ("Locals", self.curframe_locals),
             ("Globals", self.curframe.f_globals),
         ]
         self.shell.find_line_magic("pdoc")(arg, namespaces=namespaces)
 
     def do_pfile(self, arg):
         """Print (or run through pager) the file where an object is defined.
 
         The debugger interface to %pfile.
         """
         namespaces = [
             ("Locals", self.curframe_locals),
             ("Globals", self.curframe.f_globals),
         ]
         self.shell.find_line_magic("pfile")(arg, namespaces=namespaces)
 
     def do_pinfo(self, arg):
         """Provide detailed information about an object.
 
         The debugger interface to %pinfo, i.e., obj?."""
         namespaces = [
             ("Locals", self.curframe_locals),
             ("Globals", self.curframe.f_globals),
         ]
         self.shell.find_line_magic("pinfo")(arg, namespaces=namespaces)
 
     def do_pinfo2(self, arg):
         """Provide extra detailed information about an object.
 
         The debugger interface to %pinfo2, i.e., obj??."""
         namespaces = [
             ("Locals", self.curframe_locals),
             ("Globals", self.curframe.f_globals),
         ]
         self.shell.find_line_magic("pinfo2")(arg, namespaces=namespaces)
 
     def do_psource(self, arg):
         """Print (or run through pager) the source code for an object."""
         namespaces = [
             ("Locals", self.curframe_locals),
             ("Globals", self.curframe.f_globals),
         ]
         self.shell.find_line_magic("psource")(arg, namespaces=namespaces)
 
     def do_where(self, arg):
         """w(here)
         Print a stack trace, with the most recent frame at the bottom.
         An arrow indicates the "current frame", which determines the
         context of most commands. 'bt' is an alias for this command.
 
         Take a number as argument as an (optional) number of context line to
         print"""
         if arg:
             try:
                 context = int(arg)
             except ValueError as err:
                 self.error(err)
                 return
             self.print_stack_trace(context)
         else:
             self.print_stack_trace()
 
     do_w = do_where
 
     def break_anywhere(self, frame):
         """
         _stop_in_decorator_internals is overly restrictive, as we may still want
         to trace function calls, so we need to also update break_anywhere so
         that is we don't `stop_here`, because of debugger skip, we may still
         stop at any point inside the function
 
         """
 
         sup = super().break_anywhere(frame)
         if sup:
             return sup
         if self._predicates["debuggerskip"]:
             if DEBUGGERSKIP in frame.f_code.co_varnames:
                 return True
             if frame.f_back and self._get_frame_locals(frame.f_back).get(DEBUGGERSKIP):
                 return True
         return False
 
     def _is_in_decorator_internal_and_should_skip(self, frame):
         """
         Utility to tell us whether we are in a decorator internal and should stop.
 
         """
         # if we are disabled don't skip
         if not self._predicates["debuggerskip"]:
             return False
 
         return self._cachable_skip(frame)
 
     @lru_cache(1024)
     def _cached_one_parent_frame_debuggerskip(self, frame):
         """
         Cache looking up for DEBUGGERSKIP on parent frame.
 
         This should speedup walking through deep frame when one of the highest
         one does have a debugger skip.
 
         This is likely to introduce fake positive though.
         """
         while getattr(frame, "f_back", None):
             frame = frame.f_back
             if self._get_frame_locals(frame).get(DEBUGGERSKIP):
                 return True
         return None
 
     @lru_cache(1024)
     def _cachable_skip(self, frame):
         # if frame is tagged, skip by default.
         if DEBUGGERSKIP in frame.f_code.co_varnames:
             return True
 
         # if one of the parent frame value set to True skip as well.
         if self._cached_one_parent_frame_debuggerskip(frame):
             return True
 
         return False
 
     def stop_here(self, frame):
         if self._is_in_decorator_internal_and_should_skip(frame) is True:
             return False
 
         hidden = False
         if self.skip_hidden:
             hidden = self._hidden_predicate(frame)
         if hidden:
             if self.report_skipped:
                 Colors = self.color_scheme_table.active_colors
                 ColorsNormal = Colors.Normal
                 print(
                     f"{Colors.excName}    [... skipped 1 hidden frame]{ColorsNormal}\n"
                 )
         return super().stop_here(frame)
 
     def do_up(self, arg):
         """u(p) [count]
         Move the current frame count (default one) levels up in the
         stack trace (to an older frame).
 
         Will skip hidden frames.
         """
         # modified version of upstream that skips
         # frames with __tracebackhide__
         if self.curindex == 0:
             self.error("Oldest frame")
             return
         try:
             count = int(arg or 1)
         except ValueError:
             self.error("Invalid frame count (%s)" % arg)
             return
         skipped = 0
         if count < 0:
             _newframe = 0
         else:
             counter = 0
             hidden_frames = self.hidden_frames(self.stack)
             for i in range(self.curindex - 1, -1, -1):
                 if hidden_frames[i] and self.skip_hidden:
                     skipped += 1
                     continue
                 counter += 1
                 if counter >= count:
                     break
             else:
                 # if no break occurred.
                 self.error(
                     "all frames above hidden, use `skip_hidden False` to get get into those."
                 )
                 return
 
             Colors = self.color_scheme_table.active_colors
             ColorsNormal = Colors.Normal
             _newframe = i
         self._select_frame(_newframe)
         if skipped:
             print(
                 f"{Colors.excName}    [... skipped {skipped} hidden frame(s)]{ColorsNormal}\n"
             )
 
     def do_down(self, arg):
         """d(own) [count]
         Move the current frame count (default one) levels down in the
         stack trace (to a newer frame).
 
         Will skip hidden frames.
         """
         if self.curindex + 1 == len(self.stack):
             self.error("Newest frame")
             return
         try:
             count = int(arg or 1)
         except ValueError:
             self.error("Invalid frame count (%s)" % arg)
             return
         if count < 0:
             _newframe = len(self.stack) - 1
         else:
             counter = 0
             skipped = 0
             hidden_frames = self.hidden_frames(self.stack)
             for i in range(self.curindex + 1, len(self.stack)):
                 if hidden_frames[i] and self.skip_hidden:
                     skipped += 1
                     continue
                 counter += 1
                 if counter >= count:
                     break
             else:
                 self.error(
                     "all frames below hidden, use `skip_hidden False` to get get into those."
                 )
                 return
 
             Colors = self.color_scheme_table.active_colors
             ColorsNormal = Colors.Normal
             if skipped:
                 print(
                     f"{Colors.excName}    [... skipped {skipped} hidden frame(s)]{ColorsNormal}\n"
                 )
             _newframe = i
 
         self._select_frame(_newframe)
 
     do_d = do_down
     do_u = do_up
 
     def do_context(self, context):
         """context number_of_lines
         Set the number of lines of source code to show when displaying
         stacktrace information.
         """
         try:
             new_context = int(context)
             if new_context <= 0:
                 raise ValueError()
             self.context = new_context
         except ValueError:
             self.error(
                 f"The 'context' command requires a positive integer argument (current value {self.context})."
             )
 
 
 class InterruptiblePdb(Pdb):
     """Version of debugger where KeyboardInterrupt exits the debugger altogether."""
 
     def cmdloop(self, intro=None):
         """Wrap cmdloop() such that KeyboardInterrupt stops the debugger."""
         try:
             return OldPdb.cmdloop(self, intro=intro)
         except KeyboardInterrupt:
             self.stop_here = lambda frame: False
             self.do_quit("")
             sys.settrace(None)
             self.quitting = False
             raise
 
     def _cmdloop(self):
         while True:
             try:
                 # keyboard interrupts allow for an easy way to cancel
                 # the current command, so allow them during interactive input
                 self.allow_kbdint = True
                 self.cmdloop()
                 self.allow_kbdint = False
                 break
             except KeyboardInterrupt:
                 self.message('--KeyboardInterrupt--')
                 raise
 
 
 def set_trace(frame=None, header=None):
     """
     Start debugging from `frame`.
 
     If frame is not specified, debugging starts from caller's frame.
     """
     pdb = Pdb()
     if header is not None:
         pdb.message(header)
     pdb.set_trace(frame or sys._getframe().f_back)

IPython/core/displayhook.py

@@ -1,336 +1,336
 # -*- coding: utf-8 -*-
 """Displayhook for IPython.
 
 This defines a callable class that IPython uses for `sys.displayhook`.
 """
 
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 import builtins as builtin_mod
 import sys
 import io as _io
 import tokenize
 
 from traitlets.config.configurable import Configurable
 from traitlets import Instance, Float
 from warnings import warn
 
 # TODO: Move the various attributes (cache_size, [others now moved]). Some
 # of these are also attributes of InteractiveShell. They should be on ONE object
 # only and the other objects should ask that one object for their values.
 
 class DisplayHook(Configurable):
     """The custom IPython displayhook to replace sys.displayhook.
 
     This class does many things, but the basic idea is that it is a callable
     that gets called anytime user code returns a value.
     """
 
     shell = Instance('IPython.core.interactiveshell.InteractiveShellABC',
                      allow_none=True)
     exec_result = Instance('IPython.core.interactiveshell.ExecutionResult',
                            allow_none=True)
     cull_fraction = Float(0.2)
 
     def __init__(self, shell=None, cache_size=1000, **kwargs):
         super(DisplayHook, self).__init__(shell=shell, **kwargs)
         cache_size_min = 3
         if cache_size <= 0:
             self.do_full_cache = 0
             cache_size = 0
         elif cache_size < cache_size_min:
             self.do_full_cache = 0
             cache_size = 0
             warn('caching was disabled (min value for cache size is %s).' %
                  cache_size_min,stacklevel=3)
         else:
             self.do_full_cache = 1
 
         self.cache_size = cache_size
 
         # we need a reference to the user-level namespace
         self.shell = shell
         
         self._,self.__,self.___ = '','',''
 
         # these are deliberately global:
         to_user_ns = {'_':self._,'__':self.__,'___':self.___}
         self.shell.user_ns.update(to_user_ns)
 
     @property
     def prompt_count(self):
         return self.shell.execution_count
 
     #-------------------------------------------------------------------------
     # Methods used in __call__. Override these methods to modify the behavior
     # of the displayhook.
     #-------------------------------------------------------------------------
 
     def check_for_underscore(self):
         """Check if the user has set the '_' variable by hand."""
         # If something injected a '_' variable in __builtin__, delete
         # ipython's automatic one so we don't clobber that.  gettext() in
         # particular uses _, so we need to stay away from it.
         if '_' in builtin_mod.__dict__:
             try:
                 user_value = self.shell.user_ns['_']
                 if user_value is not self._:
                     return
                 del self.shell.user_ns['_']
             except KeyError:
                 pass
 
     def quiet(self):
         """Should we silence the display hook because of ';'?"""
         # do not print output if input ends in ';'
         
         try:
             cell = self.shell.history_manager.input_hist_parsed[-1]
         except IndexError:
             # some uses of ipshellembed may fail here
             return False
         
         return self.semicolon_at_end_of_expression(cell)
 
     @staticmethod
     def semicolon_at_end_of_expression(expression):
         """Parse Python expression and detects whether last token is ';'"""
 
         sio = _io.StringIO(expression)
         tokens = list(tokenize.generate_tokens(sio.readline))
 
         for token in reversed(tokens):
             if token[0] in (tokenize.ENDMARKER, tokenize.NL, tokenize.NEWLINE, tokenize.COMMENT):
                 continue
             if (token[0] == tokenize.OP) and (token[1] == ';'):
                 return True
             else:
                 return False
 
     def start_displayhook(self):
         """Start the displayhook, initializing resources."""
         pass
 
     def write_output_prompt(self):
         """Write the output prompt.
 
         The default implementation simply writes the prompt to
         ``sys.stdout``.
         """
         # Use write, not print which adds an extra space.
         sys.stdout.write(self.shell.separate_out)
         outprompt = 'Out[{}]: '.format(self.shell.execution_count)
         if self.do_full_cache:
             sys.stdout.write(outprompt)
 
     def compute_format_data(self, result):
         """Compute format data of the object to be displayed.
 
         The format data is a generalization of the :func:`repr` of an object.
         In the default implementation the format data is a :class:`dict` of
         key value pair where the keys are valid MIME types and the values
         are JSON'able data structure containing the raw data for that MIME
         type. It is up to frontends to determine pick a MIME to to use and
         display that data in an appropriate manner.
 
         This method only computes the format data for the object and should
         NOT actually print or write that to a stream.
 
         Parameters
         ----------
         result : object
             The Python object passed to the display hook, whose format will be
             computed.
 
         Returns
         -------
         (format_dict, md_dict) : dict
             format_dict is a :class:`dict` whose keys are valid MIME types and values are
             JSON'able raw data for that MIME type. It is recommended that
             all return values of this should always include the "text/plain"
             MIME type representation of the object.
             md_dict is a :class:`dict` with the same MIME type keys
             of metadata associated with each output.
 
         """
         return self.shell.display_formatter.format(result)
 
     # This can be set to True by the write_output_prompt method in a subclass
     prompt_end_newline = False
 
     def write_format_data(self, format_dict, md_dict=None) -> None:
         """Write the format data dict to the frontend.
 
         This default version of this method simply writes the plain text
         representation of the object to ``sys.stdout``. Subclasses should
         override this method to send the entire `format_dict` to the
         frontends.
 
         Parameters
         ----------
         format_dict : dict
             The format dict for the object passed to `sys.displayhook`.
         md_dict : dict (optional)
             The metadata dict to be associated with the display data.
         """
         if 'text/plain' not in format_dict:
             # nothing to do
             return
         # We want to print because we want to always make sure we have a
         # newline, even if all the prompt separators are ''. This is the
         # standard IPython behavior.
         result_repr = format_dict['text/plain']
         if '\n' in result_repr:
             # So that multi-line strings line up with the left column of
             # the screen, instead of having the output prompt mess up
             # their first line.
             # We use the prompt template instead of the expanded prompt
             # because the expansion may add ANSI escapes that will interfere
             # with our ability to determine whether or not we should add
             # a newline.
             if not self.prompt_end_newline:
                 # But avoid extraneous empty lines.
                 result_repr = '\n' + result_repr
 
         try:
             print(result_repr)
         except UnicodeEncodeError:
             # If a character is not supported by the terminal encoding replace
             # it with its \u or \x representation
             print(result_repr.encode(sys.stdout.encoding,'backslashreplace').decode(sys.stdout.encoding))
 
     def update_user_ns(self, result):
         """Update user_ns with various things like _, __, _1, etc."""
 
         # Avoid recursive reference when displaying _oh/Out
         if self.cache_size and result is not self.shell.user_ns['_oh']:
             if len(self.shell.user_ns['_oh']) >= self.cache_size and self.do_full_cache:
                 self.cull_cache()
 
             # Don't overwrite '_' and friends if '_' is in __builtin__
             # (otherwise we cause buggy behavior for things like gettext). and
             # do not overwrite _, __ or ___ if one of these has been assigned
             # by the user.
             update_unders = True
             for unders in ['_'*i for i in range(1,4)]:
-                if unders not in self.shell.user_ns:
+                if not unders in self.shell.user_ns:
                     continue
                 if getattr(self, unders) is not self.shell.user_ns.get(unders):
                     update_unders = False
 
             self.___ = self.__
             self.__ = self._
             self._ = result
 
             if ('_' not in builtin_mod.__dict__) and (update_unders):
                 self.shell.push({'_':self._,
                                  '__':self.__,
                                 '___':self.___}, interactive=False)
 
             # hackish access to top-level  namespace to create _1,_2... dynamically
             to_main = {}
             if self.do_full_cache:
                 new_result = '_%s' % self.prompt_count
                 to_main[new_result] = result
                 self.shell.push(to_main, interactive=False)
                 self.shell.user_ns['_oh'][self.prompt_count] = result
 
     def fill_exec_result(self, result):
         if self.exec_result is not None:
             self.exec_result.result = result
 
     def log_output(self, format_dict):
         """Log the output."""
         if 'text/plain' not in format_dict:
             # nothing to do
             return
         if self.shell.logger.log_output:
             self.shell.logger.log_write(format_dict['text/plain'], 'output')
         self.shell.history_manager.output_hist_reprs[self.prompt_count] = \
                                                     format_dict['text/plain']
 
     def finish_displayhook(self):
         """Finish up all displayhook activities."""
         sys.stdout.write(self.shell.separate_out2)
         sys.stdout.flush()
 
     def __call__(self, result=None):
         """Printing with history cache management.
 
         This is invoked every time the interpreter needs to print, and is
         activated by setting the variable sys.displayhook to it.
         """
         self.check_for_underscore()
         if result is not None and not self.quiet():
             self.start_displayhook()
             self.write_output_prompt()
             format_dict, md_dict = self.compute_format_data(result)
             self.update_user_ns(result)
             self.fill_exec_result(result)
             if format_dict:
                 self.write_format_data(format_dict, md_dict)
                 self.log_output(format_dict)
             self.finish_displayhook()
 
     def cull_cache(self):
         """Output cache is full, cull the oldest entries"""
         oh = self.shell.user_ns.get('_oh', {})
         sz = len(oh)
         cull_count = max(int(sz * self.cull_fraction), 2)
         warn('Output cache limit (currently {sz} entries) hit.\n'
              'Flushing oldest {cull_count} entries.'.format(sz=sz, cull_count=cull_count))
         
         for i, n in enumerate(sorted(oh)):
             if i >= cull_count:
                 break
             self.shell.user_ns.pop('_%i' % n, None)
             oh.pop(n, None)
         
 
     def flush(self):
         if not self.do_full_cache:
             raise ValueError("You shouldn't have reached the cache flush "
                              "if full caching is not enabled!")
         # delete auto-generated vars from global namespace
 
         for n in range(1,self.prompt_count + 1):
             key = '_'+repr(n)
             try:
                 del self.shell.user_ns_hidden[key]
             except KeyError:
                 pass
             try:
                 del self.shell.user_ns[key]
             except KeyError:
                 pass
         # In some embedded circumstances, the user_ns doesn't have the
         # '_oh' key set up.
         oh = self.shell.user_ns.get('_oh', None)
         if oh is not None:
             oh.clear()
 
         # Release our own references to objects:
         self._, self.__, self.___ = '', '', ''
 
         if '_' not in builtin_mod.__dict__:
             self.shell.user_ns.update({'_':self._,'__':self.__,'___':self.___})
         import gc
         # TODO: Is this really needed?
         # IronPython blocks here forever
         if sys.platform != "cli":
             gc.collect()
 
 
 class CapturingDisplayHook(object):
     def __init__(self, shell, outputs=None):
         self.shell = shell
         if outputs is None:
             outputs = []
         self.outputs = outputs
 
     def __call__(self, result=None):
         if result is None:
             return
         format_dict, md_dict = self.shell.display_formatter.format(result)
         self.outputs.append({ 'data': format_dict, 'metadata': md_dict })

IPython/core/displaypub.py

@@ -1,146 +1,149
 """An interface for publishing rich data to frontends.
 
 There are two components of the display system:
 
 * Display formatters, which take a Python object and compute the
   representation of the object in various formats (text, HTML, SVG, etc.).
 * The display publisher that is used to send the representation data to the
   various frontends.
 
 This module defines the logic display publishing. The display publisher uses
 the ``display_data`` message type that is defined in the IPython messaging
 spec.
 """
 
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 
 import sys
 
 from traitlets.config.configurable import Configurable
 from traitlets import List
 
+# This used to be defined here - it is imported for backwards compatibility
+from .display_functions import publish_display_data
+
 import typing as t
 
 # -----------------------------------------------------------------------------
 # Main payload class
 #-----------------------------------------------------------------------------
 
 
 class DisplayPublisher(Configurable):
     """A traited class that publishes display data to frontends.
 
     Instances of this class are created by the main IPython object and should
     be accessed there.
     """
 
     def __init__(self, shell=None, *args, **kwargs):
         self.shell = shell
         super().__init__(*args, **kwargs)
 
     def _validate_data(self, data, metadata=None):
         """Validate the display data.
 
         Parameters
         ----------
         data : dict
             The formata data dictionary.
         metadata : dict
             Any metadata for the data.
         """
 
         if not isinstance(data, dict):
             raise TypeError('data must be a dict, got: %r' % data)
         if metadata is not None:
             if not isinstance(metadata, dict):
                 raise TypeError('metadata must be a dict, got: %r' % data)
 
     # use * to indicate transient, update are keyword-only
     def publish(self, data, metadata=None, source=None, *, transient=None, update=False, **kwargs) -> None:
         """Publish data and metadata to all frontends.
 
         See the ``display_data`` message in the messaging documentation for
         more details about this message type.
 
         The following MIME types are currently implemented:
 
         * text/plain
         * text/html
         * text/markdown
         * text/latex
         * application/json
         * application/javascript
         * image/png
         * image/jpeg
         * image/svg+xml
 
         Parameters
         ----------
         data : dict
             A dictionary having keys that are valid MIME types (like
             'text/plain' or 'image/svg+xml') and values that are the data for
             that MIME type. The data itself must be a JSON'able data
             structure. Minimally all data should have the 'text/plain' data,
             which can be displayed by all frontends. If more than the plain
             text is given, it is up to the frontend to decide which
             representation to use.
         metadata : dict
             A dictionary for metadata related to the data. This can contain
             arbitrary key, value pairs that frontends can use to interpret
             the data.  Metadata specific to each mime-type can be specified
             in the metadata dict with the same mime-type keys as
             the data itself.
         source : str, deprecated
             Unused.
         transient : dict, keyword-only
             A dictionary for transient data.
             Data in this dictionary should not be persisted as part of saving this output.
             Examples include 'display_id'.
         update : bool, keyword-only, default: False
             If True, only update existing outputs with the same display_id,
             rather than creating a new output.
         """
 
         handlers: t.Dict = {}
         if self.shell is not None:
             handlers = getattr(self.shell, "mime_renderers", {})
 
         for mime, handler in handlers.items():
             if mime in data:
                 handler(data[mime], metadata.get(mime, None))
                 return
 
         if 'text/plain' in data:
             print(data['text/plain'])
 
     def clear_output(self, wait=False):
         """Clear the output of the cell receiving output."""
         print('\033[2K\r', end='')
         sys.stdout.flush()
         print('\033[2K\r', end='')
         sys.stderr.flush()
 
 
 class CapturingDisplayPublisher(DisplayPublisher):
     """A DisplayPublisher that stores"""
 
     outputs: List = List()
 
     def publish(
         self, data, metadata=None, source=None, *, transient=None, update=False
     ):
         self.outputs.append(
             {
                 "data": data,
                 "metadata": metadata,
                 "transient": transient,
                 "update": update,
             }
         )
 
     def clear_output(self, wait=False):
         super(CapturingDisplayPublisher, self).clear_output(wait)
 
         # empty the list, *do not* reassign a new list
         self.outputs.clear()

IPython/core/extensions.py

@@ -1,132 +1,135
 # encoding: utf-8
 """A class for managing IPython extensions."""
 
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
+import os
+import os.path
 import sys
 from importlib import import_module, reload
 
 from traitlets.config.configurable import Configurable
+from IPython.utils.path import ensure_dir_exists
 from traitlets import Instance
 
 
 #-----------------------------------------------------------------------------
 # Main class
 #-----------------------------------------------------------------------------
 
 BUILTINS_EXTS = {"storemagic": False, "autoreload": False}
 
 
 class ExtensionManager(Configurable):
     """A class to manage IPython extensions.
 
     An IPython extension is an importable Python module that has
     a function with the signature::
 
         def load_ipython_extension(ipython):
             # Do things with ipython
 
     This function is called after your extension is imported and the
     currently active :class:`InteractiveShell` instance is passed as
     the only argument.  You can do anything you want with IPython at
     that point, including defining new magic and aliases, adding new
     components, etc.
 
     You can also optionally define an :func:`unload_ipython_extension(ipython)`
     function, which will be called if the user unloads or reloads the extension.
     The extension manager will only call :func:`load_ipython_extension` again
     if the extension is reloaded.
 
     You can put your extension modules anywhere you want, as long as
     they can be imported by Python's standard import mechanism.
     """
 
     shell = Instance('IPython.core.interactiveshell.InteractiveShellABC', allow_none=True)
 
     def __init__(self, shell=None, **kwargs):
         super(ExtensionManager, self).__init__(shell=shell, **kwargs)
         self.loaded = set()
 
     def load_extension(self, module_str: str):
         """Load an IPython extension by its module name.
 
         Returns the string "already loaded" if the extension is already loaded,
         "no load function" if the module doesn't have a load_ipython_extension
         function, or None if it succeeded.
         """
         try:
             return self._load_extension(module_str)
         except ModuleNotFoundError:
             if module_str in BUILTINS_EXTS:
                 BUILTINS_EXTS[module_str] = True
                 return self._load_extension("IPython.extensions." + module_str)
             raise
 
     def _load_extension(self, module_str: str):
         if module_str in self.loaded:
             return "already loaded"
 
         assert self.shell is not None
 
         with self.shell.builtin_trap:
             if module_str not in sys.modules:
                 mod = import_module(module_str)
             mod = sys.modules[module_str]
             if self._call_load_ipython_extension(mod):
                 self.loaded.add(module_str)
             else:
                 return "no load function"
 
     def unload_extension(self, module_str: str):
         """Unload an IPython extension by its module name.
 
         This function looks up the extension's name in ``sys.modules`` and
         simply calls ``mod.unload_ipython_extension(self)``.
 
         Returns the string "no unload function" if the extension doesn't define
         a function to unload itself, "not loaded" if the extension isn't loaded,
         otherwise None.
         """
         if BUILTINS_EXTS.get(module_str, False) is True:
             module_str = "IPython.extensions." + module_str
         if module_str not in self.loaded:
             return "not loaded"
 
         if module_str in sys.modules:
             mod = sys.modules[module_str]
             if self._call_unload_ipython_extension(mod):
                 self.loaded.discard(module_str)
             else:
                 return "no unload function"
 
     def reload_extension(self, module_str: str):
         """Reload an IPython extension by calling reload.
 
         If the module has not been loaded before,
         :meth:`InteractiveShell.load_extension` is called. Otherwise
         :func:`reload` is called and then the :func:`load_ipython_extension`
         function of the module, if it exists is called.
         """
 
         if BUILTINS_EXTS.get(module_str, False) is True:
             module_str = "IPython.extensions." + module_str
 
         if (module_str in self.loaded) and (module_str in sys.modules):
             self.unload_extension(module_str)
             mod = sys.modules[module_str]
             reload(mod)
             if self._call_load_ipython_extension(mod):
                 self.loaded.add(module_str)
         else:
             self.load_extension(module_str)
 
     def _call_load_ipython_extension(self, mod):
         if hasattr(mod, 'load_ipython_extension'):
             mod.load_ipython_extension(self.shell)
             return True
 
     def _call_unload_ipython_extension(self, mod):
         if hasattr(mod, 'unload_ipython_extension'):
             mod.unload_ipython_extension(self.shell)
             return True

IPython/core/history.py

@@ -1,1140 +1,1141
 """History related magics and functionality"""
 
 from __future__ import annotations
 
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 
 import atexit
 import datetime
 import re
 
 
 import threading
 from pathlib import Path
 
 from decorator import decorator
 from traitlets import (
     Any,
     Bool,
     Dict,
     Instance,
     Integer,
     List,
     TraitError,
     Unicode,
     Union,
     default,
     observe,
 )
 from traitlets.config.configurable import LoggingConfigurable
 
 from IPython.paths import locate_profile
 from IPython.utils.decorators import undoc
 from typing import Iterable, Tuple, Optional, TYPE_CHECKING
 import typing
 from warnings import warn
 
 if TYPE_CHECKING:
     from IPython.core.interactiveshell import InteractiveShell
     from IPython.config.Configuration import Configuration
 
 try:
     from sqlite3 import DatabaseError, OperationalError
     import sqlite3
 
     sqlite3_found = True
 except ModuleNotFoundError:
     sqlite3_found = False
 
     class DatabaseError(Exception):  # type: ignore [no-redef]
         pass
 
     class OperationalError(Exception):  # type: ignore [no-redef]
         pass
 
 
 InOrInOut = typing.Union[str, Tuple[str, Optional[str]]]
 
 # -----------------------------------------------------------------------------
 # Classes and functions
 # -----------------------------------------------------------------------------
 
 
 @undoc
 class DummyDB:
     """Dummy DB that will act as a black hole for history.
 
     Only used in the absence of sqlite"""
 
     def execute(*args: typing.Any, **kwargs: typing.Any) -> typing.List:
         return []
 
     def commit(self, *args, **kwargs):  # type: ignore [no-untyped-def]
         pass
 
     def __enter__(self, *args, **kwargs):  # type: ignore [no-untyped-def]
         pass
 
     def __exit__(self, *args, **kwargs):  # type: ignore [no-untyped-def]
         pass
 
 
 @decorator
 def only_when_enabled(f, self, *a, **kw):  # type: ignore [no-untyped-def]
     """Decorator: return an empty list in the absence of sqlite."""
     if not self.enabled:
         return []
     else:
         return f(self, *a, **kw)
 
 
 # use 16kB as threshold for whether a corrupt history db should be saved
 # that should be at least 100 entries or so
 _SAVE_DB_SIZE = 16384
 
 
 @decorator
 def catch_corrupt_db(f, self, *a, **kw):  # type: ignore [no-untyped-def]
     """A decorator which wraps HistoryAccessor method calls to catch errors from
     a corrupt SQLite database, move the old database out of the way, and create
     a new one.
 
     We avoid clobbering larger databases because this may be triggered due to filesystem issues,
     not just a corrupt file.
     """
     try:
         return f(self, *a, **kw)
     except (DatabaseError, OperationalError) as e:
         self._corrupt_db_counter += 1
         self.log.error("Failed to open SQLite history %s (%s).", self.hist_file, e)
         if self.hist_file != ":memory:":
             if self._corrupt_db_counter > self._corrupt_db_limit:
                 self.hist_file = ":memory:"
                 self.log.error(
                     "Failed to load history too many times, history will not be saved."
                 )
             elif self.hist_file.is_file():
                 # move the file out of the way
                 base = str(self.hist_file.parent / self.hist_file.stem)
                 ext = self.hist_file.suffix
                 size = self.hist_file.stat().st_size
                 if size >= _SAVE_DB_SIZE:
                     # if there's significant content, avoid clobbering
                     now = datetime.datetime.now().isoformat().replace(":", ".")
                     newpath = base + "-corrupt-" + now + ext
                     # don't clobber previous corrupt backups
                     for i in range(100):
                         if not Path(newpath).exists():
                             break
                         else:
                             newpath = base + "-corrupt-" + now + ("-%i" % i) + ext
                 else:
                     # not much content, possibly empty; don't worry about clobbering
                     # maybe we should just delete it?
                     newpath = base + "-corrupt" + ext
                 self.hist_file.rename(newpath)
                 self.log.error(
                     "History file was moved to %s and a new file created.", newpath
                 )
             self.init_db()
             return []
         else:
             # Failed with :memory:, something serious is wrong
             raise
 
 
 class HistoryAccessorBase(LoggingConfigurable):
     """An abstract class for History Accessors"""
 
     def get_tail(
         self,
         n: int = 10,
         raw: bool = True,
         output: bool = False,
         include_latest: bool = False,
     ) -> Iterable[Tuple[int, int, InOrInOut]]:
         raise NotImplementedError
 
     def search(
         self,
         pattern: str = "*",
         raw: bool = True,
         search_raw: bool = True,
         output: bool = False,
         n: Optional[int] = None,
         unique: bool = False,
     ) -> Iterable[Tuple[int, int, InOrInOut]]:
         raise NotImplementedError
 
     def get_range(
         self,
         session: int,
         start: int = 1,
         stop: Optional[int] = None,
         raw: bool = True,
         output: bool = False,
     ) -> Iterable[Tuple[int, int, InOrInOut]]:
         raise NotImplementedError
 
     def get_range_by_str(
         self, rangestr: str, raw: bool = True, output: bool = False
     ) -> Iterable[Tuple[int, int, InOrInOut]]:
         raise NotImplementedError
 
 
 class HistoryAccessor(HistoryAccessorBase):
     """Access the history database without adding to it.
 
     This is intended for use by standalone history tools. IPython shells use
     HistoryManager, below, which is a subclass of this."""
 
     # counter for init_db retries, so we don't keep trying over and over
     _corrupt_db_counter = 0
     # after two failures, fallback on :memory:
     _corrupt_db_limit = 2
 
     # String holding the path to the history file
     hist_file = Union(
         [Instance(Path), Unicode()],
         help="""Path to file to use for SQLite history database.
 
         By default, IPython will put the history database in the IPython
         profile directory.  If you would rather share one history among
         profiles, you can set this value in each, so that they are consistent.
 
         Due to an issue with fcntl, SQLite is known to misbehave on some NFS
         mounts.  If you see IPython hanging, try setting this to something on a
         local disk, e.g::
 
             ipython --HistoryManager.hist_file=/tmp/ipython_hist.sqlite
 
         you can also use the specific value `:memory:` (including the colon
         at both end but not the back ticks), to avoid creating an history file.
 
         """,
     ).tag(config=True)
 
     enabled = Bool(
         sqlite3_found,
         help="""enable the SQLite history
 
         set enabled=False to disable the SQLite history,
         in which case there will be no stored history, no SQLite connection,
         and no background saving thread.  This may be necessary in some
         threaded environments where IPython is embedded.
         """,
     ).tag(config=True)
 
     connection_options = Dict(
         help="""Options for configuring the SQLite connection
 
         These options are passed as keyword args to sqlite3.connect
         when establishing database connections.
         """
     ).tag(config=True)
 
     @default("connection_options")
     def _default_connection_options(self) -> typing.Dict[str, bool]:
         return dict(check_same_thread=False)
 
     # The SQLite database
     db = Any()
 
     @observe("db")
     @only_when_enabled
     def _db_changed(self, change):  # type: ignore [no-untyped-def]
         """validate the db, since it can be an Instance of two different types"""
         new = change["new"]
         connection_types = (DummyDB, sqlite3.Connection)
         if not isinstance(new, connection_types):
             msg = "%s.db must be sqlite3 Connection or DummyDB, not %r" % (
                 self.__class__.__name__,
                 new,
             )
             raise TraitError(msg)
 
     def __init__(
         self, profile: str = "default", hist_file: str = "", **traits: typing.Any
     ) -> None:
         """Create a new history accessor.
 
         Parameters
         ----------
         profile : str
             The name of the profile from which to open history.
         hist_file : str
             Path to an SQLite history database stored by IPython. If specified,
             hist_file overrides profile.
         config : :class:`~traitlets.config.loader.Config`
             Config object. hist_file can also be set through this.
         """
         super(HistoryAccessor, self).__init__(**traits)
         # defer setting hist_file from kwarg until after init,
         # otherwise the default kwarg value would clobber any value
         # set by config
         if hist_file:
             self.hist_file = hist_file
 
         try:
             self.hist_file
         except TraitError:
             # No one has set the hist_file, yet.
             self.hist_file = self._get_hist_file_name(profile)
 
         self.init_db()
 
     def _get_hist_file_name(self, profile: str = "default") -> Path:
         """Find the history file for the given profile name.
 
         This is overridden by the HistoryManager subclass, to use the shell's
         active profile.
 
         Parameters
         ----------
         profile : str
             The name of a profile which has a history file.
         """
         return Path(locate_profile(profile)) / "history.sqlite"
 
     @catch_corrupt_db
     def init_db(self) -> None:
         """Connect to the database, and create tables if necessary."""
         if not self.enabled:
             self.db = DummyDB()
             return
 
         # use detect_types so that timestamps return datetime objects
         kwargs = dict(detect_types=sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES)
         kwargs.update(self.connection_options)
         self.db = sqlite3.connect(str(self.hist_file), **kwargs)  # type: ignore [call-overload]
         with self.db:
             self.db.execute(
                 """CREATE TABLE IF NOT EXISTS sessions (session integer
                             primary key autoincrement, start timestamp,
                             end timestamp, num_cmds integer, remark text)"""
             )
             self.db.execute(
                 """CREATE TABLE IF NOT EXISTS history
                     (session integer, line integer, source text, source_raw text,
                     PRIMARY KEY (session, line))"""
             )
             # Output history is optional, but ensure the table's there so it can be
             # enabled later.
             self.db.execute(
                 """CREATE TABLE IF NOT EXISTS output_history
                             (session integer, line integer, output text,
                             PRIMARY KEY (session, line))"""
             )
         # success! reset corrupt db count
         self._corrupt_db_counter = 0
 
     def writeout_cache(self) -> None:
         """Overridden by HistoryManager to dump the cache before certain
         database lookups."""
         pass
 
     ## -------------------------------
     ## Methods for retrieving history:
     ## -------------------------------
     def _run_sql(
         self,
         sql: str,
         params: typing.Tuple,
         raw: bool = True,
         output: bool = False,
         latest: bool = False,
     ) -> Iterable[Tuple[int, int, InOrInOut]]:
         """Prepares and runs an SQL query for the history database.
 
         Parameters
         ----------
         sql : str
             Any filtering expressions to go after SELECT ... FROM ...
         params : tuple
             Parameters passed to the SQL query (to replace "?")
         raw, output : bool
             See :meth:`get_range`
         latest : bool
             Select rows with max (session, line)
 
         Returns
         -------
         Tuples as :meth:`get_range`
         """
         toget = "source_raw" if raw else "source"
         sqlfrom = "history"
         if output:
             sqlfrom = "history LEFT JOIN output_history USING (session, line)"
             toget = "history.%s, output_history.output" % toget
         if latest:
             toget += ", MAX(session * 128 * 1024 + line)"
         this_querry = "SELECT session, line, %s FROM %s " % (toget, sqlfrom) + sql
         cur = self.db.execute(this_querry, params)
         if latest:
             cur = (row[:-1] for row in cur)
         if output:  # Regroup into 3-tuples, and parse JSON
             return ((ses, lin, (inp, out)) for ses, lin, inp, out in cur)
         return cur
 
     @only_when_enabled
     @catch_corrupt_db
     def get_session_info(
         self, session: int
     ) -> Tuple[int, datetime.datetime, Optional[datetime.datetime], Optional[int], str]:
         """Get info about a session.
 
         Parameters
         ----------
         session : int
             Session number to retrieve.
 
         Returns
         -------
         session_id : int
             Session ID number
         start : datetime
             Timestamp for the start of the session.
         end : datetime
             Timestamp for the end of the session, or None if IPython crashed.
         num_cmds : int
             Number of commands run, or None if IPython crashed.
         remark : str
             A manually set description.
         """
         query = "SELECT * from sessions where session == ?"
         return self.db.execute(query, (session,)).fetchone()
 
     @catch_corrupt_db
     def get_last_session_id(self) -> Optional[int]:
         """Get the last session ID currently in the database.
 
         Within IPython, this should be the same as the value stored in
         :attr:`HistoryManager.session_number`.
         """
         for record in self.get_tail(n=1, include_latest=True):
             return record[0]
         return None
 
     @catch_corrupt_db
     def get_tail(
         self,
         n: int = 10,
         raw: bool = True,
         output: bool = False,
         include_latest: bool = False,
     ) -> Iterable[Tuple[int, int, InOrInOut]]:
         """Get the last n lines from the history database.
 
         Parameters
         ----------
         n : int
             The number of lines to get
         raw, output : bool
             See :meth:`get_range`
         include_latest : bool
             If False (default), n+1 lines are fetched, and the latest one
             is discarded. This is intended to be used where the function
             is called by a user command, which it should not return.
 
         Returns
         -------
         Tuples as :meth:`get_range`
         """
         self.writeout_cache()
         if not include_latest:
             n += 1
         cur = self._run_sql(
             "ORDER BY session DESC, line DESC LIMIT ?", (n,), raw=raw, output=output
         )
         if not include_latest:
             return reversed(list(cur)[1:])
         return reversed(list(cur))
 
     @catch_corrupt_db
     def search(
         self,
         pattern: str = "*",
         raw: bool = True,
         search_raw: bool = True,
         output: bool = False,
         n: Optional[int] = None,
         unique: bool = False,
     ) -> Iterable[Tuple[int, int, InOrInOut]]:
         """Search the database using unix glob-style matching (wildcards
         * and ?).
 
         Parameters
         ----------
         pattern : str
             The wildcarded pattern to match when searching
         search_raw : bool
             If True, search the raw input, otherwise, the parsed input
         raw, output : bool
             See :meth:`get_range`
         n : None or int
             If an integer is given, it defines the limit of
             returned entries.
         unique : bool
             When it is true, return only unique entries.
 
         Returns
         -------
         Tuples as :meth:`get_range`
         """
         tosearch = "source_raw" if search_raw else "source"
         if output:
             tosearch = "history." + tosearch
         self.writeout_cache()
         sqlform = "WHERE %s GLOB ?" % tosearch
         params: typing.Tuple[typing.Any, ...] = (pattern,)
         if unique:
             sqlform += " GROUP BY {0}".format(tosearch)
         if n is not None:
             sqlform += " ORDER BY session DESC, line DESC LIMIT ?"
             params += (n,)
         elif unique:
             sqlform += " ORDER BY session, line"
         cur = self._run_sql(sqlform, params, raw=raw, output=output, latest=unique)
         if n is not None:
             return reversed(list(cur))
         return cur
 
     @catch_corrupt_db
     def get_range(
         self,
         session: int,
         start: int = 1,
         stop: Optional[int] = None,
         raw: bool = True,
         output: bool = False,
     ) -> Iterable[Tuple[int, int, InOrInOut]]:
         """Retrieve input by session.
 
         Parameters
         ----------
         session : int
             Session number to retrieve.
         start : int
             First line to retrieve.
         stop : int
             End of line range (excluded from output itself). If None, retrieve
             to the end of the session.
         raw : bool
             If True, return untranslated input
         output : bool
             If True, attempt to include output. This will be 'real' Python
             objects for the current session, or text reprs from previous
             sessions if db_log_output was enabled at the time. Where no output
             is found, None is used.
 
         Returns
         -------
         entries
             An iterator over the desired lines. Each line is a 3-tuple, either
             (session, line, input) if output is False, or
             (session, line, (input, output)) if output is True.
         """
         params: typing.Tuple[typing.Any, ...]
         if stop:
             lineclause = "line >= ? AND line < ?"
             params = (session, start, stop)
         else:
             lineclause = "line>=?"
             params = (session, start)
 
         return self._run_sql(
             "WHERE session==? AND %s" % lineclause, params, raw=raw, output=output
         )
 
     def get_range_by_str(
         self, rangestr: str, raw: bool = True, output: bool = False
     ) -> Iterable[Tuple[int, int, InOrInOut]]:
         """Get lines of history from a string of ranges, as used by magic
         commands %hist, %save, %macro, etc.
 
         Parameters
         ----------
         rangestr : str
             A string specifying ranges, e.g. "5 ~2/1-4". If empty string is used,
             this will return everything from current session's history.
 
             See the documentation of :func:`%history` for the full details.
 
         raw, output : bool
             As :meth:`get_range`
 
         Returns
         -------
         Tuples as :meth:`get_range`
         """
         for sess, s, e in extract_hist_ranges(rangestr):
             for line in self.get_range(sess, s, e, raw=raw, output=output):
                 yield line
 
 
 class HistoryManager(HistoryAccessor):
     """A class to organize all history-related functionality in one place."""
 
     # Public interface
 
     # An instance of the IPython shell we are attached to
     shell = Instance(
         "IPython.core.interactiveshell.InteractiveShellABC", allow_none=False
     )
     # Lists to hold processed and raw history. These start with a blank entry
     # so that we can index them starting from 1
     input_hist_parsed = List([""])
     input_hist_raw = List([""])
     # A list of directories visited during session
     dir_hist: List = List()
 
     @default("dir_hist")
     def _dir_hist_default(self) -> typing.List[Path]:
         try:
             return [Path.cwd()]
         except OSError:
             return []
 
     # A dict of output history, keyed with ints from the shell's
     # execution count.
     output_hist = Dict()
     # The text/plain repr of outputs.
     output_hist_reprs: typing.Dict[int, str] = Dict()  # type: ignore [assignment]
 
     # The number of the current session in the history database
     session_number: int = Integer()  # type: ignore [assignment]
 
     db_log_output = Bool(
         False, help="Should the history database include output? (default: no)"
     ).tag(config=True)
     db_cache_size = Integer(
         0,
         help="Write to database every x commands (higher values save disk access & power).\n"
         "Values of 1 or less effectively disable caching.",
     ).tag(config=True)
     # The input and output caches
     db_input_cache: List[Tuple[int, str, str]] = List()
     db_output_cache: List[Tuple[int, str]] = List()
 
     # History saving in separate thread
     save_thread = Instance("IPython.core.history.HistorySavingThread", allow_none=True)
     save_flag = Instance(threading.Event, allow_none=False)
 
     # Private interface
     # Variables used to store the three last inputs from the user.  On each new
     # history update, we populate the user's namespace with these, shifted as
     # necessary.
     _i00 = Unicode("")
     _i = Unicode("")
     _ii = Unicode("")
     _iii = Unicode("")
 
     # A regex matching all forms of the exit command, so that we don't store
     # them in the history (it's annoying to rewind the first entry and land on
     # an exit call).
     _exit_re = re.compile(r"(exit|quit)(\s*\(.*\))?$")
 
     def __init__(
         self,
         shell: InteractiveShell,
         config: Optional[Configuration] = None,
         **traits: typing.Any,
     ):
         """Create a new history manager associated with a shell instance."""
         super().__init__(shell=shell, config=config, **traits)
         self.save_flag = threading.Event()
         self.db_input_cache_lock = threading.Lock()
         self.db_output_cache_lock = threading.Lock()
 
         try:
             self.new_session()
         except OperationalError:
             self.log.error(
                 "Failed to create history session in %s. History will not be saved.",
                 self.hist_file,
                 exc_info=True,
             )
             self.hist_file = ":memory:"
 
         if self.enabled and self.hist_file != ":memory:":
             self.save_thread = HistorySavingThread(self)
             try:
                 self.save_thread.start()
             except RuntimeError:
                 self.log.error(
                     "Failed to start history saving thread. History will not be saved.",
                     exc_info=True,
                 )
                 self.hist_file = ":memory:"
 
     def _get_hist_file_name(self, profile: Optional[str] = None) -> Path:
         """Get default history file name based on the Shell's profile.
 
         The profile parameter is ignored, but must exist for compatibility with
         the parent class."""
         profile_dir = self.shell.profile_dir.location
         return Path(profile_dir) / "history.sqlite"
 
     @only_when_enabled
     def new_session(self, conn: Optional[sqlite3.Connection] = None) -> None:
         """Get a new session number."""
         if conn is None:
             conn = self.db
 
         with conn:
             cur = conn.execute(
                 """INSERT INTO sessions VALUES (NULL, ?, NULL,
                             NULL, '') """,
                 (datetime.datetime.now().isoformat(" "),),
             )
             assert isinstance(cur.lastrowid, int)
             self.session_number = cur.lastrowid
 
     def end_session(self) -> None:
         """Close the database session, filling in the end time and line count."""
         self.writeout_cache()
         with self.db:
             self.db.execute(
                 """UPDATE sessions SET end=?, num_cmds=? WHERE
                             session==?""",
                 (
                     datetime.datetime.now().isoformat(" "),
                     len(self.input_hist_parsed) - 1,
                     self.session_number,
                 ),
             )
         self.session_number = 0
 
     def name_session(self, name: str) -> None:
         """Give the current session a name in the history database."""
         warn(
             "name_session is deprecated in IPython 9.0 and will be removed in future versions",
             DeprecationWarning,
             stacklevel=2,
         )
         with self.db:
             self.db.execute(
                 "UPDATE sessions SET remark=? WHERE session==?",
                 (name, self.session_number),
             )
 
     def reset(self, new_session: bool = True) -> None:
         """Clear the session history, releasing all object references, and
         optionally open a new session."""
         self.output_hist.clear()
         # The directory history can't be completely empty
         self.dir_hist[:] = [Path.cwd()]
 
         if new_session:
             if self.session_number:
                 self.end_session()
             self.input_hist_parsed[:] = [""]
             self.input_hist_raw[:] = [""]
             self.new_session()
 
     # ------------------------------
     # Methods for retrieving history
     # ------------------------------
     def get_session_info(
         self, session: int = 0
     ) -> Tuple[int, datetime.datetime, Optional[datetime.datetime], Optional[int], str]:
         """Get info about a session.
 
         Parameters
         ----------
         session : int
             Session number to retrieve. The current session is 0, and negative
             numbers count back from current session, so -1 is the previous session.
 
         Returns
         -------
         session_id : int
             Session ID number
         start : datetime
             Timestamp for the start of the session.
         end : datetime
             Timestamp for the end of the session, or None if IPython crashed.
         num_cmds : int
             Number of commands run, or None if IPython crashed.
         remark : str
             A manually set description.
         """
         if session <= 0:
             session += self.session_number
 
         return super(HistoryManager, self).get_session_info(session=session)
 
     @catch_corrupt_db
     def get_tail(
         self,
         n: int = 10,
         raw: bool = True,
         output: bool = False,
         include_latest: bool = False,
     ) -> Iterable[Tuple[int, int, InOrInOut]]:
         """Get the last n lines from the history database.
 
         Most recent entry last.
 
         Completion will be reordered so that that the last ones are when
         possible from current session.
 
         Parameters
         ----------
         n : int
             The number of lines to get
         raw, output : bool
             See :meth:`get_range`
         include_latest : bool
             If False (default), n+1 lines are fetched, and the latest one
             is discarded. This is intended to be used where the function
             is called by a user command, which it should not return.
 
         Returns
         -------
         Tuples as :meth:`get_range`
         """
         self.writeout_cache()
         if not include_latest:
             n += 1
         # cursor/line/entry
         this_cur = list(
             self._run_sql(
                 "WHERE session == ? ORDER BY line DESC LIMIT ?  ",
                 (self.session_number, n),
                 raw=raw,
                 output=output,
             )
         )
         other_cur = list(
             self._run_sql(
                 "WHERE session != ? ORDER BY session DESC, line DESC LIMIT ?",
                 (self.session_number, n),
                 raw=raw,
                 output=output,
             )
         )
 
         everything: typing.List[Tuple[int, int, InOrInOut]] = this_cur + other_cur
 
         everything = everything[:n]
 
         if not include_latest:
             return list(everything)[:0:-1]
         return list(everything)[::-1]
 
     def _get_range_session(
         self,
         start: int = 1,
         stop: Optional[int] = None,
         raw: bool = True,
         output: bool = False,
     ) -> Iterable[Tuple[int, int, InOrInOut]]:
         """Get input and output history from the current session. Called by
         get_range, and takes similar parameters."""
         input_hist = self.input_hist_raw if raw else self.input_hist_parsed
 
         n = len(input_hist)
         if start < 0:
             start += n
         if not stop or (stop > n):
             stop = n
         elif stop < 0:
             stop += n
         line: InOrInOut
         for i in range(start, stop):
             if output:
                 line = (input_hist[i], self.output_hist_reprs.get(i))
             else:
                 line = input_hist[i]
             yield (0, i, line)
 
     def get_range(
         self,
         session: int = 0,
         start: int = 1,
         stop: Optional[int] = None,
         raw: bool = True,
         output: bool = False,
     ) -> Iterable[Tuple[int, int, InOrInOut]]:
         """Retrieve input by session.
 
         Parameters
         ----------
         session : int
             Session number to retrieve. The current session is 0, and negative
             numbers count back from current session, so -1 is previous session.
         start : int
             First line to retrieve.
         stop : int
             End of line range (excluded from output itself). If None, retrieve
             to the end of the session.
         raw : bool
             If True, return untranslated input
         output : bool
             If True, attempt to include output. This will be 'real' Python
             objects for the current session, or text reprs from previous
             sessions if db_log_output was enabled at the time. Where no output
             is found, None is used.
 
         Returns
         -------
         entries
             An iterator over the desired lines. Each line is a 3-tuple, either
             (session, line, input) if output is False, or
             (session, line, (input, output)) if output is True.
         """
         if session <= 0:
             session += self.session_number
         if session == self.session_number:  # Current session
             return self._get_range_session(start, stop, raw, output)
         return super(HistoryManager, self).get_range(session, start, stop, raw, output)
 
     ## ----------------------------
     ## Methods for storing history:
     ## ----------------------------
     def store_inputs(
         self, line_num: int, source: str, source_raw: Optional[str] = None
     ) -> None:
         """Store source and raw input in history and create input cache
         variables ``_i*``.
 
         Parameters
         ----------
         line_num : int
             The prompt number of this input.
         source : str
             Python input.
         source_raw : str, optional
             If given, this is the raw input without any IPython transformations
             applied to it.  If not given, ``source`` is used.
         """
         if source_raw is None:
             source_raw = source
         source = source.rstrip("\n")
         source_raw = source_raw.rstrip("\n")
 
         # do not store exit/quit commands
         if self._exit_re.match(source_raw.strip()):
             return
 
         self.input_hist_parsed.append(source)
         self.input_hist_raw.append(source_raw)
 
         with self.db_input_cache_lock:
             self.db_input_cache.append((line_num, source, source_raw))
             # Trigger to flush cache and write to DB.
             if len(self.db_input_cache) >= self.db_cache_size:
                 self.save_flag.set()
 
         # update the auto _i variables
         self._iii = self._ii
         self._ii = self._i
         self._i = self._i00
         self._i00 = source_raw
 
         # hackish access to user namespace to create _i1,_i2... dynamically
         new_i = "_i%s" % line_num
         to_main = {"_i": self._i, "_ii": self._ii, "_iii": self._iii, new_i: self._i00}
 
         if self.shell is not None:
             self.shell.push(to_main, interactive=False)
 
     def store_output(self, line_num: int) -> None:
         """If database output logging is enabled, this saves all the
         outputs from the indicated prompt number to the database. It's
         called by run_cell after code has been executed.
 
         Parameters
         ----------
         line_num : int
             The line number from which to save outputs
         """
         if (not self.db_log_output) or (line_num not in self.output_hist_reprs):
             return
+        lnum: int = line_num
         output = self.output_hist_reprs[line_num]
 
         with self.db_output_cache_lock:
             self.db_output_cache.append((line_num, output))
         if self.db_cache_size <= 1:
             self.save_flag.set()
 
     def _writeout_input_cache(self, conn: sqlite3.Connection) -> None:
         with conn:
             for line in self.db_input_cache:
                 conn.execute(
                     "INSERT INTO history VALUES (?, ?, ?, ?)",
                     (self.session_number,) + line,
                 )
 
     def _writeout_output_cache(self, conn: sqlite3.Connection) -> None:
         with conn:
             for line in self.db_output_cache:
                 conn.execute(
                     "INSERT INTO output_history VALUES (?, ?, ?)",
                     (self.session_number,) + line,
                 )
 
     @only_when_enabled
     def writeout_cache(self, conn: Optional[sqlite3.Connection] = None) -> None:
         """Write any entries in the cache to the database."""
         if conn is None:
             conn = self.db
 
         with self.db_input_cache_lock:
             try:
                 self._writeout_input_cache(conn)
             except sqlite3.IntegrityError:
                 self.new_session(conn)
                 print(
                     "ERROR! Session/line number was not unique in",
                     "database. History logging moved to new session",
                     self.session_number,
                 )
                 try:
                     # Try writing to the new session. If this fails, don't
                     # recurse
                     self._writeout_input_cache(conn)
                 except sqlite3.IntegrityError:
                     pass
             finally:
                 self.db_input_cache = []
 
         with self.db_output_cache_lock:
             try:
                 self._writeout_output_cache(conn)
             except sqlite3.IntegrityError:
                 print(
                     "!! Session/line number for output was not unique",
                     "in database. Output will not be stored.",
                 )
             finally:
                 self.db_output_cache = []
 
 
 class HistorySavingThread(threading.Thread):
     """This thread takes care of writing history to the database, so that
     the UI isn't held up while that happens.
 
     It waits for the HistoryManager's save_flag to be set, then writes out
     the history cache. The main thread is responsible for setting the flag when
     the cache size reaches a defined threshold."""
 
     daemon = True
     stop_now = False
     enabled = True
     history_manager: HistoryManager
 
     def __init__(self, history_manager: HistoryManager) -> None:
         super(HistorySavingThread, self).__init__(name="IPythonHistorySavingThread")
         self.history_manager = history_manager
         self.enabled = history_manager.enabled
 
     @only_when_enabled
     def run(self) -> None:
         atexit.register(self.stop)
         # We need a separate db connection per thread:
         try:
             self.db = sqlite3.connect(
                 str(self.history_manager.hist_file),
                 **self.history_manager.connection_options,
             )
             while True:
                 self.history_manager.save_flag.wait()
                 if self.stop_now:
                     self.db.close()
                     return
                 self.history_manager.save_flag.clear()
                 self.history_manager.writeout_cache(self.db)
         except Exception as e:
             print(
                 (
                     "The history saving thread hit an unexpected error (%s)."
                     "History will not be written to the database."
                 )
                 % repr(e)
             )
         finally:
             atexit.unregister(self.stop)
 
     def stop(self) -> None:
         """This can be called from the main thread to safely stop this thread.
 
         Note that it does not attempt to write out remaining history before
         exiting. That should be done by calling the HistoryManager's
         end_session method."""
         self.stop_now = True
         self.history_manager.save_flag.set()
         self.join()
 
 
 # To match, e.g. ~5/8-~2/3
 range_re = re.compile(
     r"""
 ((?P<startsess>~?\d+)/)?
 (?P<start>\d+)?
 ((?P<sep>[\-:])
  ((?P<endsess>~?\d+)/)?
  (?P<end>\d+))?
 $""",
     re.VERBOSE,
 )
 
 
 def extract_hist_ranges(ranges_str: str) -> Iterable[Tuple[int, int, Optional[int]]]:
     """Turn a string of history ranges into 3-tuples of (session, start, stop).
 
     Empty string results in a `[(0, 1, None)]`, i.e. "everything from current
     session".
 
     Examples
     --------
     >>> list(extract_hist_ranges("~8/5-~7/4 2"))
     [(-8, 5, None), (-7, 1, 5), (0, 2, 3)]
     """
     if ranges_str == "":
         yield (0, 1, None)  # Everything from current session
         return
 
     for range_str in ranges_str.split():
         rmatch = range_re.match(range_str)
         if not rmatch:
             continue
         start = rmatch.group("start")
         if start:
             start = int(start)
             end = rmatch.group("end")
             # If no end specified, get (a, a + 1)
             end = int(end) if end else start + 1
         else:  # start not specified
             if not rmatch.group("startsess"):  # no startsess
                 continue
             start = 1
             end = None  # provide the entire session hist
 
         if rmatch.group("sep") == "-":  # 1-3 == 1:4 --> [1, 2, 3]
             assert end is not None
             end += 1
         startsess = rmatch.group("startsess") or "0"
         endsess = rmatch.group("endsess") or startsess
         startsess = int(startsess.replace("~", "-"))
         endsess = int(endsess.replace("~", "-"))
         assert endsess >= startsess, "start session must be earlier than end session"
 
         if endsess == startsess:
             yield (startsess, start, end)
             continue
         # Multiple sessions in one range:
         yield (startsess, start, None)
         for sess in range(startsess + 1, endsess):
             yield (sess, 1, None)
         yield (endsess, 1, end)
 
 
 def _format_lineno(session: int, line: int) -> str:
     """Helper function to format line numbers properly."""
     if session == 0:
         return str(line)
     return "%s#%s" % (session, line)

IPython/core/interactiveshell.py

@@ -1,3984 +1,3985
 # -*- coding: utf-8 -*-
 """Main IPython class."""
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2001 Janko Hauser <[e-mail]>
 #  Copyright (C) 2001-2007 Fernando Perez. <[e-mail]>
 #  Copyright (C) 2008-2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 
 import abc
 import ast
 import atexit
 import bdb
 import builtins as builtin_mod
 import functools
 import inspect
 import os
 import re
 import runpy
 import shutil
 import subprocess
 import sys
 import tempfile
 import traceback
 import types
 import warnings
 from ast import stmt
 from io import open as io_open
 from logging import error
 from pathlib import Path
 from typing import Callable
-from typing import List as ListType, Any as AnyType
+from typing import List as ListType, Dict as DictType, Any as AnyType
 from typing import Optional, Sequence, Tuple
 from warnings import warn
 
 try:
     from pickleshare import PickleShareDB
 except ModuleNotFoundError:
 
     class PickleShareDB:  # type: ignore [no-redef]
         _mock = True
 
         def __init__(self, path):
             pass
 
         def get(self, key, default=None):
             warn(
                 f"This is now an optional IPython functionality, using {key} requires you to install the `pickleshare` library.",
                 stacklevel=2,
             )
             return default
 
         def __getitem__(self, key):
             warn(
                 f"This is now an optional IPython functionality, using {key} requires you to install the `pickleshare` library.",
                 stacklevel=2,
             )
             return None
 
         def __setitem__(self, key, value):
             warn(
                 f"This is now an optional IPython functionality, setting {key} requires you to install the `pickleshare` library.",
                 stacklevel=2,
             )
 
         def __delitem__(self, key):
             warn(
                 f"This is now an optional IPython functionality, deleting {key} requires you to install the `pickleshare` library.",
                 stacklevel=2,
             )
 
 
 from tempfile import TemporaryDirectory
 from traitlets import (
     Any,
     Bool,
     CaselessStrEnum,
     Dict,
     Enum,
     Instance,
     Integer,
     List,
     Type,
     Unicode,
     default,
     observe,
     validate,
 )
 from traitlets.config.configurable import SingletonConfigurable
 from traitlets.utils.importstring import import_item
 
 import IPython.core.hooks
 from IPython.core import magic, oinspect, page, prefilter, ultratb
 from IPython.core.alias import Alias, AliasManager
 from IPython.core.autocall import ExitAutocall
 from IPython.core.builtin_trap import BuiltinTrap
 from IPython.core.compilerop import CachingCompiler
 from IPython.core.debugger import InterruptiblePdb
 from IPython.core.display_trap import DisplayTrap
 from IPython.core.displayhook import DisplayHook
 from IPython.core.displaypub import DisplayPublisher
 from IPython.core.error import InputRejected, UsageError
 from IPython.core.events import EventManager, available_events
 from IPython.core.extensions import ExtensionManager
 from IPython.core.formatters import DisplayFormatter
 from IPython.core.history import HistoryManager
 from IPython.core.inputtransformer2 import ESC_MAGIC, ESC_MAGIC2
 from IPython.core.logger import Logger
 from IPython.core.macro import Macro
 from IPython.core.payload import PayloadManager
 from IPython.core.prefilter import PrefilterManager
 from IPython.core.profiledir import ProfileDir
 from IPython.core.usage import default_banner
 from IPython.display import display
 from IPython.paths import get_ipython_dir
 from IPython.testing.skipdoctest import skip_doctest
-from IPython.utils import PyColorize, openpy, py3compat
+from IPython.utils import PyColorize, io, openpy, py3compat
 from IPython.utils.decorators import undoc
 from IPython.utils.io import ask_yes_no
 from IPython.utils.ipstruct import Struct
 from IPython.utils.path import ensure_dir_exists, get_home_dir, get_py_filename
 from IPython.utils.process import getoutput, system
 from IPython.utils.strdispatch import StrDispatch
 from IPython.utils.syspathcontext import prepended_to_syspath
 from IPython.utils.text import DollarFormatter, LSString, SList, format_screen
 from IPython.core.oinspect import OInfo
 
-from ast import Module
-
-# we still need to run things using the asyncio eventloop, but there is no
-# async integration
-
-from .async_helpers import (
-    _asyncio_runner,
-    _curio_runner,
-    _pseudo_sync_runner,
-    _should_be_async,
-    _trio_runner,
-)
 
 sphinxify: Optional[Callable]
 
 try:
     import docrepr.sphinxify as sphx
 
     def sphinxify(oinfo):
         wrapped_docstring = sphx.wrap_main_docstring(oinfo)
 
         def sphinxify_docstring(docstring):
             with TemporaryDirectory() as dirname:
                 return {
                     "text/html": sphx.sphinxify(wrapped_docstring, dirname),
                     "text/plain": docstring,
                 }
 
         return sphinxify_docstring
 except ImportError:
     sphinxify = None
 
 
 class ProvisionalWarning(DeprecationWarning):
     """
     Warning class for unstable features
     """
     pass
 
+from ast import Module
 
 _assign_nodes = (ast.AugAssign, ast.AnnAssign, ast.Assign)
 _single_targets_nodes = (ast.AugAssign, ast.AnnAssign)
 
 #-----------------------------------------------------------------------------
+# Await Helpers
+#-----------------------------------------------------------------------------
+
+# we still need to run things using the asyncio eventloop, but there is no
+# async integration
+from .async_helpers import (
+    _asyncio_runner,
+    _curio_runner,
+    _pseudo_sync_runner,
+    _should_be_async,
+    _trio_runner,
+)
+
+#-----------------------------------------------------------------------------
 # Globals
 #-----------------------------------------------------------------------------
 
 # compiled regexps for autoindent management
 dedent_re = re.compile(r'^\s+raise|^\s+return|^\s+pass')
 
 #-----------------------------------------------------------------------------
 # Utilities
 #-----------------------------------------------------------------------------
 
 
 def is_integer_string(s: str):
     """
     Variant of "str.isnumeric()" that allow negative values and other ints.
     """
     try:
         int(s)
         return True
     except ValueError:
         return False
     raise ValueError("Unexpected error")
 
 
 @undoc
 def softspace(file, newvalue):
     """Copied from code.py, to remove the dependency"""
 
     oldvalue = 0
     try:
         oldvalue = file.softspace
     except AttributeError:
         pass
     try:
         file.softspace = newvalue
     except (AttributeError, TypeError):
         # "attribute-less object" or "read-only attributes"
         pass
     return oldvalue
 
 @undoc
 def no_op(*a, **kw):
     pass
 
 
-class SpaceInInput(Exception):
-    pass
+class SpaceInInput(Exception): pass
 
 
 class SeparateUnicode(Unicode):
     r"""A Unicode subclass to validate separate_in, separate_out, etc.
 
     This is a Unicode based trait that converts '0'->'' and ``'\\n'->'\n'``.
     """
 
     def validate(self, obj, value):
-        if value == "0":
-            value = ""
-        value = value.replace("\\n", "\n")
+        if value == '0': value = ''
+        value = value.replace('\\n','\n')
         return super(SeparateUnicode, self).validate(obj, value)
 
 
 @undoc
 class DummyMod(object):
     """A dummy module used for IPython's interactive module when
     a namespace must be assigned to the module's __dict__."""
     __spec__ = None
 
 
 class ExecutionInfo(object):
     """The arguments used for a call to :meth:`InteractiveShell.run_cell`
 
     Stores information about what is going to happen.
     """
     raw_cell = None
     store_history = False
     silent = False
     shell_futures = True
     cell_id = None
 
     def __init__(self, raw_cell, store_history, silent, shell_futures, cell_id):
         self.raw_cell = raw_cell
         self.store_history = store_history
         self.silent = silent
         self.shell_futures = shell_futures
         self.cell_id = cell_id
 
     def __repr__(self):
         name = self.__class__.__qualname__
         raw_cell = (
             (self.raw_cell[:50] + "..") if len(self.raw_cell) > 50 else self.raw_cell
         )
         return (
             '<%s object at %x, raw_cell="%s" store_history=%s silent=%s shell_futures=%s cell_id=%s>'
             % (
                 name,
                 id(self),
                 raw_cell,
                 self.store_history,
                 self.silent,
                 self.shell_futures,
                 self.cell_id,
             )
         )
 
 
 class ExecutionResult:
     """The result of a call to :meth:`InteractiveShell.run_cell`
 
     Stores information about what took place.
     """
 
     execution_count: Optional[int] = None
     error_before_exec: Optional[bool] = None
     error_in_exec: Optional[BaseException] = None
     info = None
     result = None
 
     def __init__(self, info):
         self.info = info
 
     @property
     def success(self):
         return (self.error_before_exec is None) and (self.error_in_exec is None)
 
     def raise_error(self):
         """Reraises error if `success` is `False`, otherwise does nothing"""
         if self.error_before_exec is not None:
             raise self.error_before_exec
         if self.error_in_exec is not None:
             raise self.error_in_exec
 
     def __repr__(self):
         name = self.__class__.__qualname__
         return '<%s object at %x, execution_count=%s error_before_exec=%s error_in_exec=%s info=%s result=%s>' %\
                 (name, id(self), self.execution_count, self.error_before_exec, self.error_in_exec, repr(self.info), repr(self.result))
 
 @functools.wraps(io_open)
 def _modified_open(file, *args, **kwargs):
     if file in {0, 1, 2}:
         raise ValueError(
             f"IPython won't let you open fd={file} by default "
             "as it is likely to crash IPython. If you know what you are doing, "
             "you can use builtins' open."
         )
 
     return io_open(file, *args, **kwargs)
 
 class InteractiveShell(SingletonConfigurable):
     """An enhanced, interactive shell for Python."""
 
     _instance = None
 
     ast_transformers: List[ast.NodeTransformer] = List(
         [],
         help="""
         A list of ast.NodeTransformer subclass instances, which will be applied
         to user input before code is run.
         """,
     ).tag(config=True)
 
     autocall = Enum((0,1,2), default_value=0, help=
         """
         Make IPython automatically call any callable object even if you didn't
         type explicit parentheses. For example, 'str 43' becomes 'str(43)'
         automatically. The value can be '0' to disable the feature, '1' for
         'smart' autocall, where it is not applied if there are no more
         arguments on the line, and '2' for 'full' autocall, where all callable
         objects are automatically called (even if no arguments are present).
         """
     ).tag(config=True)
 
     autoindent = Bool(True, help=
         """
         Autoindent IPython code entered interactively.
         """
     ).tag(config=True)
 
     autoawait = Bool(True, help=
         """
         Automatically run await statement in the top level repl.
         """
     ).tag(config=True)
 
     loop_runner_map ={
         'asyncio':(_asyncio_runner, True),
         'curio':(_curio_runner, True),
         'trio':(_trio_runner, True),
         'sync': (_pseudo_sync_runner, False)
     }
 
     loop_runner = Any(default_value="IPython.core.interactiveshell._asyncio_runner",
         allow_none=True,
         help="""Select the loop runner that will be used to execute top-level asynchronous code"""
     ).tag(config=True)
 
     @default('loop_runner')
     def _default_loop_runner(self):
         return import_item("IPython.core.interactiveshell._asyncio_runner")
 
     @validate('loop_runner')
     def _import_runner(self, proposal):
         if isinstance(proposal.value, str):
             if proposal.value in self.loop_runner_map:
                 runner, autoawait = self.loop_runner_map[proposal.value]
                 self.autoawait = autoawait
                 return runner
             runner = import_item(proposal.value)
             if not callable(runner):
                 raise ValueError('loop_runner must be callable')
             return runner
         if not callable(proposal.value):
             raise ValueError('loop_runner must be callable')
         return proposal.value
 
     automagic = Bool(True, help=
         """
         Enable magic commands to be called without the leading %.
         """
     ).tag(config=True)
 
     banner1 = Unicode(default_banner,
         help="""The part of the banner to be printed before the profile"""
     ).tag(config=True)
     banner2 = Unicode('',
         help="""The part of the banner to be printed after the profile"""
     ).tag(config=True)
 
     cache_size = Integer(1000, help=
         """
         Set the size of the output cache.  The default is 1000, you can
         change it permanently in your config file.  Setting it to 0 completely
         disables the caching system, and the minimum value accepted is 3 (if
         you provide a value less than 3, it is reset to 0 and a warning is
         issued).  This limit is defined because otherwise you'll spend more
         time re-flushing a too small cache than working
         """
     ).tag(config=True)
     color_info = Bool(True, help=
         """
         Use colors for displaying information about objects. Because this
         information is passed through a pager (like 'less'), and some pagers
         get confused with color codes, this capability can be turned off.
         """
     ).tag(config=True)
     colors = CaselessStrEnum(('Neutral', 'NoColor','LightBG','Linux'),
                              default_value='Neutral',
         help="Set the color scheme (NoColor, Neutral, Linux, or LightBG)."
     ).tag(config=True)
     debug = Bool(False).tag(config=True)
     disable_failing_post_execute = Bool(False,
         help="Don't call post-execute functions that have failed in the past."
     ).tag(config=True)
     display_formatter = Instance(DisplayFormatter, allow_none=True)
     displayhook_class = Type(DisplayHook)
     display_pub_class = Type(DisplayPublisher)
     compiler_class = Type(CachingCompiler)
     inspector_class = Type(
         oinspect.Inspector, help="Class to use to instantiate the shell inspector"
     ).tag(config=True)
 
     sphinxify_docstring = Bool(False, help=
         """
         Enables rich html representation of docstrings. (This requires the
         docrepr module).
         """).tag(config=True)
 
     @observe("sphinxify_docstring")
     def _sphinxify_docstring_changed(self, change):
         if change['new']:
             warn("`sphinxify_docstring` is provisional since IPython 5.0 and might change in future versions." , ProvisionalWarning)
 
     enable_html_pager = Bool(False, help=
         """
         (Provisional API) enables html representation in mime bundles sent
         to pagers.
         """).tag(config=True)
 
     @observe("enable_html_pager")
     def _enable_html_pager_changed(self, change):
         if change['new']:
             warn("`enable_html_pager` is provisional since IPython 5.0 and might change in future versions.", ProvisionalWarning)
 
     data_pub_class = None
 
     exit_now = Bool(False)
     exiter = Instance(ExitAutocall)
     @default('exiter')
     def _exiter_default(self):
         return ExitAutocall(self)
     # Monotonically increasing execution counter
     execution_count = Integer(1)
     filename = Unicode("<ipython console>")
     ipython_dir= Unicode('').tag(config=True) # Set to get_ipython_dir() in __init__
 
     # Used to transform cells before running them, and check whether code is complete
     input_transformer_manager = Instance('IPython.core.inputtransformer2.TransformerManager',
                                          ())
 
     @property
     def input_transformers_cleanup(self):
         return self.input_transformer_manager.cleanup_transforms
 
     input_transformers_post: List = List(
         [],
         help="A list of string input transformers, to be applied after IPython's "
              "own input transformations."
     )
 
     @property
     def input_splitter(self):
         """Make this available for backward compatibility (pre-7.0 release) with existing code.
 
         For example, ipykernel ipykernel currently uses
         `shell.input_splitter.check_complete`
         """
         from warnings import warn
         warn("`input_splitter` is deprecated since IPython 7.0, prefer `input_transformer_manager`.",
              DeprecationWarning, stacklevel=2
         )
         return self.input_transformer_manager
 
     logstart = Bool(False, help=
         """
         Start logging to the default log file in overwrite mode.
         Use `logappend` to specify a log file to **append** logs to.
         """
     ).tag(config=True)
     logfile = Unicode('', help=
         """
         The name of the logfile to use.
         """
     ).tag(config=True)
     logappend = Unicode('', help=
         """
         Start logging to the given file in append mode.
         Use `logfile` to specify a log file to **overwrite** logs to.
         """
     ).tag(config=True)
     object_info_string_level = Enum((0,1,2), default_value=0,
     ).tag(config=True)
     pdb = Bool(False, help=
         """
         Automatically call the pdb debugger after every exception.
         """
     ).tag(config=True)
     display_page = Bool(False,
         help="""If True, anything that would be passed to the pager
         will be displayed as regular output instead."""
     ).tag(config=True)
 
 
     show_rewritten_input = Bool(True,
         help="Show rewritten input, e.g. for autocall."
     ).tag(config=True)
 
     quiet = Bool(False).tag(config=True)
 
     history_length = Integer(10000,
         help='Total length of command history'
     ).tag(config=True)
 
     history_load_length = Integer(1000, help=
         """
         The number of saved history entries to be loaded
         into the history buffer at startup.
         """
     ).tag(config=True)
 
     ast_node_interactivity = Enum(['all', 'last', 'last_expr', 'none', 'last_expr_or_assign'],
                                   default_value='last_expr',
                                   help="""
         'all', 'last', 'last_expr' or 'none', 'last_expr_or_assign' specifying
         which nodes should be run interactively (displaying output from expressions).
         """
     ).tag(config=True)
 
     warn_venv = Bool(
         True,
         help="Warn if running in a virtual environment with no IPython installed (so IPython from the global environment is used).",
     ).tag(config=True)
 
     # TODO: this part of prompt management should be moved to the frontends.
     # Use custom TraitTypes that convert '0'->'' and '\\n'->'\n'
     separate_in = SeparateUnicode('\n').tag(config=True)
     separate_out = SeparateUnicode('').tag(config=True)
     separate_out2 = SeparateUnicode('').tag(config=True)
     wildcards_case_sensitive = Bool(True).tag(config=True)
     xmode = CaselessStrEnum(('Context', 'Plain', 'Verbose', 'Minimal'),
                             default_value='Context',
                             help="Switch modes for the IPython exception handlers."
                             ).tag(config=True)
 
     # Subcomponents of InteractiveShell
     alias_manager = Instance("IPython.core.alias.AliasManager", allow_none=True)
     prefilter_manager = Instance(
         "IPython.core.prefilter.PrefilterManager", allow_none=True
     )
     builtin_trap = Instance("IPython.core.builtin_trap.BuiltinTrap")
     display_trap = Instance("IPython.core.display_trap.DisplayTrap")
     extension_manager = Instance(
         "IPython.core.extensions.ExtensionManager", allow_none=True
     )
     payload_manager = Instance("IPython.core.payload.PayloadManager", allow_none=True)
     history_manager = Instance(
         "IPython.core.history.HistoryAccessorBase", allow_none=True
     )
     magics_manager = Instance("IPython.core.magic.MagicsManager")
 
     profile_dir = Instance('IPython.core.application.ProfileDir', allow_none=True)
     @property
     def profile(self):
         if self.profile_dir is not None:
             name = os.path.basename(self.profile_dir.location)
             return name.replace('profile_','')
 
 
     # Private interface
     _post_execute = Dict()
 
     # Tracks any GUI loop loaded for pylab
     pylab_gui_select = None
 
     last_execution_succeeded = Bool(True, help='Did last executed command succeeded')
 
     last_execution_result = Instance('IPython.core.interactiveshell.ExecutionResult', help='Result of executing the last command', allow_none=True)
 
     def __init__(self, ipython_dir=None, profile_dir=None,
                  user_module=None, user_ns=None,
                  custom_exceptions=((), None), **kwargs):
         # This is where traits with a config_key argument are updated
         # from the values on config.
         super(InteractiveShell, self).__init__(**kwargs)
         if 'PromptManager' in self.config:
             warn('As of IPython 5.0 `PromptManager` config will have no effect'
                  ' and has been replaced by TerminalInteractiveShell.prompts_class')
         self.configurables = [self]
 
         # These are relatively independent and stateless
         self.init_ipython_dir(ipython_dir)
         self.init_profile_dir(profile_dir)
         self.init_instance_attrs()
         self.init_environment()
 
         # Check if we're in a virtualenv, and set up sys.path.
         self.init_virtualenv()
 
         # Create namespaces (user_ns, user_global_ns, etc.)
         self.init_create_namespaces(user_module, user_ns)
         # This has to be done after init_create_namespaces because it uses
         # something in self.user_ns, but before init_sys_modules, which
         # is the first thing to modify sys.
         # TODO: When we override sys.stdout and sys.stderr before this class
         # is created, we are saving the overridden ones here. Not sure if this
         # is what we want to do.
         self.save_sys_module_state()
         self.init_sys_modules()
 
         # While we're trying to have each part of the code directly access what
         # it needs without keeping redundant references to objects, we have too
         # much legacy code that expects ip.db to exist.
         self.db = PickleShareDB(os.path.join(self.profile_dir.location, 'db'))
 
         self.init_history()
         self.init_encoding()
         self.init_prefilter()
 
         self.init_syntax_highlighting()
         self.init_hooks()
         self.init_events()
         self.init_pushd_popd_magic()
         self.init_user_ns()
         self.init_logger()
         self.init_builtins()
 
         # The following was in post_config_initialization
         self.init_inspector()
         self.raw_input_original = input
         self.init_completer()
         # TODO: init_io() needs to happen before init_traceback handlers
         # because the traceback handlers hardcode the stdout/stderr streams.
         # This logic in in debugger.Pdb and should eventually be changed.
         self.init_io()
         self.init_traceback_handlers(custom_exceptions)
         self.init_prompts()
         self.init_display_formatter()
         self.init_display_pub()
         self.init_data_pub()
         self.init_displayhook()
         self.init_magics()
         self.init_alias()
         self.init_logstart()
         self.init_pdb()
         self.init_extension_manager()
         self.init_payload()
         self.events.trigger('shell_initialized', self)
         atexit.register(self.atexit_operations)
 
         # The trio runner is used for running Trio in the foreground thread. It
         # is different from `_trio_runner(async_fn)` in `async_helpers.py`
         # which calls `trio.run()` for every cell. This runner runs all cells
         # inside a single Trio event loop. If used, it is set from
         # `ipykernel.kernelapp`.
         self.trio_runner = None
 
     def get_ipython(self):
         """Return the currently running IPython instance."""
         return self
 
     #-------------------------------------------------------------------------
     # Trait changed handlers
     #-------------------------------------------------------------------------
     @observe('ipython_dir')
     def _ipython_dir_changed(self, change):
         ensure_dir_exists(change['new'])
 
     def set_autoindent(self,value=None):
         """Set the autoindent flag.
 
         If called with no arguments, it acts as a toggle."""
         if value is None:
             self.autoindent = not self.autoindent
         else:
             self.autoindent = value
 
     def set_trio_runner(self, tr):
         self.trio_runner = tr
 
     #-------------------------------------------------------------------------
     # init_* methods called by __init__
     #-------------------------------------------------------------------------
 
     def init_ipython_dir(self, ipython_dir):
         if ipython_dir is not None:
             self.ipython_dir = ipython_dir
             return
 
         self.ipython_dir = get_ipython_dir()
 
     def init_profile_dir(self, profile_dir):
         if profile_dir is not None:
             self.profile_dir = profile_dir
             return
         self.profile_dir = ProfileDir.create_profile_dir_by_name(
             self.ipython_dir, "default"
         )
 
     def init_instance_attrs(self):
         self.more = False
 
         # command compiler
         self.compile = self.compiler_class()
 
         # Make an empty namespace, which extension writers can rely on both
         # existing and NEVER being used by ipython itself.  This gives them a
         # convenient location for storing additional information and state
         # their extensions may require, without fear of collisions with other
         # ipython names that may develop later.
         self.meta = Struct()
 
         # Temporary files used for various purposes.  Deleted at exit.
         # The files here are stored with Path from Pathlib
         self.tempfiles = []
         self.tempdirs = []
 
         # keep track of where we started running (mainly for crash post-mortem)
         # This is not being used anywhere currently.
         self.starting_dir = os.getcwd()
 
         # Indentation management
         self.indent_current_nsp = 0
 
         # Dict to track post-execution functions that have been registered
         self._post_execute = {}
 
     def init_environment(self):
         """Any changes we need to make to the user's environment."""
         pass
 
     def init_encoding(self):
         # Get system encoding at startup time.  Certain terminals (like Emacs
         # under Win32 have it set to None, and we need to have a known valid
         # encoding to use in the raw_input() method
         try:
             self.stdin_encoding = sys.stdin.encoding or 'ascii'
         except AttributeError:
             self.stdin_encoding = 'ascii'
 
 
     @observe('colors')
     def init_syntax_highlighting(self, changes=None):
         # Python source parser/formatter for syntax highlighting
         pyformat = PyColorize.Parser(style=self.colors, parent=self).format
         self.pycolorize = lambda src: pyformat(src,'str')
 
     def refresh_style(self):
         # No-op here, used in subclass
         pass
 
     def init_pushd_popd_magic(self):
         # for pushd/popd management
         self.home_dir = get_home_dir()
 
         self.dir_stack = []
 
     def init_logger(self):
         self.logger = Logger(self.home_dir, logfname='ipython_log.py',
                              logmode='rotate')
 
     def init_logstart(self):
         """Initialize logging in case it was requested at the command line.
         """
         if self.logappend:
             self.magic('logstart %s append' % self.logappend)
         elif self.logfile:
             self.magic('logstart %s' % self.logfile)
         elif self.logstart:
             self.magic('logstart')
 
 
     def init_builtins(self):
         # A single, static flag that we set to True.  Its presence indicates
         # that an IPython shell has been created, and we make no attempts at
         # removing on exit or representing the existence of more than one
         # IPython at a time.
         builtin_mod.__dict__['__IPYTHON__'] = True
         builtin_mod.__dict__['display'] = display
 
         self.builtin_trap = BuiltinTrap(shell=self)
 
     @observe('colors')
     def init_inspector(self, changes=None):
         # Object inspector
         self.inspector = self.inspector_class(
             oinspect.InspectColors,
             PyColorize.ANSICodeColors,
             self.colors,
             self.object_info_string_level,
         )
 
     def init_io(self):
         # implemented in subclasses, TerminalInteractiveShell does call
         # colorama.init().
         pass
 
     def init_prompts(self):
         # Set system prompts, so that scripts can decide if they are running
         # interactively.
         sys.ps1 = 'In : '
         sys.ps2 = '...: '
         sys.ps3 = 'Out: '
 
     def init_display_formatter(self):
         self.display_formatter = DisplayFormatter(parent=self)
         self.configurables.append(self.display_formatter)
 
     def init_display_pub(self):
         self.display_pub = self.display_pub_class(parent=self, shell=self)
         self.configurables.append(self.display_pub)
 
     def init_data_pub(self):
         if not self.data_pub_class:
             self.data_pub = None
             return
         self.data_pub = self.data_pub_class(parent=self)
         self.configurables.append(self.data_pub)
 
     def init_displayhook(self):
         # Initialize displayhook, set in/out prompts and printing system
         self.displayhook = self.displayhook_class(
             parent=self,
             shell=self,
             cache_size=self.cache_size,
         )
         self.configurables.append(self.displayhook)
         # This is a context manager that installs/revmoes the displayhook at
         # the appropriate time.
         self.display_trap = DisplayTrap(hook=self.displayhook)
 
     @staticmethod
     def get_path_links(p: Path):
         """Gets path links including all symlinks
 
         Examples
         --------
         In [1]: from IPython.core.interactiveshell import InteractiveShell
 
         In [2]: import sys, pathlib
 
         In [3]: paths = InteractiveShell.get_path_links(pathlib.Path(sys.executable))
 
         In [4]: len(paths) == len(set(paths))
         Out[4]: True
 
         In [5]: bool(paths)
         Out[5]: True
         """
         paths = [p]
         while p.is_symlink():
             new_path = Path(os.readlink(p))
             if not new_path.is_absolute():
                 new_path = p.parent / new_path
             p = new_path
             paths.append(p)
         return paths
 
     def init_virtualenv(self):
         """Add the current virtualenv to sys.path so the user can import modules from it.
         This isn't perfect: it doesn't use the Python interpreter with which the
         virtualenv was built, and it ignores the --no-site-packages option. A
         warning will appear suggesting the user installs IPython in the
         virtualenv, but for many cases, it probably works well enough.
 
         Adapted from code snippets online.
 
         http://blog.ufsoft.org/2009/1/29/ipython-and-virtualenv
         """
         if 'VIRTUAL_ENV' not in os.environ:
             # Not in a virtualenv
             return
         elif os.environ["VIRTUAL_ENV"] == "":
             warn("Virtual env path set to '', please check if this is intended.")
             return
 
         p = Path(sys.executable)
         p_venv = Path(os.environ["VIRTUAL_ENV"])
 
         # fallback venv detection:
         # stdlib venv may symlink sys.executable, so we can't use realpath.
         # but others can symlink *to* the venv Python, so we can't just use sys.executable.
         # So we just check every item in the symlink tree (generally <= 3)
         paths = self.get_path_links(p)
 
         # In Cygwin paths like "c:\..." and '\cygdrive\c\...' are possible
         if len(p_venv.parts) > 2 and p_venv.parts[1] == "cygdrive":
             drive_name = p_venv.parts[2]
             p_venv = (drive_name + ":/") / Path(*p_venv.parts[3:])
 
         if any(p_venv == p.parents[1] for p in paths):
             # Our exe is inside or has access to the virtualenv, don't need to do anything.
             return
 
         if sys.platform == "win32":
             virtual_env = str(Path(os.environ["VIRTUAL_ENV"], "Lib", "site-packages"))
         else:
             virtual_env_path = Path(
                 os.environ["VIRTUAL_ENV"], "lib", "python{}.{}", "site-packages"
             )
             p_ver = sys.version_info[:2]
 
             # Predict version from py[thon]-x.x in the $VIRTUAL_ENV
             re_m = re.search(r"\bpy(?:thon)?([23])\.(\d+)\b", os.environ["VIRTUAL_ENV"])
             if re_m:
                 predicted_path = Path(str(virtual_env_path).format(*re_m.groups()))
                 if predicted_path.exists():
                     p_ver = re_m.groups()
 
             virtual_env = str(virtual_env_path).format(*p_ver)
         if self.warn_venv:
             warn(
                 "Attempting to work in a virtualenv. If you encounter problems, "
                 "please install IPython inside the virtualenv."
             )
         import site
         sys.path.insert(0, virtual_env)
         site.addsitedir(virtual_env)
 
     #-------------------------------------------------------------------------
     # Things related to injections into the sys module
     #-------------------------------------------------------------------------
 
     def save_sys_module_state(self):
         """Save the state of hooks in the sys module.
 
         This has to be called after self.user_module is created.
         """
         self._orig_sys_module_state = {'stdin': sys.stdin,
                                        'stdout': sys.stdout,
                                        'stderr': sys.stderr,
                                        'excepthook': sys.excepthook}
         self._orig_sys_modules_main_name = self.user_module.__name__
         self._orig_sys_modules_main_mod = sys.modules.get(self.user_module.__name__)
 
     def restore_sys_module_state(self):
         """Restore the state of the sys module."""
         try:
             for k, v in self._orig_sys_module_state.items():
                 setattr(sys, k, v)
         except AttributeError:
             pass
         # Reset what what done in self.init_sys_modules
         if self._orig_sys_modules_main_mod is not None:
             sys.modules[self._orig_sys_modules_main_name] = self._orig_sys_modules_main_mod
 
     #-------------------------------------------------------------------------
     # Things related to the banner
     #-------------------------------------------------------------------------
 
     @property
     def banner(self):
         banner = self.banner1
         if self.profile and self.profile != 'default':
             banner += '\nIPython profile: %s\n' % self.profile
         if self.banner2:
             banner += '\n' + self.banner2
         return banner
 
     def show_banner(self, banner=None):
         if banner is None:
             banner = self.banner
         sys.stdout.write(banner)
 
     #-------------------------------------------------------------------------
     # Things related to hooks
     #-------------------------------------------------------------------------
 
     def init_hooks(self):
         # hooks holds pointers used for user-side customizations
         self.hooks = Struct()
 
         self.strdispatchers = {}
 
         # Set all default hooks, defined in the IPython.hooks module.
         hooks = IPython.core.hooks
         for hook_name in hooks.__all__:
             # default hooks have priority 100, i.e. low; user hooks should have
             # 0-100 priority
             self.set_hook(hook_name, getattr(hooks, hook_name), 100)
 
         if self.display_page:
             self.set_hook('show_in_pager', page.as_hook(page.display_page), 90)
 
     def set_hook(self, name, hook, priority=50, str_key=None, re_key=None):
         """set_hook(name,hook) -> sets an internal IPython hook.
 
         IPython exposes some of its internal API as user-modifiable hooks.  By
         adding your function to one of these hooks, you can modify IPython's
         behavior to call at runtime your own routines."""
 
         # At some point in the future, this should validate the hook before it
         # accepts it.  Probably at least check that the hook takes the number
         # of args it's supposed to.
 
         f = types.MethodType(hook,self)
 
         # check if the hook is for strdispatcher first
         if str_key is not None:
             sdp = self.strdispatchers.get(name, StrDispatch())
             sdp.add_s(str_key, f, priority )
             self.strdispatchers[name] = sdp
             return
         if re_key is not None:
             sdp = self.strdispatchers.get(name, StrDispatch())
             sdp.add_re(re.compile(re_key), f, priority )
             self.strdispatchers[name] = sdp
             return
 
         dp = getattr(self.hooks, name, None)
         if name not in IPython.core.hooks.__all__:
             print("Warning! Hook '%s' is not one of %s" % \
                   (name, IPython.core.hooks.__all__ ))
 
         if name in IPython.core.hooks.deprecated:
             alternative = IPython.core.hooks.deprecated[name]
             raise ValueError(
                 "Hook {} has been deprecated since IPython 5.0. Use {} instead.".format(
                     name, alternative
                 )
             )
 
         if not dp:
             dp = IPython.core.hooks.CommandChainDispatcher()
 
         try:
             dp.add(f,priority)
         except AttributeError:
             # it was not commandchain, plain old func - replace
             dp = f
 
         setattr(self.hooks,name, dp)
 
     #-------------------------------------------------------------------------
     # Things related to events
     #-------------------------------------------------------------------------
 
     def init_events(self):
         self.events = EventManager(self, available_events)
 
         self.events.register("pre_execute", self._clear_warning_registry)
 
     def register_post_execute(self, func):
         """DEPRECATED: Use ip.events.register('post_run_cell', func)
 
         Register a function for calling after code execution.
         """
         raise ValueError(
             "ip.register_post_execute is deprecated since IPython 1.0, use "
             "ip.events.register('post_run_cell', func) instead."
         )
 
     def _clear_warning_registry(self):
         # clear the warning registry, so that different code blocks with
         # overlapping line number ranges don't cause spurious suppression of
         # warnings (see gh-6611 for details)
         if "__warningregistry__" in self.user_global_ns:
             del self.user_global_ns["__warningregistry__"]
 
     #-------------------------------------------------------------------------
     # Things related to the "main" module
     #-------------------------------------------------------------------------
 
     def new_main_mod(self, filename, modname):
         """Return a new 'main' module object for user code execution.
 
         ``filename`` should be the path of the script which will be run in the
         module. Requests with the same filename will get the same module, with
         its namespace cleared.
 
         ``modname`` should be the module name - normally either '__main__' or
         the basename of the file without the extension.
 
         When scripts are executed via %run, we must keep a reference to their
         __main__ module around so that Python doesn't
         clear it, rendering references to module globals useless.
 
         This method keeps said reference in a private dict, keyed by the
         absolute path of the script. This way, for multiple executions of the
         same script we only keep one copy of the namespace (the last one),
         thus preventing memory leaks from old references while allowing the
         objects from the last execution to be accessible.
         """
         filename = os.path.abspath(filename)
         try:
             main_mod = self._main_mod_cache[filename]
         except KeyError:
             main_mod = self._main_mod_cache[filename] = types.ModuleType(
                         modname,
                         doc="Module created for script run in IPython")
         else:
             main_mod.__dict__.clear()
             main_mod.__name__ = modname
 
         main_mod.__file__ = filename
         # It seems pydoc (and perhaps others) needs any module instance to
         # implement a __nonzero__ method
         main_mod.__nonzero__ = lambda : True
 
         return main_mod
 
     def clear_main_mod_cache(self):
         """Clear the cache of main modules.
 
         Mainly for use by utilities like %reset.
 
         Examples
         --------
         In [15]: import IPython
 
         In [16]: m = _ip.new_main_mod(IPython.__file__, 'IPython')
 
         In [17]: len(_ip._main_mod_cache) > 0
         Out[17]: True
 
         In [18]: _ip.clear_main_mod_cache()
 
         In [19]: len(_ip._main_mod_cache) == 0
         Out[19]: True
         """
         self._main_mod_cache.clear()
 
     #-------------------------------------------------------------------------
     # Things related to debugging
     #-------------------------------------------------------------------------
 
     def init_pdb(self):
         # Set calling of pdb on exceptions
         # self.call_pdb is a property
         self.call_pdb = self.pdb
 
     def _get_call_pdb(self):
         return self._call_pdb
 
     def _set_call_pdb(self,val):
 
         if val not in (0,1,False,True):
             raise ValueError('new call_pdb value must be boolean')
 
         # store value in instance
         self._call_pdb = val
 
         # notify the actual exception handlers
         self.InteractiveTB.call_pdb = val
 
     call_pdb = property(_get_call_pdb,_set_call_pdb,None,
                         'Control auto-activation of pdb at exceptions')
 
     def debugger(self,force=False):
         """Call the pdb debugger.
 
         Keywords:
 
           - force(False): by default, this routine checks the instance call_pdb
             flag and does not actually invoke the debugger if the flag is false.
             The 'force' option forces the debugger to activate even if the flag
             is false.
         """
 
         if not (force or self.call_pdb):
             return
 
         if not hasattr(sys,'last_traceback'):
             error('No traceback has been produced, nothing to debug.')
             return
 
         self.InteractiveTB.debugger(force=True)
 
     #-------------------------------------------------------------------------
     # Things related to IPython's various namespaces
     #-------------------------------------------------------------------------
     default_user_namespaces = True
 
     def init_create_namespaces(self, user_module=None, user_ns=None):
         # Create the namespace where the user will operate.  user_ns is
         # normally the only one used, and it is passed to the exec calls as
         # the locals argument.  But we do carry a user_global_ns namespace
         # given as the exec 'globals' argument,  This is useful in embedding
         # situations where the ipython shell opens in a context where the
         # distinction between locals and globals is meaningful.  For
         # non-embedded contexts, it is just the same object as the user_ns dict.
 
         # FIXME. For some strange reason, __builtins__ is showing up at user
         # level as a dict instead of a module. This is a manual fix, but I
         # should really track down where the problem is coming from. Alex
         # Schmolck reported this problem first.
 
         # A useful post by Alex Martelli on this topic:
         # Re: inconsistent value from __builtins__
         # Von: Alex Martelli <[e-mail]>
         # Datum: Freitag 01 Oktober 2004 04:45:34 nachmittags/abends
         # Gruppen: comp.lang.python
 
         # Michael Hohn <[e-mail]> wrote:
         # > >>> print type(builtin_check.get_global_binding('__builtins__'))
         # > <type 'dict'>
         # > >>> print type(__builtins__)
         # > <type 'module'>
         # > Is this difference in return value intentional?
 
         # Well, it's documented that '__builtins__' can be either a dictionary
         # or a module, and it's been that way for a long time. Whether it's
         # intentional (or sensible), I don't know. In any case, the idea is
         # that if you need to access the built-in namespace directly, you
         # should start with "import __builtin__" (note, no 's') which will
         # definitely give you a module. Yeah, it's somewhat confusing:-(.
 
         # These routines return a properly built module and dict as needed by
         # the rest of the code, and can also be used by extension writers to
         # generate properly initialized namespaces.
         if (user_ns is not None) or (user_module is not None):
             self.default_user_namespaces = False
         self.user_module, self.user_ns = self.prepare_user_module(user_module, user_ns)
 
         # A record of hidden variables we have added to the user namespace, so
         # we can list later only variables defined in actual interactive use.
         self.user_ns_hidden = {}
 
         # Now that FakeModule produces a real module, we've run into a nasty
         # problem: after script execution (via %run), the module where the user
         # code ran is deleted.  Now that this object is a true module (needed
         # so doctest and other tools work correctly), the Python module
         # teardown mechanism runs over it, and sets to None every variable
         # present in that module.  Top-level references to objects from the
         # script survive, because the user_ns is updated with them.  However,
         # calling functions defined in the script that use other things from
         # the script will fail, because the function's closure had references
         # to the original objects, which are now all None.  So we must protect
         # these modules from deletion by keeping a cache.
         #
         # To avoid keeping stale modules around (we only need the one from the
         # last run), we use a dict keyed with the full path to the script, so
         # only the last version of the module is held in the cache.  Note,
         # however, that we must cache the module *namespace contents* (their
         # __dict__).  Because if we try to cache the actual modules, old ones
         # (uncached) could be destroyed while still holding references (such as
         # those held by GUI objects that tend to be long-lived)>
         #
         # The %reset command will flush this cache.  See the cache_main_mod()
         # and clear_main_mod_cache() methods for details on use.
 
         # This is the cache used for 'main' namespaces
         self._main_mod_cache = {}
 
         # A table holding all the namespaces IPython deals with, so that
         # introspection facilities can search easily.
         self.ns_table = {'user_global':self.user_module.__dict__,
                          'user_local':self.user_ns,
                          'builtin':builtin_mod.__dict__
                          }
 
     @property
     def user_global_ns(self):
         return self.user_module.__dict__
 
     def prepare_user_module(self, user_module=None, user_ns=None):
         """Prepare the module and namespace in which user code will be run.
 
         When IPython is started normally, both parameters are None: a new module
         is created automatically, and its __dict__ used as the namespace.
 
         If only user_module is provided, its __dict__ is used as the namespace.
         If only user_ns is provided, a dummy module is created, and user_ns
         becomes the global namespace. If both are provided (as they may be
         when embedding), user_ns is the local namespace, and user_module
         provides the global namespace.
 
         Parameters
         ----------
         user_module : module, optional
             The current user module in which IPython is being run. If None,
             a clean module will be created.
         user_ns : dict, optional
             A namespace in which to run interactive commands.
 
         Returns
         -------
         A tuple of user_module and user_ns, each properly initialised.
         """
         if user_module is None and user_ns is not None:
             user_ns.setdefault("__name__", "__main__")
             user_module = DummyMod()
             user_module.__dict__ = user_ns
 
         if user_module is None:
             user_module = types.ModuleType("__main__",
                 doc="Automatically created module for IPython interactive environment")
 
         # We must ensure that __builtin__ (without the final 's') is always
         # available and pointing to the __builtin__ *module*.  For more details:
         # http://mail.python.org/pipermail/python-dev/2001-April/014068.html
         user_module.__dict__.setdefault('__builtin__', builtin_mod)
         user_module.__dict__.setdefault('__builtins__', builtin_mod)
 
         if user_ns is None:
             user_ns = user_module.__dict__
 
         return user_module, user_ns
 
     def init_sys_modules(self):
         # We need to insert into sys.modules something that looks like a
         # module but which accesses the IPython namespace, for shelve and
         # pickle to work interactively. Normally they rely on getting
         # everything out of __main__, but for embedding purposes each IPython
         # instance has its own private namespace, so we can't go shoving
         # everything into __main__.
 
         # note, however, that we should only do this for non-embedded
         # ipythons, which really mimic the __main__.__dict__ with their own
         # namespace.  Embedded instances, on the other hand, should not do
         # this because they need to manage the user local/global namespaces
         # only, but they live within a 'normal' __main__ (meaning, they
         # shouldn't overtake the execution environment of the script they're
         # embedded in).
 
         # This is overridden in the InteractiveShellEmbed subclass to a no-op.
         main_name = self.user_module.__name__
         sys.modules[main_name] = self.user_module
 
     def init_user_ns(self):
         """Initialize all user-visible namespaces to their minimum defaults.
 
         Certain history lists are also initialized here, as they effectively
         act as user namespaces.
 
         Notes
         -----
         All data structures here are only filled in, they are NOT reset by this
         method.  If they were not empty before, data will simply be added to
         them.
         """
         # This function works in two parts: first we put a few things in
         # user_ns, and we sync that contents into user_ns_hidden so that these
         # initial variables aren't shown by %who.  After the sync, we add the
         # rest of what we *do* want the user to see with %who even on a new
         # session (probably nothing, so they really only see their own stuff)
 
         # The user dict must *always* have a __builtin__ reference to the
         # Python standard __builtin__ namespace,  which must be imported.
         # This is so that certain operations in prompt evaluation can be
         # reliably executed with builtins.  Note that we can NOT use
         # __builtins__ (note the 's'),  because that can either be a dict or a
         # module, and can even mutate at runtime, depending on the context
         # (Python makes no guarantees on it).  In contrast, __builtin__ is
         # always a module object, though it must be explicitly imported.
 
         # For more details:
         # http://mail.python.org/pipermail/python-dev/2001-April/014068.html
         ns = {}
 
         # make global variables for user access to the histories
         ns['_ih'] = self.history_manager.input_hist_parsed
         ns['_oh'] = self.history_manager.output_hist
         ns['_dh'] = self.history_manager.dir_hist
 
         # user aliases to input and output histories.  These shouldn't show up
         # in %who, as they can have very large reprs.
         ns['In']  = self.history_manager.input_hist_parsed
         ns['Out'] = self.history_manager.output_hist
 
         # Store myself as the public api!!!
         ns['get_ipython'] = self.get_ipython
 
         ns['exit'] = self.exiter
         ns['quit'] = self.exiter
         ns["open"] = _modified_open
 
         # Sync what we've added so far to user_ns_hidden so these aren't seen
         # by %who
         self.user_ns_hidden.update(ns)
 
         # Anything put into ns now would show up in %who.  Think twice before
         # putting anything here, as we really want %who to show the user their
         # stuff, not our variables.
 
         # Finally, update the real user's namespace
         self.user_ns.update(ns)
 
     @property
     def all_ns_refs(self):
         """Get a list of references to all the namespace dictionaries in which
         IPython might store a user-created object.
 
         Note that this does not include the displayhook, which also caches
         objects from the output."""
         return [self.user_ns, self.user_global_ns, self.user_ns_hidden] + \
                [m.__dict__ for m in self._main_mod_cache.values()]
 
     def reset(self, new_session=True, aggressive=False):
         """Clear all internal namespaces, and attempt to release references to
         user objects.
 
         If new_session is True, a new history session will be opened.
         """
         # Clear histories
         assert self.history_manager is not None
         self.history_manager.reset(new_session)
         # Reset counter used to index all histories
         if new_session:
             self.execution_count = 1
 
         # Reset last execution result
         self.last_execution_succeeded = True
         self.last_execution_result = None
 
         # Flush cached output items
         if self.displayhook.do_full_cache:
             self.displayhook.flush()
 
         # The main execution namespaces must be cleared very carefully,
         # skipping the deletion of the builtin-related keys, because doing so
         # would cause errors in many object's __del__ methods.
         if self.user_ns is not self.user_global_ns:
             self.user_ns.clear()
         ns = self.user_global_ns
         drop_keys = set(ns.keys())
         drop_keys.discard('__builtin__')
         drop_keys.discard('__builtins__')
         drop_keys.discard('__name__')
         for k in drop_keys:
             del ns[k]
 
         self.user_ns_hidden.clear()
 
         # Restore the user namespaces to minimal usability
         self.init_user_ns()
         if aggressive and not hasattr(self, "_sys_modules_keys"):
             print("Cannot restore sys.module, no snapshot")
         elif aggressive:
             print("culling sys module...")
             current_keys = set(sys.modules.keys())
             for k in current_keys - self._sys_modules_keys:
                 if k.startswith("multiprocessing"):
                     continue
                 del sys.modules[k]
 
         # Restore the default and user aliases
         self.alias_manager.clear_aliases()
         self.alias_manager.init_aliases()
 
         # Now define aliases that only make sense on the terminal, because they
         # need direct access to the console in a way that we can't emulate in
         # GUI or web frontend
         if os.name == 'posix':
             for cmd in ('clear', 'more', 'less', 'man'):
                 if cmd not in self.magics_manager.magics['line']:
                     self.alias_manager.soft_define_alias(cmd, cmd)
 
         # Flush the private list of module references kept for script
         # execution protection
         self.clear_main_mod_cache()
 
     def del_var(self, varname, by_name=False):
         """Delete a variable from the various namespaces, so that, as
         far as possible, we're not keeping any hidden references to it.
 
         Parameters
         ----------
         varname : str
             The name of the variable to delete.
         by_name : bool
             If True, delete variables with the given name in each
             namespace. If False (default), find the variable in the user
             namespace, and delete references to it.
         """
         if varname in ('__builtin__', '__builtins__'):
             raise ValueError("Refusing to delete %s" % varname)
 
         ns_refs = self.all_ns_refs
 
         if by_name:                    # Delete by name
             for ns in ns_refs:
                 try:
                     del ns[varname]
                 except KeyError:
                     pass
         else:                         # Delete by object
             try:
                 obj = self.user_ns[varname]
             except KeyError as e:
                 raise NameError("name '%s' is not defined" % varname) from e
             # Also check in output history
             assert self.history_manager is not None
             ns_refs.append(self.history_manager.output_hist)
             for ns in ns_refs:
                 to_delete = [n for n, o in ns.items() if o is obj]
                 for name in to_delete:
                     del ns[name]
 
             # Ensure it is removed from the last execution result
             if self.last_execution_result.result is obj:
                 self.last_execution_result = None
 
             # displayhook keeps extra references, but not in a dictionary
             for name in ('_', '__', '___'):
                 if getattr(self.displayhook, name) is obj:
                     setattr(self.displayhook, name, None)
 
     def reset_selective(self, regex=None):
         """Clear selective variables from internal namespaces based on a
         specified regular expression.
 
         Parameters
         ----------
         regex : string or compiled pattern, optional
             A regular expression pattern that will be used in searching
             variable names in the users namespaces.
         """
         if regex is not None:
             try:
                 m = re.compile(regex)
             except TypeError as e:
                 raise TypeError('regex must be a string or compiled pattern') from e
             # Search for keys in each namespace that match the given regex
             # If a match is found, delete the key/value pair.
             for ns in self.all_ns_refs:
                 for var in ns:
                     if m.search(var):
                         del ns[var]
 
     def push(self, variables, interactive=True):
         """Inject a group of variables into the IPython user namespace.
 
         Parameters
         ----------
         variables : dict, str or list/tuple of str
             The variables to inject into the user's namespace.  If a dict, a
             simple update is done.  If a str, the string is assumed to have
             variable names separated by spaces.  A list/tuple of str can also
             be used to give the variable names.  If just the variable names are
             give (list/tuple/str) then the variable values looked up in the
             callers frame.
         interactive : bool
             If True (default), the variables will be listed with the ``who``
             magic.
         """
         vdict = None
 
         # We need a dict of name/value pairs to do namespace updates.
         if isinstance(variables, dict):
             vdict = variables
         elif isinstance(variables, (str, list, tuple)):
             if isinstance(variables, str):
                 vlist = variables.split()
             else:
                 vlist = variables
             vdict = {}
             cf = sys._getframe(1)
             for name in vlist:
                 try:
                     vdict[name] = eval(name, cf.f_globals, cf.f_locals)
-                except Exception:
+                except:
                     print('Could not get variable %s from %s' %
                            (name,cf.f_code.co_name))
         else:
             raise ValueError('variables must be a dict/str/list/tuple')
 
         # Propagate variables to user namespace
         self.user_ns.update(vdict)
 
         # And configure interactive visibility
         user_ns_hidden = self.user_ns_hidden
         if interactive:
             for name in vdict:
                 user_ns_hidden.pop(name, None)
         else:
             user_ns_hidden.update(vdict)
 
     def drop_by_id(self, variables):
         """Remove a dict of variables from the user namespace, if they are the
         same as the values in the dictionary.
 
         This is intended for use by extensions: variables that they've added can
         be taken back out if they are unloaded, without removing any that the
         user has overwritten.
 
         Parameters
         ----------
         variables : dict
             A dictionary mapping object names (as strings) to the objects.
         """
         for name, obj in variables.items():
             if name in self.user_ns and self.user_ns[name] is obj:
                 del self.user_ns[name]
                 self.user_ns_hidden.pop(name, None)
 
     #-------------------------------------------------------------------------
     # Things related to object introspection
     #-------------------------------------------------------------------------
     @staticmethod
     def _find_parts(oname: str) -> Tuple[bool, ListType[str]]:
         """
         Given an object name, return a list of parts of this object name.
 
         Basically split on docs when using attribute access,
         and extract the value when using square bracket.
 
 
         For example foo.bar[3].baz[x] -> foo, bar, 3, baz, x
 
 
         Returns
         -------
         parts_ok: bool
             whether we were properly able to parse parts.
         parts: list of str
             extracted parts
 
 
 
         """
         raw_parts = oname.split(".")
         parts = []
         parts_ok = True
         for p in raw_parts:
             if p.endswith("]"):
                 var, *indices = p.split("[")
                 if not var.isidentifier():
                     parts_ok = False
                     break
                 parts.append(var)
                 for ind in indices:
                     if ind[-1] != "]" and not is_integer_string(ind[:-1]):
                         parts_ok = False
                         break
                     parts.append(ind[:-1])
                 continue
 
             if not p.isidentifier():
                 parts_ok = False
             parts.append(p)
 
         return parts_ok, parts
 
     def _ofind(
         self, oname: str, namespaces: Optional[Sequence[Tuple[str, AnyType]]] = None
     ) -> OInfo:
         """Find an object in the available namespaces.
 
 
         Returns
         -------
         OInfo with fields:
           - ismagic
           - isalias
           - found
           - obj
           - namespac
           - parent
 
         Has special code to detect magic functions.
         """
         oname = oname.strip()
         parts_ok, parts = self._find_parts(oname)
 
         if (
             not oname.startswith(ESC_MAGIC)
             and not oname.startswith(ESC_MAGIC2)
             and not parts_ok
         ):
             return OInfo(
                 ismagic=False,
                 isalias=False,
                 found=False,
                 obj=None,
                 namespace=None,
                 parent=None,
             )
 
         if namespaces is None:
             # Namespaces to search in:
             # Put them in a list. The order is important so that we
             # find things in the same order that Python finds them.
             namespaces = [ ('Interactive', self.user_ns),
                            ('Interactive (global)', self.user_global_ns),
                            ('Python builtin', builtin_mod.__dict__),
                            ]
 
         ismagic = False
         isalias = False
         found = False
         ospace = None
         parent = None
         obj = None
 
 
         # Look for the given name by splitting it in parts.  If the head is
         # found, then we look for all the remaining parts as members, and only
         # declare success if we can find them all.
         oname_parts = parts
         oname_head, oname_rest = oname_parts[0],oname_parts[1:]
         for nsname,ns in namespaces:
             try:
                 obj = ns[oname_head]
             except KeyError:
                 continue
             else:
                 for idx, part in enumerate(oname_rest):
                     try:
                         parent = obj
                         # The last part is looked up in a special way to avoid
                         # descriptor invocation as it may raise or have side
                         # effects.
                         if idx == len(oname_rest) - 1:
                             obj = self._getattr_property(obj, part)
                         else:
                             if is_integer_string(part):
                                 obj = obj[int(part)]
                             else:
                                 obj = getattr(obj, part)
-                    except Exception:
+                    except:
                         # Blanket except b/c some badly implemented objects
                         # allow __getattr__ to raise exceptions other than
                         # AttributeError, which then crashes IPython.
                         break
                 else:
                     # If we finish the for loop (no break), we got all members
                     found = True
                     ospace = nsname
                     break  # namespace loop
 
         # Try to see if it's magic
         if not found:
             obj = None
             if oname.startswith(ESC_MAGIC2):
                 oname = oname.lstrip(ESC_MAGIC2)
                 obj = self.find_cell_magic(oname)
             elif oname.startswith(ESC_MAGIC):
                 oname = oname.lstrip(ESC_MAGIC)
                 obj = self.find_line_magic(oname)
             else:
                 # search without prefix, so run? will find %run?
                 obj = self.find_line_magic(oname)
                 if obj is None:
                     obj = self.find_cell_magic(oname)
             if obj is not None:
                 found = True
                 ospace = 'IPython internal'
                 ismagic = True
                 isalias = isinstance(obj, Alias)
 
         # Last try: special-case some literals like '', [], {}, etc:
         if not found and oname_head in ["''",'""','[]','{}','()']:
             obj = eval(oname_head)
             found = True
             ospace = 'Interactive'
 
         return OInfo(
             obj=obj,
             found=found,
             parent=parent,
             ismagic=ismagic,
             isalias=isalias,
             namespace=ospace,
         )
 
     @staticmethod
     def _getattr_property(obj, attrname):
         """Property-aware getattr to use in object finding.
 
         If attrname represents a property, return it unevaluated (in case it has
         side effects or raises an error.
 
         """
         if not isinstance(obj, type):
             try:
                 # `getattr(type(obj), attrname)` is not guaranteed to return
                 # `obj`, but does so for property:
                 #
                 # property.__get__(self, None, cls) -> self
                 #
                 # The universal alternative is to traverse the mro manually
                 # searching for attrname in class dicts.
                 if is_integer_string(attrname):
                     return obj[int(attrname)]
                 else:
                     attr = getattr(type(obj), attrname)
             except AttributeError:
                 pass
             else:
                 # This relies on the fact that data descriptors (with both
                 # __get__ & __set__ magic methods) take precedence over
                 # instance-level attributes:
                 #
                 #    class A(object):
                 #        @property
                 #        def foobar(self): return 123
                 #    a = A()
                 #    a.__dict__['foobar'] = 345
                 #    a.foobar  # == 123
                 #
                 # So, a property may be returned right away.
                 if isinstance(attr, property):
                     return attr
 
         # Nothing helped, fall back.
         return getattr(obj, attrname)
 
     def _object_find(self, oname, namespaces=None) -> OInfo:
         """Find an object and return a struct with info about it."""
         return self._ofind(oname, namespaces)
 
     def _inspect(self, meth, oname: str, namespaces=None, **kw):
         """Generic interface to the inspector system.
 
         This function is meant to be called by pdef, pdoc & friends.
         """
         info: OInfo = self._object_find(oname, namespaces)
         if self.sphinxify_docstring:
             if sphinxify is None:
                 raise ImportError("Module ``docrepr`` required but missing")
             docformat = sphinxify(self.object_inspect(oname))
         else:
             docformat = None
         if info.found or hasattr(info.parent, oinspect.HOOK_NAME):
             pmethod = getattr(self.inspector, meth)
             # TODO: only apply format_screen to the plain/text repr of the mime
             # bundle.
             formatter = format_screen if info.ismagic else docformat
             if meth == 'pdoc':
                 pmethod(info.obj, oname, formatter)
             elif meth == 'pinfo':
                 pmethod(
                     info.obj,
                     oname,
                     formatter,
                     info,
                     enable_html_pager=self.enable_html_pager,
                     **kw,
                 )
             else:
                 pmethod(info.obj, oname)
         else:
             print('Object `%s` not found.' % oname)
             return 'not found'  # so callers can take other action
 
     def object_inspect(self, oname, detail_level=0):
         """Get object info about oname"""
         with self.builtin_trap:
             info = self._object_find(oname)
             if info.found:
                 return self.inspector.info(info.obj, oname, info=info,
                             detail_level=detail_level
                 )
             else:
                 return oinspect.object_info(name=oname, found=False)
 
     def object_inspect_text(self, oname, detail_level=0):
         """Get object info as formatted text"""
         return self.object_inspect_mime(oname, detail_level)['text/plain']
 
     def object_inspect_mime(self, oname, detail_level=0, omit_sections=()):
         """Get object info as a mimebundle of formatted representations.
 
         A mimebundle is a dictionary, keyed by mime-type.
         It must always have the key `'text/plain'`.
         """
         with self.builtin_trap:
             info = self._object_find(oname)
             if info.found:
                 docformat = (
                     sphinxify(self.object_inspect(oname))
                     if self.sphinxify_docstring
                     else None
                 )
                 return self.inspector._get_info(
                     info.obj,
                     oname,
                     info=info,
                     detail_level=detail_level,
                     formatter=docformat,
                     omit_sections=omit_sections,
                 )
             else:
                 raise KeyError(oname)
 
     #-------------------------------------------------------------------------
     # Things related to history management
     #-------------------------------------------------------------------------
 
     def init_history(self):
         """Sets up the command history, and starts regular autosaves."""
         self.history_manager = HistoryManager(shell=self, parent=self)
         self.configurables.append(self.history_manager)
 
     #-------------------------------------------------------------------------
     # Things related to exception handling and tracebacks (not debugging)
     #-------------------------------------------------------------------------
 
     debugger_cls = InterruptiblePdb
 
     def init_traceback_handlers(self, custom_exceptions):
         # Syntax error handler.
         self.SyntaxTB = ultratb.SyntaxTB(color_scheme='NoColor', parent=self)
 
         # The interactive one is initialized with an offset, meaning we always
         # want to remove the topmost item in the traceback, which is our own
         # internal code. Valid modes: ['Plain','Context','Verbose','Minimal']
         self.InteractiveTB = ultratb.AutoFormattedTB(mode = 'Plain',
                                                      color_scheme='NoColor',
                                                      tb_offset = 1,
                                    debugger_cls=self.debugger_cls, parent=self)
 
         # The instance will store a pointer to the system-wide exception hook,
         # so that runtime code (such as magics) can access it.  This is because
         # during the read-eval loop, it may get temporarily overwritten.
         self.sys_excepthook = sys.excepthook
 
         # and add any custom exception handlers the user may have specified
         self.set_custom_exc(*custom_exceptions)
 
         # Set the exception mode
         self.InteractiveTB.set_mode(mode=self.xmode)
 
     def set_custom_exc(self, exc_tuple, handler):
         """set_custom_exc(exc_tuple, handler)
 
         Set a custom exception handler, which will be called if any of the
         exceptions in exc_tuple occur in the mainloop (specifically, in the
         run_code() method).
 
         Parameters
         ----------
         exc_tuple : tuple of exception classes
             A *tuple* of exception classes, for which to call the defined
             handler.  It is very important that you use a tuple, and NOT A
             LIST here, because of the way Python's except statement works.  If
             you only want to trap a single exception, use a singleton tuple::
 
                 exc_tuple == (MyCustomException,)
 
         handler : callable
             handler must have the following signature::
 
                 def my_handler(self, etype, value, tb, tb_offset=None):
                     ...
                     return structured_traceback
 
             Your handler must return a structured traceback (a list of strings),
             or None.
 
             This will be made into an instance method (via types.MethodType)
             of IPython itself, and it will be called if any of the exceptions
             listed in the exc_tuple are caught. If the handler is None, an
             internal basic one is used, which just prints basic info.
 
             To protect IPython from crashes, if your handler ever raises an
             exception or returns an invalid result, it will be immediately
             disabled.
 
         Notes
         -----
         WARNING: by putting in your own exception handler into IPython's main
         execution loop, you run a very good chance of nasty crashes.  This
         facility should only be used if you really know what you are doing.
         """
 
         if not isinstance(exc_tuple, tuple):
             raise TypeError("The custom exceptions must be given as a tuple.")
 
         def dummy_handler(self, etype, value, tb, tb_offset=None):
             print('*** Simple custom exception handler ***')
             print('Exception type :', etype)
             print('Exception value:', value)
             print('Traceback      :', tb)
 
         def validate_stb(stb):
             """validate structured traceback return type
 
             return type of CustomTB *should* be a list of strings, but allow
             single strings or None, which are harmless.
 
             This function will *always* return a list of strings,
             and will raise a TypeError if stb is inappropriate.
             """
             msg = "CustomTB must return list of strings, not %r" % stb
             if stb is None:
                 return []
             elif isinstance(stb, str):
                 return [stb]
             elif not isinstance(stb, list):
                 raise TypeError(msg)
             # it's a list
             for line in stb:
                 # check every element
                 if not isinstance(line, str):
                     raise TypeError(msg)
             return stb
 
         if handler is None:
             wrapped = dummy_handler
         else:
             def wrapped(self,etype,value,tb,tb_offset=None):
                 """wrap CustomTB handler, to protect IPython from user code
 
                 This makes it harder (but not impossible) for custom exception
                 handlers to crash IPython.
                 """
                 try:
                     stb = handler(self,etype,value,tb,tb_offset=tb_offset)
                     return validate_stb(stb)
-                except Exception:
+                except:
                     # clear custom handler immediately
                     self.set_custom_exc((), None)
                     print("Custom TB Handler failed, unregistering", file=sys.stderr)
                     # show the exception in handler first
                     stb = self.InteractiveTB.structured_traceback(*sys.exc_info())
                     print(self.InteractiveTB.stb2text(stb))
                     print("The original exception:")
                     stb = self.InteractiveTB.structured_traceback(
                         etype, value, tb, tb_offset=tb_offset
                     )
                 return stb
 
         self.CustomTB = types.MethodType(wrapped,self)
         self.custom_exceptions = exc_tuple
 
     def excepthook(self, etype, value, tb):
         """One more defense for GUI apps that call sys.excepthook.
 
         GUI frameworks like wxPython trap exceptions and call
         sys.excepthook themselves.  I guess this is a feature that
         enables them to keep running after exceptions that would
         otherwise kill their mainloop. This is a bother for IPython
         which expects to catch all of the program exceptions with a try:
         except: statement.
 
         Normally, IPython sets sys.excepthook to a CrashHandler instance, so if
         any app directly invokes sys.excepthook, it will look to the user like
         IPython crashed.  In order to work around this, we can disable the
         CrashHandler and replace it with this excepthook instead, which prints a
         regular traceback using our InteractiveTB.  In this fashion, apps which
         call sys.excepthook will generate a regular-looking exception from
         IPython, and the CrashHandler will only be triggered by real IPython
         crashes.
 
         This hook should be used sparingly, only in places which are not likely
         to be true IPython errors.
         """
         self.showtraceback((etype, value, tb), tb_offset=0)
 
     def _get_exc_info(self, exc_tuple=None):
         """get exc_info from a given tuple, sys.exc_info() or sys.last_type etc.
 
         Ensures sys.last_type,value,traceback hold the exc_info we found,
         from whichever source.
 
         raises ValueError if none of these contain any information
         """
         if exc_tuple is None:
             etype, value, tb = sys.exc_info()
         else:
             etype, value, tb = exc_tuple
 
         if etype is None:
             if hasattr(sys, 'last_type'):
                 etype, value, tb = sys.last_type, sys.last_value, \
                                    sys.last_traceback
 
         if etype is None:
             raise ValueError("No exception to find")
 
         # Now store the exception info in sys.last_type etc.
         # WARNING: these variables are somewhat deprecated and not
         # necessarily safe to use in a threaded environment, but tools
         # like pdb depend on their existence, so let's set them.  If we
         # find problems in the field, we'll need to revisit their use.
         sys.last_type = etype
         sys.last_value = value
         sys.last_traceback = tb
 
         return etype, value, tb
 
     def show_usage_error(self, exc):
         """Show a short message for UsageErrors
 
         These are special exceptions that shouldn't show a traceback.
         """
         print("UsageError: %s" % exc, file=sys.stderr)
 
     def get_exception_only(self, exc_tuple=None):
         """
         Return as a string (ending with a newline) the exception that
         just occurred, without any traceback.
         """
         etype, value, tb = self._get_exc_info(exc_tuple)
         msg = traceback.format_exception_only(etype, value)
         return ''.join(msg)
 
     def showtraceback(self, exc_tuple=None, filename=None, tb_offset=None,
                       exception_only=False, running_compiled_code=False):
         """Display the exception that just occurred.
 
         If nothing is known about the exception, this is the method which
         should be used throughout the code for presenting user tracebacks,
         rather than directly invoking the InteractiveTB object.
 
         A specific showsyntaxerror() also exists, but this method can take
         care of calling it if needed, so unless you are explicitly catching a
         SyntaxError exception, don't try to analyze the stack manually and
         simply call this method."""
 
         try:
             try:
                 etype, value, tb = self._get_exc_info(exc_tuple)
             except ValueError:
                 print('No traceback available to show.', file=sys.stderr)
                 return
 
             if issubclass(etype, SyntaxError):
                 # Though this won't be called by syntax errors in the input
                 # line, there may be SyntaxError cases with imported code.
                 self.showsyntaxerror(filename, running_compiled_code)
             elif etype is UsageError:
                 self.show_usage_error(value)
             else:
                 if exception_only:
                     stb = ['An exception has occurred, use %tb to see '
                            'the full traceback.\n']
                     stb.extend(self.InteractiveTB.get_exception_only(etype,
                                                                      value))
                 else:
 
                     def contains_exceptiongroup(val):
                         if val is None:
                             return False
                         return isinstance(
                             val, BaseExceptionGroup
                         ) or contains_exceptiongroup(val.__context__)
 
                     if contains_exceptiongroup(value):
                         # fall back to native exception formatting until ultratb
                         # supports exception groups
                         traceback.print_exc()
                     else:
                         try:
                             # Exception classes can customise their traceback - we
                             # use this in IPython.parallel for exceptions occurring
                             # in the engines. This should return a list of strings.
                             if hasattr(value, "_render_traceback_"):
                                 stb = value._render_traceback_()
                             else:
                                 stb = self.InteractiveTB.structured_traceback(
                                     etype, value, tb, tb_offset=tb_offset
                                 )
 
                         except Exception:
                             print(
                                 "Unexpected exception formatting exception. Falling back to standard exception"
                             )
                             traceback.print_exc()
                             return None
 
                         self._showtraceback(etype, value, stb)
                     if self.call_pdb:
                         # drop into debugger
                         self.debugger(force=True)
                     return
 
                 # Actually show the traceback
                 self._showtraceback(etype, value, stb)
 
         except KeyboardInterrupt:
             print('\n' + self.get_exception_only(), file=sys.stderr)
 
     def _showtraceback(self, etype, evalue, stb: str):
         """Actually show a traceback.
 
         Subclasses may override this method to put the traceback on a different
         place, like a side channel.
         """
         val = self.InteractiveTB.stb2text(stb)
         try:
             print(val)
         except UnicodeEncodeError:
             print(val.encode("utf-8", "backslashreplace").decode())
 
     def showsyntaxerror(self, filename=None, running_compiled_code=False):
         """Display the syntax error that just occurred.
 
         This doesn't display a stack trace because there isn't one.
 
         If a filename is given, it is stuffed in the exception instead
         of what was there before (because Python's parser always uses
         "<string>" when reading from a string).
 
         If the syntax error occurred when running a compiled code (i.e. running_compile_code=True),
         longer stack trace will be displayed.
         """
         etype, value, last_traceback = self._get_exc_info()
 
         if filename and issubclass(etype, SyntaxError):
             try:
                 value.filename = filename
-            except Exception:
+            except:
                 # Not the format we expect; leave it alone
                 pass
 
         # If the error occurred when executing compiled code, we should provide full stacktrace.
         elist = traceback.extract_tb(last_traceback) if running_compiled_code else []
         stb = self.SyntaxTB.structured_traceback(etype, value, elist)
         self._showtraceback(etype, value, stb)
 
     # This is overridden in TerminalInteractiveShell to show a message about
     # the %paste magic.
     def showindentationerror(self):
         """Called by _run_cell when there's an IndentationError in code entered
         at the prompt.
 
         This is overridden in TerminalInteractiveShell to show a message about
         the %paste magic."""
         self.showsyntaxerror()
 
     @skip_doctest
     def set_next_input(self, s, replace=False):
         """ Sets the 'default' input string for the next command line.
 
         Example::
 
             In [1]: _ip.set_next_input("Hello Word")
             In [2]: Hello Word_  # cursor is here
         """
         self.rl_next_input = s
 
     def _indent_current_str(self):
         """return the current level of indentation as a string"""
         return self.input_splitter.get_indent_spaces() * ' '
 
     #-------------------------------------------------------------------------
     # Things related to text completion
     #-------------------------------------------------------------------------
 
     def init_completer(self):
         """Initialize the completion machinery.
 
         This creates completion machinery that can be used by client code,
         either interactively in-process (typically triggered by the readline
         library), programmatically (such as in test suites) or out-of-process
         (typically over the network by remote frontends).
         """
         from IPython.core.completer import IPCompleter
         from IPython.core.completerlib import (
             cd_completer,
             magic_run_completer,
             module_completer,
             reset_completer,
         )
 
         self.Completer = IPCompleter(shell=self,
                                      namespace=self.user_ns,
                                      global_namespace=self.user_global_ns,
                                      parent=self,
                                      )
         self.configurables.append(self.Completer)
 
         # Add custom completers to the basic ones built into IPCompleter
         sdisp = self.strdispatchers.get('complete_command', StrDispatch())
         self.strdispatchers['complete_command'] = sdisp
         self.Completer.custom_completers = sdisp
 
         self.set_hook('complete_command', module_completer, str_key = 'import')
         self.set_hook('complete_command', module_completer, str_key = 'from')
         self.set_hook('complete_command', module_completer, str_key = '%aimport')
         self.set_hook('complete_command', magic_run_completer, str_key = '%run')
         self.set_hook('complete_command', cd_completer, str_key = '%cd')
         self.set_hook('complete_command', reset_completer, str_key = '%reset')
 
     @skip_doctest
     def complete(self, text, line=None, cursor_pos=None):
         """Return the completed text and a list of completions.
 
         Parameters
         ----------
         text : string
             A string of text to be completed on.  It can be given as empty and
             instead a line/position pair are given.  In this case, the
             completer itself will split the line like readline does.
         line : string, optional
             The complete line that text is part of.
         cursor_pos : int, optional
             The position of the cursor on the input line.
 
         Returns
         -------
         text : string
             The actual text that was completed.
         matches : list
             A sorted list with all possible completions.
 
         Notes
         -----
         The optional arguments allow the completion to take more context into
         account, and are part of the low-level completion API.
 
         This is a wrapper around the completion mechanism, similar to what
         readline does at the command line when the TAB key is hit.  By
         exposing it as a method, it can be used by other non-readline
         environments (such as GUIs) for text completion.
 
         Examples
         --------
         In [1]: x = 'hello'
 
         In [2]: _ip.complete('x.l')
         Out[2]: ('x.l', ['x.ljust', 'x.lower', 'x.lstrip'])
         """
 
         # Inject names into __builtin__ so we can complete on the added names.
         with self.builtin_trap:
             return self.Completer.complete(text, line, cursor_pos)
 
     def set_custom_completer(self, completer, pos=0) -> None:
         """Adds a new custom completer function.
 
         The position argument (defaults to 0) is the index in the completers
         list where you want the completer to be inserted.
 
         `completer` should have the following signature::
 
             def completion(self: Completer, text: string) -> List[str]:
                 raise NotImplementedError
 
         It will be bound to the current Completer instance and pass some text
         and return a list with current completions to suggest to the user.
         """
 
         newcomp = types.MethodType(completer, self.Completer)
         self.Completer.custom_matchers.insert(pos,newcomp)
 
     def set_completer_frame(self, frame=None):
         """Set the frame of the completer."""
         if frame:
             self.Completer.namespace = frame.f_locals
             self.Completer.global_namespace = frame.f_globals
         else:
             self.Completer.namespace = self.user_ns
             self.Completer.global_namespace = self.user_global_ns
 
     #-------------------------------------------------------------------------
     # Things related to magics
     #-------------------------------------------------------------------------
 
     def init_magics(self):
         from IPython.core import magics as m
         self.magics_manager = magic.MagicsManager(shell=self,
                                    parent=self,
                                    user_magics=m.UserMagics(self))
         self.configurables.append(self.magics_manager)
 
         # Expose as public API from the magics manager
         self.register_magics = self.magics_manager.register
 
         self.register_magics(m.AutoMagics, m.BasicMagics, m.CodeMagics,
             m.ConfigMagics, m.DisplayMagics, m.ExecutionMagics,
             m.ExtensionMagics, m.HistoryMagics, m.LoggingMagics,
             m.NamespaceMagics, m.OSMagics, m.PackagingMagics,
             m.PylabMagics, m.ScriptMagics,
         )
         self.register_magics(m.AsyncMagics)
 
         # Register Magic Aliases
         mman = self.magics_manager
         # FIXME: magic aliases should be defined by the Magics classes
         # or in MagicsManager, not here
         mman.register_alias('ed', 'edit')
         mman.register_alias('hist', 'history')
         mman.register_alias('rep', 'recall')
         mman.register_alias('SVG', 'svg', 'cell')
         mman.register_alias('HTML', 'html', 'cell')
         mman.register_alias('file', 'writefile', 'cell')
 
         # FIXME: Move the color initialization to the DisplayHook, which
         # should be split into a prompt manager and displayhook. We probably
         # even need a centralize colors management object.
         self.run_line_magic('colors', self.colors)
 
     # Defined here so that it's included in the documentation
     @functools.wraps(magic.MagicsManager.register_function)
     def register_magic_function(self, func, magic_kind='line', magic_name=None):
         self.magics_manager.register_function(
             func, magic_kind=magic_kind, magic_name=magic_name
         )
 
     def _find_with_lazy_load(self, /, type_, magic_name: str):
         """
         Try to find a magic potentially lazy-loading it.
 
         Parameters
         ----------
 
         type_: "line"|"cell"
             the type of magics we are trying to find/lazy load.
         magic_name: str
             The name of the magic we are trying to find/lazy load
 
 
         Note that this may have any side effects
         """
         finder = {"line": self.find_line_magic, "cell": self.find_cell_magic}[type_]
         fn = finder(magic_name)
         if fn is not None:
             return fn
         lazy = self.magics_manager.lazy_magics.get(magic_name)
         if lazy is None:
             return None
 
         self.run_line_magic("load_ext", lazy)
         res = finder(magic_name)
         return res
 
     def run_line_magic(self, magic_name: str, line: str, _stack_depth=1):
         """Execute the given line magic.
 
         Parameters
         ----------
         magic_name : str
             Name of the desired magic function, without '%' prefix.
         line : str
             The rest of the input line as a single string.
         _stack_depth : int
             If run_line_magic() is called from magic() then _stack_depth=2.
             This is added to ensure backward compatibility for use of 'get_ipython().magic()'
         """
         fn = self._find_with_lazy_load("line", magic_name)
         if fn is None:
             lazy = self.magics_manager.lazy_magics.get(magic_name)
             if lazy:
                 self.run_line_magic("load_ext", lazy)
                 fn = self.find_line_magic(magic_name)
         if fn is None:
             cm = self.find_cell_magic(magic_name)
             etpl = "Line magic function `%%%s` not found%s."
             extra = '' if cm is None else (' (But cell magic `%%%%%s` exists, '
                                     'did you mean that instead?)' % magic_name )
             raise UsageError(etpl % (magic_name, extra))
         else:
             # Note: this is the distance in the stack to the user's frame.
             # This will need to be updated if the internal calling logic gets
             # refactored, or else we'll be expanding the wrong variables.
 
             # Determine stack_depth depending on where run_line_magic() has been called
             stack_depth = _stack_depth
             if getattr(fn, magic.MAGIC_NO_VAR_EXPAND_ATTR, False):
                 # magic has opted out of var_expand
                 magic_arg_s = line
             else:
                 magic_arg_s = self.var_expand(line, stack_depth)
             # Put magic args in a list so we can call with f(*a) syntax
             args = [magic_arg_s]
             kwargs = {}
             # Grab local namespace if we need it:
             if getattr(fn, "needs_local_scope", False):
                 kwargs['local_ns'] = self.get_local_scope(stack_depth)
             with self.builtin_trap:
                 result = fn(*args, **kwargs)
 
             # The code below prevents the output from being displayed
             # when using magics with decorator @output_can_be_silenced
             # when the last Python token in the expression is a ';'.
             if getattr(fn, magic.MAGIC_OUTPUT_CAN_BE_SILENCED, False):
                 if DisplayHook.semicolon_at_end_of_expression(magic_arg_s):
                     return None
 
             return result
 
     def get_local_scope(self, stack_depth):
         """Get local scope at given stack depth.
 
         Parameters
         ----------
         stack_depth : int
             Depth relative to calling frame
         """
         return sys._getframe(stack_depth + 1).f_locals
 
     def run_cell_magic(self, magic_name, line, cell):
         """Execute the given cell magic.
 
         Parameters
         ----------
         magic_name : str
             Name of the desired magic function, without '%' prefix.
         line : str
             The rest of the first input line as a single string.
         cell : str
             The body of the cell as a (possibly multiline) string.
         """
         fn = self._find_with_lazy_load("cell", magic_name)
         if fn is None:
             lm = self.find_line_magic(magic_name)
             etpl = "Cell magic `%%{0}` not found{1}."
             extra = '' if lm is None else (' (But line magic `%{0}` exists, '
                             'did you mean that instead?)'.format(magic_name))
             raise UsageError(etpl.format(magic_name, extra))
         elif cell == '':
             message = '%%{0} is a cell magic, but the cell body is empty.'.format(magic_name)
             if self.find_line_magic(magic_name) is not None:
                 message += ' Did you mean the line magic %{0} (single %)?'.format(magic_name)
             raise UsageError(message)
         else:
             # Note: this is the distance in the stack to the user's frame.
             # This will need to be updated if the internal calling logic gets
             # refactored, or else we'll be expanding the wrong variables.
             stack_depth = 2
             if getattr(fn, magic.MAGIC_NO_VAR_EXPAND_ATTR, False):
                 # magic has opted out of var_expand
                 magic_arg_s = line
             else:
                 magic_arg_s = self.var_expand(line, stack_depth)
             kwargs = {}
             if getattr(fn, "needs_local_scope", False):
                 kwargs['local_ns'] = self.user_ns
 
             with self.builtin_trap:
                 args = (magic_arg_s, cell)
                 result = fn(*args, **kwargs)
 
             # The code below prevents the output from being displayed
             # when using magics with decorator @output_can_be_silenced
             # when the last Python token in the expression is a ';'.
             if getattr(fn, magic.MAGIC_OUTPUT_CAN_BE_SILENCED, False):
                 if DisplayHook.semicolon_at_end_of_expression(cell):
                     return None
 
             return result
 
     def find_line_magic(self, magic_name):
         """Find and return a line magic by name.
 
         Returns None if the magic isn't found."""
         return self.magics_manager.magics['line'].get(magic_name)
 
     def find_cell_magic(self, magic_name):
         """Find and return a cell magic by name.
 
         Returns None if the magic isn't found."""
         return self.magics_manager.magics['cell'].get(magic_name)
 
     def find_magic(self, magic_name, magic_kind='line'):
         """Find and return a magic of the given type by name.
 
         Returns None if the magic isn't found."""
         return self.magics_manager.magics[magic_kind].get(magic_name)
 
     def magic(self, arg_s):
         """
         DEPRECATED
 
         Deprecated since IPython 0.13 (warning added in
         8.1), use run_line_magic(magic_name, parameter_s).
 
         Call a magic function by name.
 
         Input: a string containing the name of the magic function to call and
         any additional arguments to be passed to the magic.
 
         magic('name -opt foo bar') is equivalent to typing at the ipython
         prompt:
 
         In[1]: %name -opt foo bar
 
         To call a magic without arguments, simply use magic('name').
 
         This provides a proper Python function to call IPython's magics in any
         valid Python code you can type at the interpreter, including loops and
         compound statements.
         """
         warnings.warn(
             "`magic(...)` is deprecated since IPython 0.13 (warning added in "
             "8.1), use run_line_magic(magic_name, parameter_s).",
             DeprecationWarning,
             stacklevel=2,
         )
         # TODO: should we issue a loud deprecation warning here?
         magic_name, _, magic_arg_s = arg_s.partition(' ')
         magic_name = magic_name.lstrip(prefilter.ESC_MAGIC)
         return self.run_line_magic(magic_name, magic_arg_s, _stack_depth=2)
 
     #-------------------------------------------------------------------------
     # Things related to macros
     #-------------------------------------------------------------------------
 
     def define_macro(self, name, themacro):
         """Define a new macro
 
         Parameters
         ----------
         name : str
             The name of the macro.
         themacro : str or Macro
             The action to do upon invoking the macro.  If a string, a new
             Macro object is created by passing the string to it.
         """
 
         from IPython.core import macro
 
         if isinstance(themacro, str):
             themacro = macro.Macro(themacro)
         if not isinstance(themacro, macro.Macro):
             raise ValueError('A macro must be a string or a Macro instance.')
         self.user_ns[name] = themacro
 
     #-------------------------------------------------------------------------
     # Things related to the running of system commands
     #-------------------------------------------------------------------------
 
     def system_piped(self, cmd):
         """Call the given cmd in a subprocess, piping stdout/err
 
         Parameters
         ----------
         cmd : str
             Command to execute (can not end in '&', as background processes are
             not supported.  Should not be a command that expects input
             other than simple text.
         """
         if cmd.rstrip().endswith('&'):
             # this is *far* from a rigorous test
             # We do not support backgrounding processes because we either use
             # pexpect or pipes to read from.  Users can always just call
             # os.system() or use ip.system=ip.system_raw
             # if they really want a background process.
             raise OSError("Background processes not supported.")
 
         # we explicitly do NOT return the subprocess status code, because
         # a non-None value would trigger :func:`sys.displayhook` calls.
         # Instead, we store the exit_code in user_ns.
         self.user_ns['_exit_code'] = system(self.var_expand(cmd, depth=1))
 
     def system_raw(self, cmd):
         """Call the given cmd in a subprocess using os.system on Windows or
         subprocess.call using the system shell on other platforms.
 
         Parameters
         ----------
         cmd : str
             Command to execute.
         """
         cmd = self.var_expand(cmd, depth=1)
         # warn if there is an IPython magic alternative.
         if cmd == "":
             main_cmd = ""
         else:
             main_cmd = cmd.split()[0]
         has_magic_alternatives = ("pip", "conda", "cd")
 
         if main_cmd in has_magic_alternatives:
             warnings.warn(
                 (
                     "You executed the system command !{0} which may not work "
                     "as expected. Try the IPython magic %{0} instead."
                 ).format(main_cmd)
             )
 
         # protect os.system from UNC paths on Windows, which it can't handle:
         if sys.platform == 'win32':
             from IPython.utils._process_win32 import AvoidUNCPath
             with AvoidUNCPath() as path:
                 if path is not None:
                     cmd = '"pushd %s &&"%s' % (path, cmd)
                 try:
                     ec = os.system(cmd)
                 except KeyboardInterrupt:
                     print('\n' + self.get_exception_only(), file=sys.stderr)
                     ec = -2
         else:
             # For posix the result of the subprocess.call() below is an exit
             # code, which by convention is zero for success, positive for
             # program failure.  Exit codes above 128 are reserved for signals,
             # and the formula for converting a signal to an exit code is usually
             # signal_number+128.  To more easily differentiate between exit
             # codes and signals, ipython uses negative numbers.  For instance
             # since control-c is signal 2 but exit code 130, ipython's
             # _exit_code variable will read -2.  Note that some shells like
             # csh and fish don't follow sh/bash conventions for exit codes.
             executable = os.environ.get('SHELL', None)
             try:
                 # Use env shell instead of default /bin/sh
                 ec = subprocess.call(cmd, shell=True, executable=executable)
             except KeyboardInterrupt:
                 # intercept control-C; a long traceback is not useful here
                 print('\n' + self.get_exception_only(), file=sys.stderr)
                 ec = 130
             if ec > 128:
                 ec = -(ec - 128)
 
         # We explicitly do NOT return the subprocess status code, because
         # a non-None value would trigger :func:`sys.displayhook` calls.
         # Instead, we store the exit_code in user_ns.  Note the semantics
         # of _exit_code: for control-c, _exit_code == -signal.SIGNIT,
         # but raising SystemExit(_exit_code) will give status 254!
         self.user_ns['_exit_code'] = ec
 
     # use piped system by default, because it is better behaved
     system = system_piped
 
     def getoutput(self, cmd, split=True, depth=0):
         """Get output (possibly including stderr) from a subprocess.
 
         Parameters
         ----------
         cmd : str
             Command to execute (can not end in '&', as background processes are
             not supported.
         split : bool, optional
             If True, split the output into an IPython SList.  Otherwise, an
             IPython LSString is returned.  These are objects similar to normal
             lists and strings, with a few convenience attributes for easier
             manipulation of line-based output.  You can use '?' on them for
             details.
         depth : int, optional
             How many frames above the caller are the local variables which should
             be expanded in the command string? The default (0) assumes that the
             expansion variables are in the stack frame calling this function.
         """
         if cmd.rstrip().endswith('&'):
             # this is *far* from a rigorous test
             raise OSError("Background processes not supported.")
         out = getoutput(self.var_expand(cmd, depth=depth+1))
         if split:
             out = SList(out.splitlines())
         else:
             out = LSString(out)
         return out
 
     #-------------------------------------------------------------------------
     # Things related to aliases
     #-------------------------------------------------------------------------
 
     def init_alias(self):
         self.alias_manager = AliasManager(shell=self, parent=self)
         self.configurables.append(self.alias_manager)
 
     #-------------------------------------------------------------------------
     # Things related to extensions
     #-------------------------------------------------------------------------
 
     def init_extension_manager(self):
         self.extension_manager = ExtensionManager(shell=self, parent=self)
         self.configurables.append(self.extension_manager)
 
     #-------------------------------------------------------------------------
     # Things related to payloads
     #-------------------------------------------------------------------------
 
     def init_payload(self):
         self.payload_manager = PayloadManager(parent=self)
         self.configurables.append(self.payload_manager)
 
     #-------------------------------------------------------------------------
     # Things related to the prefilter
     #-------------------------------------------------------------------------
 
     def init_prefilter(self):
         self.prefilter_manager = PrefilterManager(shell=self, parent=self)
         self.configurables.append(self.prefilter_manager)
         # Ultimately this will be refactored in the new interpreter code, but
         # for now, we should expose the main prefilter method (there's legacy
         # code out there that may rely on this).
         self.prefilter = self.prefilter_manager.prefilter_lines
 
     def auto_rewrite_input(self, cmd):
         """Print to the screen the rewritten form of the user's command.
 
         This shows visual feedback by rewriting input lines that cause
         automatic calling to kick in, like::
 
           /f x
 
         into::
 
           ------> f(x)
 
         after the user's input prompt.  This helps the user understand that the
         input line was transformed automatically by IPython.
         """
         if not self.show_rewritten_input:
             return
 
         # This is overridden in TerminalInteractiveShell to use fancy prompts
         print("------> " + cmd)
 
     #-------------------------------------------------------------------------
     # Things related to extracting values/expressions from kernel and user_ns
     #-------------------------------------------------------------------------
 
     def _user_obj_error(self):
         """return simple exception dict
 
         for use in user_expressions
         """
 
         etype, evalue, tb = self._get_exc_info()
         stb = self.InteractiveTB.get_exception_only(etype, evalue)
 
         exc_info = {
             "status": "error",
             "traceback": stb,
             "ename": etype.__name__,
             "evalue": py3compat.safe_unicode(evalue),
         }
 
         return exc_info
 
     def _format_user_obj(self, obj):
         """format a user object to display dict
 
         for use in user_expressions
         """
 
         data, md = self.display_formatter.format(obj)
         value = {
             'status' : 'ok',
             'data' : data,
             'metadata' : md,
         }
         return value
 
     def user_expressions(self, expressions):
         """Evaluate a dict of expressions in the user's namespace.
 
         Parameters
         ----------
         expressions : dict
             A dict with string keys and string values.  The expression values
             should be valid Python expressions, each of which will be evaluated
             in the user namespace.
 
         Returns
         -------
         A dict, keyed like the input expressions dict, with the rich mime-typed
         display_data of each value.
         """
         out = {}
         user_ns = self.user_ns
         global_ns = self.user_global_ns
 
         for key, expr in expressions.items():
             try:
                 value = self._format_user_obj(eval(expr, global_ns, user_ns))
-            except Exception:
+            except:
                 value = self._user_obj_error()
             out[key] = value
         return out
 
     #-------------------------------------------------------------------------
     # Things related to the running of code
     #-------------------------------------------------------------------------
 
     def ex(self, cmd):
         """Execute a normal python statement in user namespace."""
         with self.builtin_trap:
             exec(cmd, self.user_global_ns, self.user_ns)
 
     def ev(self, expr):
         """Evaluate python expression expr in user namespace.
 
         Returns the result of evaluation
         """
         with self.builtin_trap:
             return eval(expr, self.user_global_ns, self.user_ns)
 
     def safe_execfile(self, fname, *where, exit_ignore=False, raise_exceptions=False, shell_futures=False):
         """A safe version of the builtin execfile().
 
         This version will never throw an exception, but instead print
         helpful error messages to the screen.  This only works on pure
         Python files with the .py extension.
 
         Parameters
         ----------
         fname : string
             The name of the file to be executed.
         *where : tuple
             One or two namespaces, passed to execfile() as (globals,locals).
             If only one is given, it is passed as both.
         exit_ignore : bool (False)
             If True, then silence SystemExit for non-zero status (it is always
             silenced for zero status, as it is so common).
         raise_exceptions : bool (False)
             If True raise exceptions everywhere. Meant for testing.
         shell_futures : bool (False)
             If True, the code will share future statements with the interactive
             shell. It will both be affected by previous __future__ imports, and
             any __future__ imports in the code will affect the shell. If False,
             __future__ imports are not shared in either direction.
 
         """
         fname = Path(fname).expanduser().resolve()
 
         # Make sure we can open the file
         try:
             with fname.open("rb"):
                 pass
-        except Exception:
+        except:
             warn('Could not open file <%s> for safe execution.' % fname)
             return
 
         # Find things also in current directory.  This is needed to mimic the
         # behavior of running a script from the system command line, where
         # Python inserts the script's directory into sys.path
         dname = str(fname.parent)
 
         with prepended_to_syspath(dname), self.builtin_trap:
             try:
                 glob, loc = (where + (None, ))[:2]
                 py3compat.execfile(
                     fname, glob, loc,
                     self.compile if shell_futures else None)
             except SystemExit as status:
                 # If the call was made with 0 or None exit status (sys.exit(0)
                 # or sys.exit() ), don't bother showing a traceback, as both of
                 # these are considered normal by the OS:
                 # > python -c'import sys;sys.exit(0)'; echo $?
                 # 0
                 # > python -c'import sys;sys.exit()'; echo $?
                 # 0
                 # For other exit status, we show the exception unless
                 # explicitly silenced, but only in short form.
                 if status.code:
                     if raise_exceptions:
                         raise
                     if not exit_ignore:
                         self.showtraceback(exception_only=True)
-            except Exception:
+            except:
                 if raise_exceptions:
                     raise
                 # tb offset is 2 because we wrap execfile
                 self.showtraceback(tb_offset=2)
 
     def safe_execfile_ipy(self, fname, shell_futures=False, raise_exceptions=False):
         """Like safe_execfile, but for .ipy or .ipynb files with IPython syntax.
 
         Parameters
         ----------
         fname : str
             The name of the file to execute.  The filename must have a
             .ipy or .ipynb extension.
         shell_futures : bool (False)
             If True, the code will share future statements with the interactive
             shell. It will both be affected by previous __future__ imports, and
             any __future__ imports in the code will affect the shell. If False,
             __future__ imports are not shared in either direction.
         raise_exceptions : bool (False)
             If True raise exceptions everywhere.  Meant for testing.
         """
         fname = Path(fname).expanduser().resolve()
 
         # Make sure we can open the file
         try:
             with fname.open("rb"):
                 pass
-        except Exception:
+        except:
             warn('Could not open file <%s> for safe execution.' % fname)
             return
 
         # Find things also in current directory.  This is needed to mimic the
         # behavior of running a script from the system command line, where
         # Python inserts the script's directory into sys.path
         dname = str(fname.parent)
 
         def get_cells():
             """generator for sequence of code blocks to run"""
             if fname.suffix == ".ipynb":
                 from nbformat import read
                 nb = read(fname, as_version=4)
                 if not nb.cells:
                     return
                 for cell in nb.cells:
                     if cell.cell_type == 'code':
                         yield cell.source
             else:
                 yield fname.read_text(encoding="utf-8")
 
         with prepended_to_syspath(dname):
             try:
                 for cell in get_cells():
                     result = self.run_cell(cell, silent=True, shell_futures=shell_futures)
                     if raise_exceptions:
                         result.raise_error()
                     elif not result.success:
                         break
-            except Exception:
+            except:
                 if raise_exceptions:
                     raise
                 self.showtraceback()
                 warn('Unknown failure executing file: <%s>' % fname)
 
     def safe_run_module(self, mod_name, where):
         """A safe version of runpy.run_module().
 
         This version will never throw an exception, but instead print
         helpful error messages to the screen.
 
         `SystemExit` exceptions with status code 0 or None are ignored.
 
         Parameters
         ----------
         mod_name : string
             The name of the module to be executed.
         where : dict
             The globals namespace.
         """
         try:
             try:
                 where.update(
                     runpy.run_module(str(mod_name), run_name="__main__",
                                      alter_sys=True)
                             )
             except SystemExit as status:
                 if status.code:
                     raise
-        except Exception:
+        except:
             self.showtraceback()
             warn('Unknown failure executing module: <%s>' % mod_name)
 
     def run_cell(
         self,
         raw_cell,
         store_history=False,
         silent=False,
         shell_futures=True,
         cell_id=None,
     ):
         """Run a complete IPython cell.
 
         Parameters
         ----------
         raw_cell : str
             The code (including IPython code such as %magic functions) to run.
         store_history : bool
             If True, the raw and translated cell will be stored in IPython's
             history. For user code calling back into IPython's machinery, this
             should be set to False.
         silent : bool
             If True, avoid side-effects, such as implicit displayhooks and
             and logging.  silent=True forces store_history=False.
         shell_futures : bool
             If True, the code will share future statements with the interactive
             shell. It will both be affected by previous __future__ imports, and
             any __future__ imports in the code will affect the shell. If False,
             __future__ imports are not shared in either direction.
 
         Returns
         -------
         result : :class:`ExecutionResult`
         """
         result = None
         try:
             result = self._run_cell(
                 raw_cell, store_history, silent, shell_futures, cell_id
             )
         finally:
             self.events.trigger('post_execute')
             if not silent:
                 self.events.trigger('post_run_cell', result)
         return result
 
     def _run_cell(
         self,
         raw_cell: str,
         store_history: bool,
         silent: bool,
         shell_futures: bool,
         cell_id: str,
     ) -> ExecutionResult:
         """Internal method to run a complete IPython cell."""
 
         # we need to avoid calling self.transform_cell multiple time on the same thing
         # so we need to store some results:
         preprocessing_exc_tuple = None
         try:
             transformed_cell = self.transform_cell(raw_cell)
         except Exception:
             transformed_cell = raw_cell
             preprocessing_exc_tuple = sys.exc_info()
 
         assert transformed_cell is not None
         coro = self.run_cell_async(
             raw_cell,
             store_history=store_history,
             silent=silent,
             shell_futures=shell_futures,
             transformed_cell=transformed_cell,
             preprocessing_exc_tuple=preprocessing_exc_tuple,
             cell_id=cell_id,
         )
 
         # run_cell_async is async, but may not actually need an eventloop.
         # when this is the case, we want to run it using the pseudo_sync_runner
         # so that code can invoke eventloops (for example via the %run , and
         # `%paste` magic.
         if self.trio_runner:
             runner = self.trio_runner
         elif self.should_run_async(
             raw_cell,
             transformed_cell=transformed_cell,
             preprocessing_exc_tuple=preprocessing_exc_tuple,
         ):
             runner = self.loop_runner
         else:
             runner = _pseudo_sync_runner
 
         try:
             result = runner(coro)
         except BaseException as e:
             info = ExecutionInfo(
                 raw_cell, store_history, silent, shell_futures, cell_id
             )
             result = ExecutionResult(info)
             result.error_in_exec = e
             self.showtraceback(running_compiled_code=True)
         finally:
             return result
 
     def should_run_async(
         self, raw_cell: str, *, transformed_cell=None, preprocessing_exc_tuple=None
     ) -> bool:
         """Return whether a cell should be run asynchronously via a coroutine runner
 
         Parameters
         ----------
         raw_cell : str
             The code to be executed
 
         Returns
         -------
         result: bool
             Whether the code needs to be run with a coroutine runner or not
         .. versionadded:: 7.0
         """
         if not self.autoawait:
             return False
         if preprocessing_exc_tuple is not None:
             return False
         assert preprocessing_exc_tuple is None
         if transformed_cell is None:
             warnings.warn(
                 "`should_run_async` will not call `transform_cell`"
                 " automatically in the future. Please pass the result to"
                 " `transformed_cell` argument and any exception that happen"
                 " during the"
                 "transform in `preprocessing_exc_tuple` in"
                 " IPython 7.17 and above.",
                 DeprecationWarning,
                 stacklevel=2,
             )
             try:
                 cell = self.transform_cell(raw_cell)
             except Exception:
                 # any exception during transform will be raised
                 # prior to execution
                 return False
         else:
             cell = transformed_cell
         return _should_be_async(cell)
 
     async def run_cell_async(
         self,
         raw_cell: str,
         store_history=False,
         silent=False,
         shell_futures=True,
         *,
         transformed_cell: Optional[str] = None,
         preprocessing_exc_tuple: Optional[AnyType] = None,
         cell_id=None,
     ) -> ExecutionResult:
         """Run a complete IPython cell asynchronously.
 
         Parameters
         ----------
         raw_cell : str
           The code (including IPython code such as %magic functions) to run.
         store_history : bool
           If True, the raw and translated cell will be stored in IPython's
           history. For user code calling back into IPython's machinery, this
           should be set to False.
         silent : bool
           If True, avoid side-effects, such as implicit displayhooks and
           and logging.  silent=True forces store_history=False.
         shell_futures : bool
           If True, the code will share future statements with the interactive
           shell. It will both be affected by previous __future__ imports, and
           any __future__ imports in the code will affect the shell. If False,
           __future__ imports are not shared in either direction.
         transformed_cell: str
           cell that was passed through transformers
         preprocessing_exc_tuple:
           trace if the transformation failed.
 
         Returns
         -------
         result : :class:`ExecutionResult`
 
         .. versionadded:: 7.0
         """
         info = ExecutionInfo(raw_cell, store_history, silent, shell_futures, cell_id)
         result = ExecutionResult(info)
 
         if (not raw_cell) or raw_cell.isspace():
             self.last_execution_succeeded = True
             self.last_execution_result = result
             return result
 
         if silent:
             store_history = False
 
         if store_history:
             result.execution_count = self.execution_count
 
         def error_before_exec(value):
             if store_history:
                 self.execution_count += 1
             result.error_before_exec = value
             self.last_execution_succeeded = False
             self.last_execution_result = result
             return result
 
         self.events.trigger('pre_execute')
         if not silent:
             self.events.trigger('pre_run_cell', info)
 
         if transformed_cell is None:
             warnings.warn(
                 "`run_cell_async` will not call `transform_cell`"
                 " automatically in the future. Please pass the result to"
                 " `transformed_cell` argument and any exception that happen"
                 " during the"
                 "transform in `preprocessing_exc_tuple` in"
                 " IPython 7.17 and above.",
                 DeprecationWarning,
                 stacklevel=2,
             )
             # If any of our input transformation (input_transformer_manager or
             # prefilter_manager) raises an exception, we store it in this variable
             # so that we can display the error after logging the input and storing
             # it in the history.
             try:
                 cell = self.transform_cell(raw_cell)
             except Exception:
                 preprocessing_exc_tuple = sys.exc_info()
                 cell = raw_cell  # cell has to exist so it can be stored/logged
             else:
                 preprocessing_exc_tuple = None
         else:
             if preprocessing_exc_tuple is None:
                 cell = transformed_cell
             else:
                 cell = raw_cell
 
         # Do NOT store paste/cpaste magic history
         if "get_ipython().run_line_magic(" in cell and "paste" in cell:
             store_history = False
 
         # Store raw and processed history
         if store_history:
             assert self.history_manager is not None
             self.history_manager.store_inputs(self.execution_count, cell, raw_cell)
         if not silent:
             self.logger.log(cell, raw_cell)
 
         # Display the exception if input processing failed.
         if preprocessing_exc_tuple is not None:
             self.showtraceback(preprocessing_exc_tuple)
             if store_history:
                 self.execution_count += 1
             return error_before_exec(preprocessing_exc_tuple[1])
 
         # Our own compiler remembers the __future__ environment. If we want to
         # run code with a separate __future__ environment, use the default
         # compiler
         compiler = self.compile if shell_futures else self.compiler_class()
 
         with self.builtin_trap:
             cell_name = compiler.cache(cell, self.execution_count, raw_code=raw_cell)
 
             with self.display_trap:
                 # Compile to bytecode
                 try:
                     code_ast = compiler.ast_parse(cell, filename=cell_name)
                 except self.custom_exceptions as e:
                     etype, value, tb = sys.exc_info()
                     self.CustomTB(etype, value, tb)
                     return error_before_exec(e)
                 except IndentationError as e:
                     self.showindentationerror()
                     return error_before_exec(e)
                 except (OverflowError, SyntaxError, ValueError, TypeError,
                         MemoryError) as e:
                     self.showsyntaxerror()
                     return error_before_exec(e)
 
                 # Apply AST transformations
                 try:
                     code_ast = self.transform_ast(code_ast)
                 except InputRejected as e:
                     self.showtraceback()
                     return error_before_exec(e)
 
                 # Give the displayhook a reference to our ExecutionResult so it
                 # can fill in the output value.
                 self.displayhook.exec_result = result
 
                 # Execute the user code
                 interactivity = "none" if silent else self.ast_node_interactivity
 
 
                 has_raised = await self.run_ast_nodes(code_ast.body, cell_name,
                        interactivity=interactivity, compiler=compiler, result=result)
 
                 self.last_execution_succeeded = not has_raised
                 self.last_execution_result = result
 
                 # Reset this so later displayed values do not modify the
                 # ExecutionResult
                 self.displayhook.exec_result = None
 
         if store_history:
             assert self.history_manager is not None
             # Write output to the database. Does nothing unless
             # history output logging is enabled.
             self.history_manager.store_output(self.execution_count)
             # Each cell is a *single* input, regardless of how many lines it has
             self.execution_count += 1
 
         return result
 
     def transform_cell(self, raw_cell):
         """Transform an input cell before parsing it.
 
         Static transformations, implemented in IPython.core.inputtransformer2,
         deal with things like ``%magic`` and ``!system`` commands.
         These run on all input.
         Dynamic transformations, for things like unescaped magics and the exit
         autocall, depend on the state of the interpreter.
         These only apply to single line inputs.
 
         These string-based transformations are followed by AST transformations;
         see :meth:`transform_ast`.
         """
         # Static input transformations
         cell = self.input_transformer_manager.transform_cell(raw_cell)
 
         if len(cell.splitlines()) == 1:
             # Dynamic transformations - only applied for single line commands
             with self.builtin_trap:
                 # use prefilter_lines to handle trailing newlines
                 # restore trailing newline for ast.parse
                 cell = self.prefilter_manager.prefilter_lines(cell) + '\n'
 
         lines = cell.splitlines(keepends=True)
         for transform in self.input_transformers_post:
             lines = transform(lines)
         cell = ''.join(lines)
 
         return cell
 
     def transform_ast(self, node):
         """Apply the AST transformations from self.ast_transformers
 
         Parameters
         ----------
         node : ast.Node
             The root node to be transformed. Typically called with the ast.Module
             produced by parsing user input.
 
         Returns
         -------
         An ast.Node corresponding to the node it was called with. Note that it
         may also modify the passed object, so don't rely on references to the
         original AST.
         """
         for transformer in self.ast_transformers:
             try:
                 node = transformer.visit(node)
             except InputRejected:
                 # User-supplied AST transformers can reject an input by raising
                 # an InputRejected.  Short-circuit in this case so that we
                 # don't unregister the transform.
                 raise
             except Exception as e:
                 warn(
                     "AST transformer %r threw an error. It will be unregistered. %s"
                     % (transformer, e)
                 )
                 self.ast_transformers.remove(transformer)
 
         if self.ast_transformers:
             ast.fix_missing_locations(node)
         return node
 
     async def run_ast_nodes(
         self,
         nodelist: ListType[stmt],
         cell_name: str,
         interactivity="last_expr",
         compiler=compile,
         result=None,
     ):
         """Run a sequence of AST nodes. The execution mode depends on the
         interactivity parameter.
 
         Parameters
         ----------
         nodelist : list
           A sequence of AST nodes to run.
         cell_name : str
           Will be passed to the compiler as the filename of the cell. Typically
           the value returned by ip.compile.cache(cell).
         interactivity : str
           'all', 'last', 'last_expr' , 'last_expr_or_assign' or 'none',
           specifying which nodes should be run interactively (displaying output
           from expressions). 'last_expr' will run the last node interactively
           only if it is an expression (i.e. expressions in loops or other blocks
           are not displayed) 'last_expr_or_assign' will run the last expression
           or the last assignment. Other values for this parameter will raise a
           ValueError.
 
         compiler : callable
           A function with the same interface as the built-in compile(), to turn
           the AST nodes into code objects. Default is the built-in compile().
         result : ExecutionResult, optional
           An object to store exceptions that occur during execution.
 
         Returns
         -------
         True if an exception occurred while running code, False if it finished
         running.
         """
         if not nodelist:
             return
 
 
         if interactivity == 'last_expr_or_assign':
             if isinstance(nodelist[-1], _assign_nodes):
                 asg = nodelist[-1]
                 if isinstance(asg, ast.Assign) and len(asg.targets) == 1:
                     target = asg.targets[0]
                 elif isinstance(asg, _single_targets_nodes):
                     target = asg.target
                 else:
                     target = None
                 if isinstance(target, ast.Name):
                     nnode = ast.Expr(ast.Name(target.id, ast.Load()))
                     ast.fix_missing_locations(nnode)
                     nodelist.append(nnode)
             interactivity = 'last_expr'
 
         _async = False
         if interactivity == 'last_expr':
             if isinstance(nodelist[-1], ast.Expr):
                 interactivity = "last"
             else:
                 interactivity = "none"
 
         if interactivity == 'none':
             to_run_exec, to_run_interactive = nodelist, []
         elif interactivity == 'last':
             to_run_exec, to_run_interactive = nodelist[:-1], nodelist[-1:]
         elif interactivity == 'all':
             to_run_exec, to_run_interactive = [], nodelist
         else:
             raise ValueError("Interactivity was %r" % interactivity)
 
         try:
 
             def compare(code):
                 is_async = inspect.CO_COROUTINE & code.co_flags == inspect.CO_COROUTINE
                 return is_async
 
             # refactor that to just change the mod constructor.
             to_run = []
             for node in to_run_exec:
                 to_run.append((node, "exec"))
 
             for node in to_run_interactive:
                 to_run.append((node, "single"))
 
             for node, mode in to_run:
                 if mode == "exec":
                     mod = Module([node], [])
                 elif mode == "single":
                     mod = ast.Interactive([node])  # type: ignore
                 with compiler.extra_flags(
                     getattr(ast, "PyCF_ALLOW_TOP_LEVEL_AWAIT", 0x0)
                     if self.autoawait
                     else 0x0
                 ):
                     code = compiler(mod, cell_name, mode)
                     asy = compare(code)
                 if await self.run_code(code, result, async_=asy):
                     return True
 
             # Flush softspace
             if softspace(sys.stdout, 0):
                 print()
 
-        except Exception:
+        except:
             # It's possible to have exceptions raised here, typically by
             # compilation of odd code (such as a naked 'return' outside a
             # function) that did parse but isn't valid. Typically the exception
             # is a SyntaxError, but it's safest just to catch anything and show
             # the user a traceback.
 
             # We do only one try/except outside the loop to minimize the impact
             # on runtime, and also because if any node in the node list is
             # broken, we should stop execution completely.
             if result:
                 result.error_before_exec = sys.exc_info()[1]
             self.showtraceback()
             return True
 
         return False
 
     async def run_code(self, code_obj, result=None, *, async_=False):
         """Execute a code object.
 
         When an exception occurs, self.showtraceback() is called to display a
         traceback.
 
         Parameters
         ----------
         code_obj : code object
           A compiled code object, to be executed
         result : ExecutionResult, optional
           An object to store exceptions that occur during execution.
         async_ :  Bool (Experimental)
           Attempt to run top-level asynchronous code in a default loop.
 
         Returns
         -------
         False : successful execution.
         True : an error occurred.
         """
         # special value to say that anything above is IPython and should be
         # hidden.
         __tracebackhide__ = "__ipython_bottom__"
         # Set our own excepthook in case the user code tries to call it
         # directly, so that the IPython crash handler doesn't get triggered
         old_excepthook, sys.excepthook = sys.excepthook, self.excepthook
 
         # we save the original sys.excepthook in the instance, in case config
         # code (such as magics) needs access to it.
         self.sys_excepthook = old_excepthook
         outflag = True  # happens in more places, so it's easier as default
         try:
             try:
                 if async_:
                     await eval(code_obj, self.user_global_ns, self.user_ns)
                 else:
                     exec(code_obj, self.user_global_ns, self.user_ns)
             finally:
                 # Reset our crash handler in place
                 sys.excepthook = old_excepthook
         except SystemExit as e:
             if result is not None:
                 result.error_in_exec = e
             self.showtraceback(exception_only=True)
             warn("To exit: use 'exit', 'quit', or Ctrl-D.", stacklevel=1)
         except bdb.BdbQuit:
             etype, value, tb = sys.exc_info()
             if result is not None:
                 result.error_in_exec = value
             # the BdbQuit stops here
         except self.custom_exceptions:
             etype, value, tb = sys.exc_info()
             if result is not None:
                 result.error_in_exec = value
             self.CustomTB(etype, value, tb)
-        except Exception:
+        except:
             if result is not None:
                 result.error_in_exec = sys.exc_info()[1]
             self.showtraceback(running_compiled_code=True)
         else:
             outflag = False
         return outflag
 
     # For backwards compatibility
     runcode = run_code
 
     def check_complete(self, code: str) -> Tuple[str, str]:
         """Return whether a block of code is ready to execute, or should be continued
 
         Parameters
         ----------
         code : string
             Python input code, which can be multiline.
 
         Returns
         -------
         status : str
             One of 'complete', 'incomplete', or 'invalid' if source is not a
             prefix of valid code.
         indent : str
             When status is 'incomplete', this is some whitespace to insert on
             the next line of the prompt.
         """
         status, nspaces = self.input_transformer_manager.check_complete(code)
         return status, ' ' * (nspaces or 0)
 
     #-------------------------------------------------------------------------
     # Things related to GUI support and pylab
     #-------------------------------------------------------------------------
 
     active_eventloop: Optional[str] = None
 
     def enable_gui(self, gui=None):
         raise NotImplementedError('Implement enable_gui in a subclass')
 
     def enable_matplotlib(self, gui=None):
         """Enable interactive matplotlib and inline figure support.
 
         This takes the following steps:
 
         1. select the appropriate eventloop and matplotlib backend
         2. set up matplotlib for interactive use with that backend
         3. configure formatters for inline figure display
         4. enable the selected gui eventloop
 
         Parameters
         ----------
         gui : optional, string
             If given, dictates the choice of matplotlib GUI backend to use
             (should be one of IPython's supported backends, 'qt', 'osx', 'tk',
             'gtk', 'wx' or 'inline'), otherwise we use the default chosen by
             matplotlib (as dictated by the matplotlib build-time options plus the
             user's matplotlibrc configuration file).  Note that not all backends
             make sense in all contexts, for example a terminal ipython can't
             display figures inline.
         """
         from .pylabtools import _matplotlib_manages_backends
 
         if not _matplotlib_manages_backends() and gui in (None, "auto"):
             # Early import of backend_inline required for its side effect of
             # calling _enable_matplotlib_integration()
-            import matplotlib_inline.backend_inline  # noqa : F401
+            import matplotlib_inline.backend_inline
 
         from IPython.core import pylabtools as pt
         gui, backend = pt.find_gui_and_backend(gui, self.pylab_gui_select)
 
-        if gui is not None:
+        if gui != None:
             # If we have our first gui selection, store it
             if self.pylab_gui_select is None:
                 self.pylab_gui_select = gui
             # Otherwise if they are different
             elif gui != self.pylab_gui_select:
                 print('Warning: Cannot change to a different GUI toolkit: %s.'
                         ' Using %s instead.' % (gui, self.pylab_gui_select))
                 gui, backend = pt.find_gui_and_backend(self.pylab_gui_select)
 
         pt.activate_matplotlib(backend)
 
         from matplotlib_inline.backend_inline import configure_inline_support
 
         configure_inline_support(self, backend)
 
         # Now we must activate the gui pylab wants to use, and fix %run to take
         # plot updates into account
         self.enable_gui(gui)
         self.magics_manager.registry['ExecutionMagics'].default_runner = \
             pt.mpl_runner(self.safe_execfile)
 
         return gui, backend
 
     def enable_pylab(self, gui=None, import_all=True, welcome_message=False):
         """Activate pylab support at runtime.
 
         This turns on support for matplotlib, preloads into the interactive
         namespace all of numpy and pylab, and configures IPython to correctly
         interact with the GUI event loop.  The GUI backend to be used can be
         optionally selected with the optional ``gui`` argument.
 
         This method only adds preloading the namespace to InteractiveShell.enable_matplotlib.
 
         Parameters
         ----------
         gui : optional, string
             If given, dictates the choice of matplotlib GUI backend to use
             (should be one of IPython's supported backends, 'qt', 'osx', 'tk',
             'gtk', 'wx' or 'inline'), otherwise we use the default chosen by
             matplotlib (as dictated by the matplotlib build-time options plus the
             user's matplotlibrc configuration file).  Note that not all backends
             make sense in all contexts, for example a terminal ipython can't
             display figures inline.
         import_all : optional, bool, default: True
             Whether to do `from numpy import *` and `from pylab import *`
             in addition to module imports.
         welcome_message : deprecated
             This argument is ignored, no welcome message will be displayed.
         """
         from IPython.core.pylabtools import import_pylab
 
         gui, backend = self.enable_matplotlib(gui)
 
         # We want to prevent the loading of pylab to pollute the user's
         # namespace as shown by the %who* magics, so we execute the activation
         # code in an empty namespace, and we update *both* user_ns and
         # user_ns_hidden with this information.
         ns = {}
         import_pylab(ns, import_all)
         # warn about clobbered names
         ignored = {"__builtins__"}
         both = set(ns).intersection(self.user_ns).difference(ignored)
         clobbered = [ name for name in both if self.user_ns[name] is not ns[name] ]
         self.user_ns.update(ns)
         self.user_ns_hidden.update(ns)
         return gui, backend, clobbered
 
     #-------------------------------------------------------------------------
     # Utilities
     #-------------------------------------------------------------------------
 
     def var_expand(self, cmd, depth=0, formatter=DollarFormatter()):
         """Expand python variables in a string.
 
         The depth argument indicates how many frames above the caller should
         be walked to look for the local namespace where to expand variables.
 
         The global namespace for expansion is always the user's interactive
         namespace.
         """
         ns = self.user_ns.copy()
         try:
             frame = sys._getframe(depth+1)
         except ValueError:
             # This is thrown if there aren't that many frames on the stack,
             # e.g. if a script called run_line_magic() directly.
             pass
         else:
             ns.update(frame.f_locals)
 
         try:
             # We have to use .vformat() here, because 'self' is a valid and common
             # name, and expanding **ns for .format() would make it collide with
             # the 'self' argument of the method.
             cmd = formatter.vformat(cmd, args=[], kwargs=ns)
         except Exception:
             # if formatter couldn't format, just let it go untransformed
             pass
         return cmd
 
     def mktempfile(self, data=None, prefix='ipython_edit_'):
         """Make a new tempfile and return its filename.
 
         This makes a call to tempfile.mkstemp (created in a tempfile.mkdtemp),
         but it registers the created filename internally so ipython cleans it up
         at exit time.
 
         Optional inputs:
 
           - data(None): if data is given, it gets written out to the temp file
             immediately, and the file is closed again."""
 
         dir_path = Path(tempfile.mkdtemp(prefix=prefix))
         self.tempdirs.append(dir_path)
 
         handle, filename = tempfile.mkstemp(".py", prefix, dir=str(dir_path))
         os.close(handle)  # On Windows, there can only be one open handle on a file
 
         file_path = Path(filename)
         self.tempfiles.append(file_path)
 
         if data:
             file_path.write_text(data, encoding="utf-8")
         return filename
 
     def ask_yes_no(self, prompt, default=None, interrupt=None):
         if self.quiet:
             return True
         return ask_yes_no(prompt,default,interrupt)
 
     def show_usage(self):
         """Show a usage message"""
         page.page(IPython.core.usage.interactive_usage)
 
     def extract_input_lines(self, range_str, raw=False):
         """Return as a string a set of input history slices.
 
         Parameters
         ----------
         range_str : str
             The set of slices is given as a string, like "~5/6-~4/2 4:8 9",
             since this function is for use by magic functions which get their
             arguments as strings. The number before the / is the session
             number: ~n goes n back from the current session.
 
             If empty string is given, returns history of current session
             without the last input.
 
         raw : bool, optional
             By default, the processed input is used.  If this is true, the raw
             input history is used instead.
 
         Notes
         -----
         Slices can be described with two notations:
 
         * ``N:M`` -> standard python form, means including items N...(M-1).
         * ``N-M`` -> include items N..M (closed endpoint).
         """
         lines = self.history_manager.get_range_by_str(range_str, raw=raw)
         text = "\n".join(x for _, _, x in lines)
 
         # Skip the last line, as it's probably the magic that called this
         if not range_str:
             if "\n" not in text:
                 text = ""
             else:
                 text = text[: text.rfind("\n")]
 
         return text
 
     def find_user_code(self, target, raw=True, py_only=False, skip_encoding_cookie=True, search_ns=False):
         """Get a code string from history, file, url, or a string or macro.
 
         This is mainly used by magic functions.
 
         Parameters
         ----------
         target : str
             A string specifying code to retrieve. This will be tried respectively
             as: ranges of input history (see %history for syntax), url,
             corresponding .py file, filename, or an expression evaluating to a
             string or Macro in the user namespace.
 
             If empty string is given, returns complete history of current
             session, without the last line.
 
         raw : bool
             If true (default), retrieve raw history. Has no effect on the other
             retrieval mechanisms.
 
         py_only : bool (default False)
             Only try to fetch python code, do not try alternative methods to decode file
             if unicode fails.
 
         Returns
         -------
         A string of code.
         ValueError is raised if nothing is found, and TypeError if it evaluates
         to an object of another type. In each case, .args[0] is a printable
         message.
         """
         code = self.extract_input_lines(target, raw=raw)  # Grab history
         if code:
             return code
         try:
             if target.startswith(('http://', 'https://')):
                 return openpy.read_py_url(target, skip_encoding_cookie=skip_encoding_cookie)
         except UnicodeDecodeError as e:
             if not py_only :
                 # Deferred import
                 from urllib.request import urlopen
                 response = urlopen(target)
                 return response.read().decode('latin1')
             raise ValueError(("'%s' seem to be unreadable.") % target) from e
 
         potential_target = [target]
         try :
             potential_target.insert(0,get_py_filename(target))
         except IOError:
             pass
 
         for tgt in potential_target :
             if os.path.isfile(tgt):                        # Read file
                 try :
                     return openpy.read_py_file(tgt, skip_encoding_cookie=skip_encoding_cookie)
                 except UnicodeDecodeError as e:
                     if not py_only :
                         with io_open(tgt,'r', encoding='latin1') as f :
                             return f.read()
                     raise ValueError(("'%s' seem to be unreadable.") % target) from e
             elif os.path.isdir(os.path.expanduser(tgt)):
                 raise ValueError("'%s' is a directory, not a regular file." % target)
 
         if search_ns:
             # Inspect namespace to load object source
             object_info = self.object_inspect(target, detail_level=1)
             if object_info['found'] and object_info['source']:
                 return object_info['source']
 
         try:                                              # User namespace
             codeobj = eval(target, self.user_ns)
         except Exception as e:
             raise ValueError(("'%s' was not found in history, as a file, url, "
                                 "nor in the user namespace.") % target) from e
 
         if isinstance(codeobj, str):
             return codeobj
         elif isinstance(codeobj, Macro):
             return codeobj.value
 
         raise TypeError("%s is neither a string nor a macro." % target,
                         codeobj)
 
     def _atexit_once(self):
         """
         At exist operation that need to be called at most once.
         Second call to this function per instance will do nothing.
         """
 
         if not getattr(self, "_atexit_once_called", False):
             self._atexit_once_called = True
             # Clear all user namespaces to release all references cleanly.
             self.reset(new_session=False)
             # Close the history session (this stores the end time and line count)
             # this must be *before* the tempfile cleanup, in case of temporary
             # history db
             self.history_manager.end_session()
             self.history_manager = None
 
     #-------------------------------------------------------------------------
     # Things related to IPython exiting
     #-------------------------------------------------------------------------
     def atexit_operations(self):
         """This will be executed at the time of exit.
 
         Cleanup operations and saving of persistent data that is done
         unconditionally by IPython should be performed here.
 
         For things that may depend on startup flags or platform specifics (such
         as having readline or not), register a separate atexit function in the
         code that has the appropriate information, rather than trying to
         clutter
         """
         self._atexit_once()
 
         # Cleanup all tempfiles and folders left around
         for tfile in self.tempfiles:
             try:
                 tfile.unlink()
                 self.tempfiles.remove(tfile)
             except FileNotFoundError:
                 pass
         del self.tempfiles
         for tdir in self.tempdirs:
             try:
                 shutil.rmtree(tdir)
                 self.tempdirs.remove(tdir)
             except FileNotFoundError:
                 pass
         del self.tempdirs
 
         # Restore user's cursor
         if hasattr(self, "editing_mode") and self.editing_mode == "vi":
             sys.stdout.write("\x1b[0 q")
             sys.stdout.flush()
 
     def cleanup(self):
         self.restore_sys_module_state()
 
 
     # Overridden in terminal subclass to change prompts
     def switch_doctest_mode(self, mode):
         pass
 
 
 class InteractiveShellABC(metaclass=abc.ABCMeta):
     """An abstract base class for InteractiveShell."""
 
 InteractiveShellABC.register(InteractiveShell)

IPython/core/magics/__init__.py

@@ -1,45 +1,42
 """Implementation of all the magic functions built into IPython.
 """
-
-# -----------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
 #  Copyright (c) 2012 The IPython Development Team.
 #
 #  Distributed under the terms of the Modified BSD License.
 #
 #  The full license is in the file COPYING.txt, distributed with this software.
-# -----------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
 
-# -----------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
 # Imports
-# -----------------------------------------------------------------------------
-
-from ..magic import Magics as Magics, magics_class as magics_class
-from .auto import AutoMagics as AutoMagics
-from .basic import BasicMagics as BasicMagics, AsyncMagics as AsyncMagics
-from .code import CodeMagics as CodeMagics, MacroToEdit as MacroToEdit
-from .config import ConfigMagics as ConfigMagics
-from .display import DisplayMagics as DisplayMagics
-from .execution import ExecutionMagics as ExecutionMagics
-from .extension import ExtensionMagics as ExtensionMagics
-from .history import HistoryMagics as HistoryMagics
-from .logging import LoggingMagics as LoggingMagics
-from .namespace import NamespaceMagics as NamespaceMagics
-from .osm import OSMagics as OSMagics
-from .packaging import PackagingMagics as PackagingMagics
-from .pylab import PylabMagics as PylabMagics
-from .script import ScriptMagics as ScriptMagics
-
-
-# -----------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
+
+from ..magic import Magics, magics_class
+from .auto import AutoMagics
+from .basic import BasicMagics, AsyncMagics
+from .code import CodeMagics, MacroToEdit
+from .config import ConfigMagics
+from .display import DisplayMagics
+from .execution import ExecutionMagics
+from .extension import ExtensionMagics
+from .history import HistoryMagics
+from .logging import LoggingMagics
+from .namespace import NamespaceMagics
+from .osm import OSMagics
+from .packaging import PackagingMagics
+from .pylab import PylabMagics
+from .script import ScriptMagics
+
+#-----------------------------------------------------------------------------
 # Magic implementation classes
-# -----------------------------------------------------------------------------
-
+#-----------------------------------------------------------------------------
 
 @magics_class
 class UserMagics(Magics):
     """Placeholder for user-defined magics to be added at runtime.
 
     All magics are eventually merged into a single namespace at runtime, but we
     use this class to isolate the magics defined dynamically by the user into
     their own class.
     """

IPython/core/magics/auto.py

@@ -1,144 +1,144
 """Implementation of magic functions that control various automatic behaviors.
 """
 #-----------------------------------------------------------------------------
 #  Copyright (c) 2012 The IPython Development Team.
 #
 #  Distributed under the terms of the Modified BSD License.
 #
 #  The full license is in the file COPYING.txt, distributed with this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 # Our own packages
 from IPython.core.magic import Bunch, Magics, magics_class, line_magic
 from IPython.testing.skipdoctest import skip_doctest
 from logging import error
 
 #-----------------------------------------------------------------------------
 # Magic implementation classes
 #-----------------------------------------------------------------------------
 
 @magics_class
 class AutoMagics(Magics):
     """Magics that control various autoX behaviors."""
 
     def __init__(self, shell):
         super(AutoMagics, self).__init__(shell)
         # namespace for holding state we may need
         self._magic_state = Bunch()
 
     @line_magic
     def automagic(self, parameter_s=''):
         """Make magic functions callable without having to type the initial %.
 
         Without arguments toggles on/off (when off, you must call it as
         %automagic, of course).  With arguments it sets the value, and you can
         use any of (case insensitive):
 
          - on, 1, True: to activate
 
          - off, 0, False: to deactivate.
 
         Note that magic functions have lowest priority, so if there's a
         variable whose name collides with that of a magic fn, automagic won't
         work for that function (you get the variable instead). However, if you
         delete the variable (del var), the previously shadowed magic function
         becomes visible to automagic again."""
 
         arg = parameter_s.lower()
         mman = self.shell.magics_manager
         if arg in ('on', '1', 'true'):
             val = True
         elif arg in ('off', '0', 'false'):
             val = False
         else:
             val = not mman.auto_magic
         mman.auto_magic = val
         print('\n' + self.shell.magics_manager.auto_status())
 
     @skip_doctest
     @line_magic
     def autocall(self, parameter_s=''):
         """Make functions callable without having to type parentheses.
 
         Usage:
 
            %autocall [mode]
 
         The mode can be one of: 0->Off, 1->Smart, 2->Full.  If not given, the
         value is toggled on and off (remembering the previous state).
 
         In more detail, these values mean:
 
         0 -> fully disabled
 
         1 -> active, but do not apply if there are no arguments on the line.
 
         In this mode, you get::
 
           In [1]: callable
           Out[1]: <built-in function callable>
 
           In [2]: callable 'hello'
           ------> callable('hello')
           Out[2]: False
 
         2 -> Active always.  Even if no arguments are present, the callable
         object is called::
 
           In [2]: float
           ------> float()
           Out[2]: 0.0
 
         Note that even with autocall off, you can still use '/' at the start of
         a line to treat the first argument on the command line as a function
         and add parentheses to it::
 
           In [8]: /str 43
           ------> str(43)
           Out[8]: '43'
 
         # all-random (note for auto-testing)
         """
 
         valid_modes = {
             0: "Off",
             1: "Smart",
             2: "Full",
         }
 
         def errorMessage() -> str:
             error = "Valid modes: "
             for k, v in valid_modes.items():
                 error += str(k) + "->" + v + ", "
             error = error[:-2]  # remove tailing `, ` after last element
             return error
 
         if parameter_s:
-            if parameter_s not in map(str, valid_modes.keys()):
+            if not parameter_s in map(str, valid_modes.keys()):
                 error(errorMessage())
                 return
             arg = int(parameter_s)
         else:
             arg = 'toggle'
 
-        if arg not in (*list(valid_modes.keys()), "toggle"):
+        if not arg in (*list(valid_modes.keys()), "toggle"):
             error(errorMessage())
             return
 
         if arg in (valid_modes.keys()):
             self.shell.autocall = arg
         else: # toggle
             if self.shell.autocall:
                 self._magic_state.autocall_save = self.shell.autocall
                 self.shell.autocall = 0
             else:
                 try:
                     self.shell.autocall = self._magic_state.autocall_save
                 except AttributeError:
                     self.shell.autocall = self._magic_state.autocall_save = 1
 
         print("Automatic calling is:", list(valid_modes.values())[self.shell.autocall])

IPython/core/magics/execution.py

@@ -1,1616 +1,1624
 # -*- coding: utf-8 -*-
 """Implementation of execution-related magic functions."""
 
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 
 import ast
 import bdb
 import builtins as builtin_mod
+import copy
 import cProfile as profile
 import gc
 import itertools
 import math
 import os
 import pstats
 import re
 import shlex
 import sys
 import time
 import timeit
 from typing import Dict, Any
 from ast import (
+    Assign,
+    Call,
+    Expr,
+    Load,
     Module,
+    Name,
+    NodeTransformer,
+    Store,
+    parse,
+    unparse,
 )
 from io import StringIO
 from logging import error
 from pathlib import Path
 from pdb import Restart
-from textwrap import indent
+from textwrap import dedent, indent
 from warnings import warn
 
-from IPython.core import magic_arguments, page
+from IPython.core import magic_arguments, oinspect, page
 from IPython.core.displayhook import DisplayHook
 from IPython.core.error import UsageError
 from IPython.core.macro import Macro
 from IPython.core.magic import (
     Magics,
     cell_magic,
     line_cell_magic,
     line_magic,
     magics_class,
     needs_local_scope,
     no_var_expand,
     on_off,
     output_can_be_silenced,
 )
 from IPython.testing.skipdoctest import skip_doctest
 from IPython.utils.capture import capture_output
 from IPython.utils.contexts import preserve_keys
 from IPython.utils.ipstruct import Struct
 from IPython.utils.module_paths import find_mod
 from IPython.utils.path import get_py_filename, shellglob
 from IPython.utils.timing import clock, clock2
 from IPython.core.magics.ast_mod import ReplaceCodeTransformer
 
 #-----------------------------------------------------------------------------
 # Magic implementation classes
 #-----------------------------------------------------------------------------
 
 
 class TimeitResult(object):
     """
     Object returned by the timeit magic with info about the run.
 
     Contains the following attributes :
 
     loops: (int) number of loops done per measurement
     repeat: (int) number of times the measurement has been repeated
     best: (float) best execution time / number
     all_runs: (list of float) execution time of each run (in s)
     compile_time: (float) time of statement compilation (s)
 
     """
     def __init__(self, loops, repeat, best, worst, all_runs, compile_time, precision):
         self.loops = loops
         self.repeat = repeat
         self.best = best
         self.worst = worst
         self.all_runs = all_runs
         self.compile_time = compile_time
         self._precision = precision
         self.timings = [ dt / self.loops for dt in all_runs]
 
     @property
     def average(self):
         return math.fsum(self.timings) / len(self.timings)
 
     @property
     def stdev(self):
         mean = self.average
         return (math.fsum([(x - mean) ** 2 for x in self.timings]) / len(self.timings)) ** 0.5
 
     def __str__(self):
         pm = '+-'
         if hasattr(sys.stdout, 'encoding') and sys.stdout.encoding:
             try:
                 u'\xb1'.encode(sys.stdout.encoding)
                 pm = u'\xb1'
             except:
                 pass
         return "{mean} {pm} {std} per loop (mean {pm} std. dev. of {runs} run{run_plural}, {loops:,} loop{loop_plural} each)".format(
             pm=pm,
             runs=self.repeat,
             loops=self.loops,
             loop_plural="" if self.loops == 1 else "s",
             run_plural="" if self.repeat == 1 else "s",
             mean=_format_time(self.average, self._precision),
             std=_format_time(self.stdev, self._precision),
         )
 
     def _repr_pretty_(self, p , cycle):
         unic = self.__str__()
         p.text(u'<TimeitResult : '+unic+u'>')
 
 
 class TimeitTemplateFiller(ast.NodeTransformer):
     """Fill in the AST template for timing execution.
 
     This is quite closely tied to the template definition, which is in
     :meth:`ExecutionMagics.timeit`.
     """
     def __init__(self, ast_setup, ast_stmt):
         self.ast_setup = ast_setup
         self.ast_stmt = ast_stmt
 
     def visit_FunctionDef(self, node):
         "Fill in the setup statement"
         self.generic_visit(node)
         if node.name == "inner":
             node.body[:1] = self.ast_setup.body
 
         return node
 
     def visit_For(self, node):
         "Fill in the statement to be timed"
         if getattr(getattr(node.body[0], 'value', None), 'id', None) == 'stmt':
             node.body = self.ast_stmt.body
         return node
 
 
 class Timer(timeit.Timer):
     """Timer class that explicitly uses self.inner
     
     which is an undocumented implementation detail of CPython,
     not shared by PyPy.
     """
     # Timer.timeit copied from CPython 3.4.2
     def timeit(self, number=timeit.default_number):
         """Time 'number' executions of the main statement.
 
         To be precise, this executes the setup statement once, and
         then returns the time it takes to execute the main statement
         a number of times, as a float measured in seconds.  The
         argument is the number of times through the loop, defaulting
         to one million.  The main statement, the setup statement and
         the timer function to be used are passed to the constructor.
         """
         it = itertools.repeat(None, number)
         gcold = gc.isenabled()
         gc.disable()
         try:
             timing = self.inner(it, self.timer)
         finally:
             if gcold:
                 gc.enable()
         return timing
 
 
 @magics_class
 class ExecutionMagics(Magics):
     """Magics related to code execution, debugging, profiling, etc."""
 
     _transformers: Dict[str, Any] = {}
 
     def __init__(self, shell):
         super(ExecutionMagics, self).__init__(shell)
         # Default execution function used to actually run user code.
         self.default_runner = None
 
     @skip_doctest
     @no_var_expand
     @line_cell_magic
     def prun(self, parameter_s='', cell=None):
 
         """Run a statement through the python code profiler.
 
         **Usage, in line mode:**
 
           %prun [options] statement
 
         **Usage, in cell mode:**
 
           %%prun [options] [statement]
 
           code...
 
           code...
 
         In cell mode, the additional code lines are appended to the (possibly
         empty) statement in the first line.  Cell mode allows you to easily
         profile multiline blocks without having to put them in a separate
         function.
 
         The given statement (which doesn't require quote marks) is run via the
         python profiler in a manner similar to the profile.run() function.
         Namespaces are internally managed to work correctly; profile.run
         cannot be used in IPython because it makes certain assumptions about
         namespaces which do not hold under IPython.
 
         Options:
 
         -l <limit>
           you can place restrictions on what or how much of the
           profile gets printed. The limit value can be:
 
              * A string: only information for function names containing this string
                is printed.
 
              * An integer: only these many lines are printed.
 
              * A float (between 0 and 1): this fraction of the report is printed
                (for example, use a limit of 0.4 to see the topmost 40% only).
 
           You can combine several limits with repeated use of the option. For
           example, ``-l __init__ -l 5`` will print only the topmost 5 lines of
           information about class constructors.
 
         -r
           return the pstats.Stats object generated by the profiling. This
           object has all the information about the profile in it, and you can
           later use it for further analysis or in other functions.
 
         -s <key>
           sort profile by given key. You can provide more than one key
           by using the option several times: '-s key1 -s key2 -s key3...'. The
           default sorting key is 'time'.
 
           The following is copied verbatim from the profile documentation
           referenced below:
 
           When more than one key is provided, additional keys are used as
           secondary criteria when the there is equality in all keys selected
           before them.
 
           Abbreviations can be used for any key names, as long as the
           abbreviation is unambiguous.  The following are the keys currently
           defined:
 
           ============  =====================
           Valid Arg     Meaning
           ============  =====================
           "calls"       call count
           "cumulative"  cumulative time
           "file"        file name
           "module"      file name
           "pcalls"      primitive call count
           "line"        line number
           "name"        function name
           "nfl"         name/file/line
           "stdname"     standard name
           "time"        internal time
           ============  =====================
 
           Note that all sorts on statistics are in descending order (placing
           most time consuming items first), where as name, file, and line number
           searches are in ascending order (i.e., alphabetical). The subtle
           distinction between "nfl" and "stdname" is that the standard name is a
           sort of the name as printed, which means that the embedded line
           numbers get compared in an odd way.  For example, lines 3, 20, and 40
           would (if the file names were the same) appear in the string order
           "20" "3" and "40".  In contrast, "nfl" does a numeric compare of the
           line numbers.  In fact, sort_stats("nfl") is the same as
           sort_stats("name", "file", "line").
 
         -T <filename>
           save profile results as shown on screen to a text
           file. The profile is still shown on screen.
 
         -D <filename>
           save (via dump_stats) profile statistics to given
           filename. This data is in a format understood by the pstats module, and
           is generated by a call to the dump_stats() method of profile
           objects. The profile is still shown on screen.
 
         -q
           suppress output to the pager.  Best used with -T and/or -D above.
 
         If you want to run complete programs under the profiler's control, use
         ``%run -p [prof_opts] filename.py [args to program]`` where prof_opts
         contains profiler specific options as described here.
 
         You can read the complete documentation for the profile module with::
 
           In [1]: import profile; profile.help()
 
         .. versionchanged:: 7.3
             User variables are no longer expanded,
             the magic line is always left unmodified.
 
         """
         opts, arg_str = self.parse_options(parameter_s, 'D:l:rs:T:q',
                                            list_all=True, posix=False)
         if cell is not None:
             arg_str += '\n' + cell
         arg_str = self.shell.transform_cell(arg_str)
         return self._run_with_profiler(arg_str, opts, self.shell.user_ns)
 
     def _run_with_profiler(self, code, opts, namespace):
         """
         Run `code` with profiler.  Used by ``%prun`` and ``%run -p``.
 
         Parameters
         ----------
         code : str
             Code to be executed.
         opts : Struct
             Options parsed by `self.parse_options`.
         namespace : dict
             A dictionary for Python namespace (e.g., `self.shell.user_ns`).
 
         """
 
         # Fill default values for unspecified options:
         opts.merge(Struct(D=[''], l=[], s=['time'], T=['']))
 
         prof = profile.Profile()
         try:
             prof = prof.runctx(code, namespace, namespace)
             sys_exit = ''
         except SystemExit:
             sys_exit = """*** SystemExit exception caught in code being profiled."""
 
         stats = pstats.Stats(prof).strip_dirs().sort_stats(*opts.s)
 
         lims = opts.l
         if lims:
             lims = []  # rebuild lims with ints/floats/strings
             for lim in opts.l:
                 try:
                     lims.append(int(lim))
                 except ValueError:
                     try:
                         lims.append(float(lim))
                     except ValueError:
                         lims.append(lim)
 
         # Trap output.
         stdout_trap = StringIO()
         stats_stream = stats.stream
         try:
             stats.stream = stdout_trap
             stats.print_stats(*lims)
         finally:
             stats.stream = stats_stream
 
         output = stdout_trap.getvalue()
         output = output.rstrip()
 
         if 'q' not in opts:
             page.page(output)
         print(sys_exit, end=' ')
 
         dump_file = opts.D[0]
         text_file = opts.T[0]
         if dump_file:
             prof.dump_stats(dump_file)
             print(
                 f"\n*** Profile stats marshalled to file {repr(dump_file)}.{sys_exit}"
             )
         if text_file:
             pfile = Path(text_file)
             pfile.touch(exist_ok=True)
             pfile.write_text(output, encoding="utf-8")
 
             print(
                 f"\n*** Profile printout saved to text file {repr(text_file)}.{sys_exit}"
             )
 
         if 'r' in opts:
             return stats
 
         return None
 
     @line_magic
     def pdb(self, parameter_s=''):
         """Control the automatic calling of the pdb interactive debugger.
 
         Call as '%pdb on', '%pdb 1', '%pdb off' or '%pdb 0'. If called without
         argument it works as a toggle.
 
         When an exception is triggered, IPython can optionally call the
         interactive pdb debugger after the traceback printout. %pdb toggles
         this feature on and off.
 
         The initial state of this feature is set in your configuration
         file (the option is ``InteractiveShell.pdb``).
 
         If you want to just activate the debugger AFTER an exception has fired,
         without having to type '%pdb on' and rerunning your code, you can use
         the %debug magic."""
 
         par = parameter_s.strip().lower()
 
         if par:
             try:
                 new_pdb = {'off':0,'0':0,'on':1,'1':1}[par]
             except KeyError:
                 print ('Incorrect argument. Use on/1, off/0, '
                        'or nothing for a toggle.')
                 return
         else:
             # toggle
             new_pdb = not self.shell.call_pdb
 
         # set on the shell
         self.shell.call_pdb = new_pdb
         print('Automatic pdb calling has been turned',on_off(new_pdb))
 
     @magic_arguments.magic_arguments()
     @magic_arguments.argument('--breakpoint', '-b', metavar='FILE:LINE',
         help="""
         Set break point at LINE in FILE.
         """
     )
     @magic_arguments.argument('statement', nargs='*',
         help="""
         Code to run in debugger.
         You can omit this in cell magic mode.
         """
     )
     @no_var_expand
     @line_cell_magic
     @needs_local_scope
     def debug(self, line="", cell=None, local_ns=None):
         """Activate the interactive debugger.
 
         This magic command support two ways of activating debugger.
         One is to activate debugger before executing code.  This way, you
         can set a break point, to step through the code from the point.
         You can use this mode by giving statements to execute and optionally
         a breakpoint.
 
         The other one is to activate debugger in post-mortem mode.  You can
         activate this mode simply running %debug without any argument.
         If an exception has just occurred, this lets you inspect its stack
         frames interactively.  Note that this will always work only on the last
         traceback that occurred, so you must call this quickly after an
         exception that you wish to inspect has fired, because if another one
         occurs, it clobbers the previous one.
 
         If you want IPython to automatically do this on every exception, see
         the %pdb magic for more details.
 
         .. versionchanged:: 7.3
             When running code, user variables are no longer expanded,
             the magic line is always left unmodified.
 
         """
         args = magic_arguments.parse_argstring(self.debug, line)
 
         if not (args.breakpoint or args.statement or cell):
             self._debug_post_mortem()
         elif not (args.breakpoint or cell):
             # If there is no breakpoints, the line is just code to execute
             self._debug_exec(line, None, local_ns)
         else:
             # Here we try to reconstruct the code from the output of
             # parse_argstring. This might not work if the code has spaces
             # For example this fails for `print("a b")`
             code = "\n".join(args.statement)
             if cell:
                 code += "\n" + cell
             self._debug_exec(code, args.breakpoint, local_ns)
 
     def _debug_post_mortem(self):
         self.shell.debugger(force=True)
 
     def _debug_exec(self, code, breakpoint, local_ns=None):
         if breakpoint:
             (filename, bp_line) = breakpoint.rsplit(':', 1)
             bp_line = int(bp_line)
         else:
             (filename, bp_line) = (None, None)
         self._run_with_debugger(
             code, self.shell.user_ns, filename, bp_line, local_ns=local_ns
         )
 
     @line_magic
     def tb(self, s):
         """Print the last traceback.
 
         Optionally, specify an exception reporting mode, tuning the
         verbosity of the traceback. By default the currently-active exception
         mode is used. See %xmode for changing exception reporting modes.
 
         Valid modes: Plain, Context, Verbose, and Minimal.
         """
         interactive_tb = self.shell.InteractiveTB
         if s:
             # Switch exception reporting mode for this one call.
             # Ensure it is switched back.
             def xmode_switch_err(name):
                 warn('Error changing %s exception modes.\n%s' %
                     (name,sys.exc_info()[1]))
 
             new_mode = s.strip().capitalize()
             original_mode = interactive_tb.mode
             try:
                 try:
                     interactive_tb.set_mode(mode=new_mode)
                 except Exception:
                     xmode_switch_err('user')
                 else:
                     self.shell.showtraceback()
             finally:
                 interactive_tb.set_mode(mode=original_mode)
         else:
             self.shell.showtraceback()
 
     @skip_doctest
     @line_magic
     def run(self, parameter_s='', runner=None,
                   file_finder=get_py_filename):
         """Run the named file inside IPython as a program.
 
         Usage::
 
           %run [-n -i -e -G]
                [( -t [-N<N>] | -d [-b<N>] | -p [profile options] )]
                ( -m mod | filename ) [args]
 
         The filename argument should be either a pure Python script (with
         extension ``.py``), or a file with custom IPython syntax (such as
         magics). If the latter, the file can be either a script with ``.ipy``
         extension, or a Jupyter notebook with ``.ipynb`` extension. When running
         a Jupyter notebook, the output from print statements and other
         displayed objects will appear in the terminal (even matplotlib figures
         will open, if a terminal-compliant backend is being used). Note that,
         at the system command line, the ``jupyter run`` command offers similar
         functionality for executing notebooks (albeit currently with some
         differences in supported options).
 
         Parameters after the filename are passed as command-line arguments to
         the program (put in sys.argv). Then, control returns to IPython's
         prompt.
 
         This is similar to running at a system prompt ``python file args``,
         but with the advantage of giving you IPython's tracebacks, and of
         loading all variables into your interactive namespace for further use
         (unless -p is used, see below).
 
         The file is executed in a namespace initially consisting only of
         ``__name__=='__main__'`` and sys.argv constructed as indicated. It thus
         sees its environment as if it were being run as a stand-alone program
         (except for sharing global objects such as previously imported
         modules). But after execution, the IPython interactive namespace gets
         updated with all variables defined in the program (except for __name__
         and sys.argv). This allows for very convenient loading of code for
         interactive work, while giving each program a 'clean sheet' to run in.
 
         Arguments are expanded using shell-like glob match.  Patterns
         '*', '?', '[seq]' and '[!seq]' can be used.  Additionally,
         tilde '~' will be expanded into user's home directory.  Unlike
         real shells, quotation does not suppress expansions.  Use
         *two* back slashes (e.g. ``\\\\*``) to suppress expansions.
         To completely disable these expansions, you can use -G flag.
 
         On Windows systems, the use of single quotes `'` when specifying
         a file is not supported. Use double quotes `"`.
 
         Options:
 
         -n
           __name__ is NOT set to '__main__', but to the running file's name
           without extension (as python does under import).  This allows running
           scripts and reloading the definitions in them without calling code
           protected by an ``if __name__ == "__main__"`` clause.
 
         -i
           run the file in IPython's namespace instead of an empty one. This
           is useful if you are experimenting with code written in a text editor
           which depends on variables defined interactively.
 
         -e
           ignore sys.exit() calls or SystemExit exceptions in the script
           being run.  This is particularly useful if IPython is being used to
           run unittests, which always exit with a sys.exit() call.  In such
           cases you are interested in the output of the test results, not in
           seeing a traceback of the unittest module.
 
         -t
           print timing information at the end of the run.  IPython will give
           you an estimated CPU time consumption for your script, which under
           Unix uses the resource module to avoid the wraparound problems of
           time.clock().  Under Unix, an estimate of time spent on system tasks
           is also given (for Windows platforms this is reported as 0.0).
 
         If -t is given, an additional ``-N<N>`` option can be given, where <N>
         must be an integer indicating how many times you want the script to
         run.  The final timing report will include total and per run results.
 
         For example (testing the script uniq_stable.py)::
 
             In [1]: run -t uniq_stable
 
             IPython CPU timings (estimated):
               User  :    0.19597 s.
               System:        0.0 s.
 
             In [2]: run -t -N5 uniq_stable
 
             IPython CPU timings (estimated):
             Total runs performed: 5
               Times :      Total       Per run
               User  :   0.910862 s,  0.1821724 s.
               System:        0.0 s,        0.0 s.
 
         -d
           run your program under the control of pdb, the Python debugger.
           This allows you to execute your program step by step, watch variables,
           etc.  Internally, what IPython does is similar to calling::
 
               pdb.run('execfile("YOURFILENAME")')
 
           with a breakpoint set on line 1 of your file.  You can change the line
           number for this automatic breakpoint to be <N> by using the -bN option
           (where N must be an integer). For example::
 
               %run -d -b40 myscript
 
           will set the first breakpoint at line 40 in myscript.py.  Note that
           the first breakpoint must be set on a line which actually does
           something (not a comment or docstring) for it to stop execution.
 
           Or you can specify a breakpoint in a different file::
 
               %run -d -b myotherfile.py:20 myscript
 
           When the pdb debugger starts, you will see a (Pdb) prompt.  You must
           first enter 'c' (without quotes) to start execution up to the first
           breakpoint.
 
           Entering 'help' gives information about the use of the debugger.  You
           can easily see pdb's full documentation with "import pdb;pdb.help()"
           at a prompt.
 
         -p
           run program under the control of the Python profiler module (which
           prints a detailed report of execution times, function calls, etc).
 
           You can pass other options after -p which affect the behavior of the
           profiler itself. See the docs for %prun for details.
 
           In this mode, the program's variables do NOT propagate back to the
           IPython interactive namespace (because they remain in the namespace
           where the profiler executes them).
 
           Internally this triggers a call to %prun, see its documentation for
           details on the options available specifically for profiling.
 
         There is one special usage for which the text above doesn't apply:
         if the filename ends with .ipy[nb], the file is run as ipython script,
         just as if the commands were written on IPython prompt.
 
         -m
           specify module name to load instead of script path. Similar to
           the -m option for the python interpreter. Use this option last if you
           want to combine with other %run options. Unlike the python interpreter
           only source modules are allowed no .pyc or .pyo files.
           For example::
 
               %run -m example
 
           will run the example module.
 
         -G
           disable shell-like glob expansion of arguments.
 
         """
 
         # Logic to handle issue #3664
         # Add '--' after '-m <module_name>' to ignore additional args passed to a module.
         if '-m' in parameter_s and '--' not in parameter_s:
             argv = shlex.split(parameter_s, posix=(os.name == 'posix'))
             for idx, arg in enumerate(argv):
                 if arg and arg.startswith('-') and arg != '-':
                     if arg == '-m':
                         argv.insert(idx + 2, '--')
                         break
                 else:
                     # Positional arg, break
                     break
             parameter_s = ' '.join(shlex.quote(arg) for arg in argv)
 
         # get arguments and set sys.argv for program to be run.
         opts, arg_lst = self.parse_options(parameter_s,
                                            'nidtN:b:pD:l:rs:T:em:G',
                                            mode='list', list_all=1)
         if "m" in opts:
             modulename = opts["m"][0]
             modpath = find_mod(modulename)
             if modpath is None:
                 msg = '%r is not a valid modulename on sys.path'%modulename
                 raise Exception(msg)
             arg_lst = [modpath] + arg_lst
         try:
             fpath = None # initialize to make sure fpath is in scope later
             fpath = arg_lst[0]
             filename = file_finder(fpath)
         except IndexError as e:
             msg = 'you must provide at least a filename.'
             raise Exception(msg) from e
         except IOError as e:
             try:
                 msg = str(e)
             except UnicodeError:
                 msg = e.message
             if os.name == 'nt' and re.match(r"^'.*'$",fpath):
                 warn('For Windows, use double quotes to wrap a filename: %run "mypath\\myfile.py"')
             raise Exception(msg) from e
         except TypeError:
             if fpath in sys.meta_path:
                 filename = ""
             else:
                 raise
 
         if filename.lower().endswith(('.ipy', '.ipynb')):
             with preserve_keys(self.shell.user_ns, '__file__'):
                 self.shell.user_ns['__file__'] = filename
                 self.shell.safe_execfile_ipy(filename, raise_exceptions=True)
             return
 
         # Control the response to exit() calls made by the script being run
         exit_ignore = 'e' in opts
 
         # Make sure that the running script gets a proper sys.argv as if it
         # were run from a system shell.
         save_argv = sys.argv # save it for later restoring
 
         if 'G' in opts:
             args = arg_lst[1:]
         else:
             # tilde and glob expansion
             args = shellglob(map(os.path.expanduser,  arg_lst[1:]))
 
         sys.argv = [filename] + args  # put in the proper filename
 
         if 'n' in opts:
             name = Path(filename).stem
         else:
             name = '__main__'
 
         if 'i' in opts:
             # Run in user's interactive namespace
             prog_ns = self.shell.user_ns
             __name__save = self.shell.user_ns['__name__']
             prog_ns['__name__'] = name
             main_mod = self.shell.user_module
 
             # Since '%run foo' emulates 'python foo.py' at the cmd line, we must
             # set the __file__ global in the script's namespace
             # TK: Is this necessary in interactive mode?
             prog_ns['__file__'] = filename
         else:
             # Run in a fresh, empty namespace
 
             # The shell MUST hold a reference to prog_ns so after %run
             # exits, the python deletion mechanism doesn't zero it out
             # (leaving dangling references). See interactiveshell for details
             main_mod = self.shell.new_main_mod(filename, name)
             prog_ns = main_mod.__dict__
 
         # pickle fix.  See interactiveshell for an explanation.  But we need to
         # make sure that, if we overwrite __main__, we replace it at the end
         main_mod_name = prog_ns['__name__']
 
         if main_mod_name == '__main__':
             restore_main = sys.modules['__main__']
         else:
             restore_main = False
 
         # This needs to be undone at the end to prevent holding references to
         # every single object ever created.
         sys.modules[main_mod_name] = main_mod
 
         if 'p' in opts or 'd' in opts:
             if 'm' in opts:
                 code = 'run_module(modulename, prog_ns)'
                 code_ns = {
                     'run_module': self.shell.safe_run_module,
                     'prog_ns': prog_ns,
                     'modulename': modulename,
                 }
             else:
                 if 'd' in opts:
                     # allow exceptions to raise in debug mode
                     code = 'execfile(filename, prog_ns, raise_exceptions=True)'
                 else:
                     code = 'execfile(filename, prog_ns)'
                 code_ns = {
                     'execfile': self.shell.safe_execfile,
                     'prog_ns': prog_ns,
                     'filename': get_py_filename(filename),
                 }
 
         try:
             stats = None
             if 'p' in opts:
                 stats = self._run_with_profiler(code, opts, code_ns)
             else:
                 if 'd' in opts:
                     bp_file, bp_line = parse_breakpoint(
                         opts.get('b', ['1'])[0], filename)
                     self._run_with_debugger(
                         code, code_ns, filename, bp_line, bp_file)
                 else:
                     if 'm' in opts:
                         def run():
                             self.shell.safe_run_module(modulename, prog_ns)
                     else:
                         if runner is None:
                             runner = self.default_runner
                         if runner is None:
                             runner = self.shell.safe_execfile
 
                         def run():
                             runner(filename, prog_ns, prog_ns,
                                     exit_ignore=exit_ignore)
 
                     if 't' in opts:
                         # timed execution
                         try:
                             nruns = int(opts['N'][0])
                             if nruns < 1:
                                 error('Number of runs must be >=1')
                                 return
                         except (KeyError):
                             nruns = 1
                         self._run_with_timing(run, nruns)
                     else:
                         # regular execution
                         run()
 
             if 'i' in opts:
                 self.shell.user_ns['__name__'] = __name__save
             else:
                 # update IPython interactive namespace
 
                 # Some forms of read errors on the file may mean the
                 # __name__ key was never set; using pop we don't have to
                 # worry about a possible KeyError.
                 prog_ns.pop('__name__', None)
 
                 with preserve_keys(self.shell.user_ns, '__file__'):
                     self.shell.user_ns.update(prog_ns)
         finally:
             # It's a bit of a mystery why, but __builtins__ can change from
             # being a module to becoming a dict missing some key data after
             # %run.  As best I can see, this is NOT something IPython is doing
             # at all, and similar problems have been reported before:
             # http://coding.derkeiler.com/Archive/Python/comp.lang.python/2004-10/0188.html
             # Since this seems to be done by the interpreter itself, the best
             # we can do is to at least restore __builtins__ for the user on
             # exit.
             self.shell.user_ns['__builtins__'] = builtin_mod
 
             # Ensure key global structures are restored
             sys.argv = save_argv
             if restore_main:
                 sys.modules['__main__'] = restore_main
                 if '__mp_main__' in sys.modules:
                     sys.modules['__mp_main__'] = restore_main
             else:
                 # Remove from sys.modules the reference to main_mod we'd
                 # added.  Otherwise it will trap references to objects
                 # contained therein.
                 del sys.modules[main_mod_name]
 
         return stats
 
     def _run_with_debugger(
         self, code, code_ns, filename=None, bp_line=None, bp_file=None, local_ns=None
     ):
         """
         Run `code` in debugger with a break point.
 
         Parameters
         ----------
         code : str
             Code to execute.
         code_ns : dict
             A namespace in which `code` is executed.
         filename : str
             `code` is ran as if it is in `filename`.
         bp_line : int, optional
             Line number of the break point.
         bp_file : str, optional
             Path to the file in which break point is specified.
             `filename` is used if not given.
         local_ns : dict, optional
             A local namespace in which `code` is executed.
 
         Raises
         ------
         UsageError
             If the break point given by `bp_line` is not valid.
 
         """
         deb = self.shell.InteractiveTB.pdb
         if not deb:
             self.shell.InteractiveTB.pdb = self.shell.InteractiveTB.debugger_cls()
             deb = self.shell.InteractiveTB.pdb
 
         # deb.checkline() fails if deb.curframe exists but is None; it can
         # handle it not existing. https://github.com/ipython/ipython/issues/10028
         if hasattr(deb, 'curframe'):
             del deb.curframe
 
         # reset Breakpoint state, which is moronically kept
         # in a class
         bdb.Breakpoint.next = 1
         bdb.Breakpoint.bplist = {}
         bdb.Breakpoint.bpbynumber = [None]
         deb.clear_all_breaks()
         if bp_line is not None:
             # Set an initial breakpoint to stop execution
             maxtries = 10
             bp_file = bp_file or filename
             checkline = deb.checkline(bp_file, bp_line)
             if not checkline:
                 for bp in range(bp_line + 1, bp_line + maxtries + 1):
                     if deb.checkline(bp_file, bp):
                         break
                 else:
                     msg = ("\nI failed to find a valid line to set "
                            "a breakpoint\n"
                            "after trying up to line: %s.\n"
                            "Please set a valid breakpoint manually "
                            "with the -b option." % bp)
                     raise UsageError(msg)
             # if we find a good linenumber, set the breakpoint
             deb.do_break('%s:%s' % (bp_file, bp_line))
 
         if filename:
             # Mimic Pdb._runscript(...)
             deb._wait_for_mainpyfile = True
             deb.mainpyfile = deb.canonic(filename)
 
         # Start file run
         print("NOTE: Enter 'c' at the %s prompt to continue execution." % deb.prompt)
         try:
             if filename:
                 # save filename so it can be used by methods on the deb object
                 deb._exec_filename = filename
             while True:
                 try:
                     trace = sys.gettrace()
                     deb.run(code, code_ns, local_ns)
                 except Restart:
                     print("Restarting")
                     if filename:
                         deb._wait_for_mainpyfile = True
                         deb.mainpyfile = deb.canonic(filename)
                     continue
                 else:
                     break
                 finally:
                     sys.settrace(trace)
             
 
         except:
             etype, value, tb = sys.exc_info()
             # Skip three frames in the traceback: the %run one,
             # one inside bdb.py, and the command-line typed by the
             # user (run by exec in pdb itself).
             self.shell.InteractiveTB(etype, value, tb, tb_offset=3)
 
     @staticmethod
     def _run_with_timing(run, nruns):
         """
         Run function `run` and print timing information.
 
         Parameters
         ----------
         run : callable
             Any callable object which takes no argument.
         nruns : int
             Number of times to execute `run`.
 
         """
         twall0 = time.perf_counter()
         if nruns == 1:
             t0 = clock2()
             run()
             t1 = clock2()
             t_usr = t1[0] - t0[0]
             t_sys = t1[1] - t0[1]
             print("\nIPython CPU timings (estimated):")
             print("  User   : %10.2f s." % t_usr)
             print("  System : %10.2f s." % t_sys)
         else:
             runs = range(nruns)
             t0 = clock2()
             for nr in runs:
                 run()
             t1 = clock2()
             t_usr = t1[0] - t0[0]
             t_sys = t1[1] - t0[1]
             print("\nIPython CPU timings (estimated):")
             print("Total runs performed:", nruns)
             print("  Times  : %10s   %10s" % ('Total', 'Per run'))
             print("  User   : %10.2f s, %10.2f s." % (t_usr, t_usr / nruns))
             print("  System : %10.2f s, %10.2f s." % (t_sys, t_sys / nruns))
         twall1 = time.perf_counter()
         print("Wall time: %10.2f s." % (twall1 - twall0))
 
     @skip_doctest
     @no_var_expand
     @line_cell_magic
     @needs_local_scope
     def timeit(self, line='', cell=None, local_ns=None):
         """Time execution of a Python statement or expression
 
         **Usage, in line mode:**
 
           %timeit [-n<N> -r<R> [-t|-c] -q -p<P> -o] statement
 
         **or in cell mode:**
 
           %%timeit [-n<N> -r<R> [-t|-c] -q -p<P> -o] setup_code
 
           code
 
           code...
 
         Time execution of a Python statement or expression using the timeit
         module.  This function can be used both as a line and cell magic:
 
         - In line mode you can time a single-line statement (though multiple
           ones can be chained with using semicolons).
 
         - In cell mode, the statement in the first line is used as setup code
           (executed but not timed) and the body of the cell is timed.  The cell
           body has access to any variables created in the setup code.
 
         Options:
 
         -n<N>: execute the given statement <N> times in a loop. If <N> is not
         provided, <N> is determined so as to get sufficient accuracy.
 
         -r<R>: number of repeats <R>, each consisting of <N> loops, and take the
         average result.
         Default: 7
 
         -t: use time.time to measure the time, which is the default on Unix.
         This function measures wall time.
 
         -c: use time.clock to measure the time, which is the default on
         Windows and measures wall time. On Unix, resource.getrusage is used
         instead and returns the CPU user time.
 
         -p<P>: use a precision of <P> digits to display the timing result.
         Default: 3
 
         -q: Quiet, do not print result.
 
         -o: return a TimeitResult that can be stored in a variable to inspect
         the result in more details.
 
         .. versionchanged:: 7.3
             User variables are no longer expanded,
             the magic line is always left unmodified.
 
         Examples
         --------
         ::
 
           In [1]: %timeit pass
           8.26 ns ± 0.12 ns per loop (mean ± std. dev. of 7 runs, 100000000 loops each)
 
           In [2]: u = None
 
           In [3]: %timeit u is None
           29.9 ns ± 0.643 ns per loop (mean ± std. dev. of 7 runs, 10000000 loops each)
 
           In [4]: %timeit -r 4 u == None
 
           In [5]: import time
 
           In [6]: %timeit -n1 time.sleep(2)
 
         The times reported by %timeit will be slightly higher than those
         reported by the timeit.py script when variables are accessed. This is
         due to the fact that %timeit executes the statement in the namespace
         of the shell, compared with timeit.py, which uses a single setup
         statement to import function or create variables. Generally, the bias
         does not matter as long as results from timeit.py are not mixed with
         those from %timeit."""
 
         opts, stmt = self.parse_options(
             line, "n:r:tcp:qo", posix=False, strict=False, preserve_non_opts=True
         )
         if stmt == "" and cell is None:
             return
         
         timefunc = timeit.default_timer
         number = int(getattr(opts, "n", 0))
         default_repeat = 7 if timeit.default_repeat < 7 else timeit.default_repeat
         repeat = int(getattr(opts, "r", default_repeat))
         precision = int(getattr(opts, "p", 3))
         quiet = 'q' in opts
         return_result = 'o' in opts
         if hasattr(opts, "t"):
             timefunc = time.time
         if hasattr(opts, "c"):
             timefunc = clock
 
         timer = Timer(timer=timefunc)
         # this code has tight coupling to the inner workings of timeit.Timer,
         # but is there a better way to achieve that the code stmt has access
         # to the shell namespace?
         transform  = self.shell.transform_cell
 
         if cell is None:
             # called as line magic
             ast_setup = self.shell.compile.ast_parse("pass")
             ast_stmt = self.shell.compile.ast_parse(transform(stmt))
         else:
             ast_setup = self.shell.compile.ast_parse(transform(stmt))
             ast_stmt = self.shell.compile.ast_parse(transform(cell))
 
         ast_setup = self.shell.transform_ast(ast_setup)
         ast_stmt = self.shell.transform_ast(ast_stmt)
 
         # Check that these compile to valid Python code *outside* the timer func
         # Invalid code may become valid when put inside the function & loop,
         # which messes up error messages.
         # https://github.com/ipython/ipython/issues/10636
         self.shell.compile(ast_setup, "<magic-timeit-setup>", "exec")
         self.shell.compile(ast_stmt, "<magic-timeit-stmt>", "exec")
 
         # This codestring is taken from timeit.template - we fill it in as an
         # AST, so that we can apply our AST transformations to the user code
         # without affecting the timing code.
         timeit_ast_template = ast.parse('def inner(_it, _timer):\n'
                                         '    setup\n'
                                         '    _t0 = _timer()\n'
                                         '    for _i in _it:\n'
                                         '        stmt\n'
                                         '    _t1 = _timer()\n'
                                         '    return _t1 - _t0\n')
 
         timeit_ast = TimeitTemplateFiller(ast_setup, ast_stmt).visit(timeit_ast_template)
         timeit_ast = ast.fix_missing_locations(timeit_ast)
 
         # Track compilation time so it can be reported if too long
         # Minimum time above which compilation time will be reported
         tc_min = 0.1
 
         t0 = clock()
         code = self.shell.compile(timeit_ast, "<magic-timeit>", "exec")
         tc = clock()-t0
 
         ns = {}
         glob = self.shell.user_ns
         # handles global vars with same name as local vars. We store them in conflict_globs.
         conflict_globs = {}
         if local_ns and cell is None:
             for var_name, var_val in glob.items():
                 if var_name in local_ns:
                     conflict_globs[var_name] = var_val
             glob.update(local_ns)
             
         exec(code, glob, ns)
         timer.inner = ns["inner"]
 
         # This is used to check if there is a huge difference between the
         # best and worst timings.
         # Issue: https://github.com/ipython/ipython/issues/6471
         if number == 0:
             # determine number so that 0.2 <= total time < 2.0
             for index in range(0, 10):
                 number = 10 ** index
                 time_number = timer.timeit(number)
                 if time_number >= 0.2:
                     break
 
         all_runs = timer.repeat(repeat, number)
         best = min(all_runs) / number
         worst = max(all_runs) / number
         timeit_result = TimeitResult(number, repeat, best, worst, all_runs, tc, precision)
 
         # Restore global vars from conflict_globs
         if conflict_globs:
            glob.update(conflict_globs)
                 
         if not quiet :
             # Check best timing is greater than zero to avoid a
             # ZeroDivisionError.
             # In cases where the slowest timing is lesser than a microsecond
             # we assume that it does not really matter if the fastest
             # timing is 4 times faster than the slowest timing or not.
             if worst > 4 * best and best > 0 and worst > 1e-6:
                 print("The slowest run took %0.2f times longer than the "
                       "fastest. This could mean that an intermediate result "
                       "is being cached." % (worst / best))
            
             print( timeit_result )
 
             if tc > tc_min:
                 print("Compiler time: %.2f s" % tc)
         if return_result:
             return timeit_result
 
     @skip_doctest
     @no_var_expand
     @needs_local_scope
     @line_cell_magic
     @output_can_be_silenced
     def time(self,line='', cell=None, local_ns=None):
         """Time execution of a Python statement or expression.
 
         The CPU and wall clock times are printed, and the value of the
         expression (if any) is returned.  Note that under Win32, system time
         is always reported as 0, since it can not be measured.
 
         This function can be used both as a line and cell magic:
 
         - In line mode you can time a single-line statement (though multiple
           ones can be chained with using semicolons).
 
         - In cell mode, you can time the cell body (a directly
           following statement raises an error).
 
         This function provides very basic timing functionality.  Use the timeit
         magic for more control over the measurement.
 
         .. versionchanged:: 7.3
             User variables are no longer expanded,
             the magic line is always left unmodified.
 
         Examples
         --------
         ::
 
           In [1]: %time 2**128
           CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
           Wall time: 0.00
           Out[1]: 340282366920938463463374607431768211456L
 
           In [2]: n = 1000000
 
           In [3]: %time sum(range(n))
           CPU times: user 1.20 s, sys: 0.05 s, total: 1.25 s
           Wall time: 1.37
           Out[3]: 499999500000L
 
           In [4]: %time print('hello world')
           hello world
           CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
           Wall time: 0.00
 
         .. note::
             The time needed by Python to compile the given expression will be
             reported if it is more than 0.1s.
 
             In the example below, the actual exponentiation is done by Python
             at compilation time, so while the expression can take a noticeable
             amount of time to compute, that time is purely due to the
             compilation::
 
                 In [5]: %time 3**9999;
                 CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
                 Wall time: 0.00 s
 
                 In [6]: %time 3**999999;
                 CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
                 Wall time: 0.00 s
                 Compiler : 0.78 s
         """
         # fail immediately if the given expression can't be compiled
         
         if line and cell:
             raise UsageError("Can't use statement directly after '%%time'!")
         
         if cell:
             expr = self.shell.transform_cell(cell)
         else:
             expr = self.shell.transform_cell(line)
 
         # Minimum time above which parse time will be reported
         tp_min = 0.1
 
         t0 = clock()
         expr_ast = self.shell.compile.ast_parse(expr)
         tp = clock()-t0
 
         # Apply AST transformations
         expr_ast = self.shell.transform_ast(expr_ast)
 
         # Minimum time above which compilation time will be reported
         tc_min = 0.1
 
         expr_val=None
         if len(expr_ast.body)==1 and isinstance(expr_ast.body[0], ast.Expr):
             mode = 'eval'
             source = '<timed eval>'
             expr_ast = ast.Expression(expr_ast.body[0].value)
         else:
             mode = 'exec'
             source = '<timed exec>'
             # multi-line %%time case
             if len(expr_ast.body) > 1 and isinstance(expr_ast.body[-1], ast.Expr):
                 expr_val= expr_ast.body[-1]
                 expr_ast = expr_ast.body[:-1]
                 expr_ast = Module(expr_ast, [])
                 expr_val = ast.Expression(expr_val.value)
 
         t0 = clock()
         code = self.shell.compile(expr_ast, source, mode)
         tc = clock()-t0
 
         # skew measurement as little as possible
         glob = self.shell.user_ns
         wtime = time.time
         # time execution
         wall_st = wtime()
         if mode=='eval':
             st = clock2()
             try:
                 out = eval(code, glob, local_ns)
             except:
                 self.shell.showtraceback()
                 return
             end = clock2()
         else:
             st = clock2()
             try:
                 exec(code, glob, local_ns)
                 out=None
                 # multi-line %%time case
                 if expr_val is not None:
                     code_2 = self.shell.compile(expr_val, source, 'eval')
                     out = eval(code_2, glob, local_ns)
             except:
                 self.shell.showtraceback()
                 return
             end = clock2()
 
         wall_end = wtime()
         # Compute actual times and report
         wall_time = wall_end - wall_st
         cpu_user = end[0] - st[0]
         cpu_sys = end[1] - st[1]
         cpu_tot = cpu_user + cpu_sys
         # On windows cpu_sys is always zero, so only total is displayed
         if sys.platform != "win32":
             print(
                 f"CPU times: user {_format_time(cpu_user)}, sys: {_format_time(cpu_sys)}, total: {_format_time(cpu_tot)}"
             )
         else:
             print(f"CPU times: total: {_format_time(cpu_tot)}")
         print(f"Wall time: {_format_time(wall_time)}")
         if tc > tc_min:
             print(f"Compiler : {_format_time(tc)}")
         if tp > tp_min:
             print(f"Parser   : {_format_time(tp)}")
         return out
 
     @skip_doctest
     @line_magic
     def macro(self, parameter_s=''):
         """Define a macro for future re-execution. It accepts ranges of history,
         filenames or string objects.
 
         Usage:\\
           %macro [options] name n1-n2 n3-n4 ... n5 .. n6 ...
 
         Options:
 
           -r: use 'raw' input.  By default, the 'processed' history is used,
           so that magics are loaded in their transformed version to valid
           Python.  If this option is given, the raw input as typed at the
           command line is used instead.
           
           -q: quiet macro definition.  By default, a tag line is printed 
           to indicate the macro has been created, and then the contents of 
           the macro are printed.  If this option is given, then no printout
           is produced once the macro is created.
 
         This will define a global variable called `name` which is a string
         made of joining the slices and lines you specify (n1,n2,... numbers
         above) from your input history into a single string. This variable
         acts like an automatic function which re-executes those lines as if
         you had typed them. You just type 'name' at the prompt and the code
         executes.
 
         The syntax for indicating input ranges is described in %history.
 
         Note: as a 'hidden' feature, you can also use traditional python slice
         notation, where N:M means numbers N through M-1.
 
         For example, if your history contains (print using %hist -n )::
 
           44: x=1
           45: y=3
           46: z=x+y
           47: print(x)
           48: a=5
           49: print('x',x,'y',y)
 
         you can create a macro with lines 44 through 47 (included) and line 49
         called my_macro with::
 
           In [55]: %macro my_macro 44-47 49
 
         Now, typing `my_macro` (without quotes) will re-execute all this code
         in one pass.
 
         You don't need to give the line-numbers in order, and any given line
         number can appear multiple times. You can assemble macros with any
         lines from your input history in any order.
 
         The macro is a simple object which holds its value in an attribute,
         but IPython's display system checks for macros and executes them as
         code instead of printing them when you type their name.
 
         You can view a macro's contents by explicitly printing it with::
 
           print(macro_name)
 
         """
         opts,args = self.parse_options(parameter_s,'rq',mode='list')
         if not args:   # List existing macros
             return sorted(k for k,v in self.shell.user_ns.items() if isinstance(v, Macro))
         if len(args) == 1:
             raise UsageError(
                 "%macro insufficient args; usage '%macro name n1-n2 n3-4...")
         name, codefrom = args[0], " ".join(args[1:])
 
         # print('rng',ranges)  # dbg
         try:
             lines = self.shell.find_user_code(codefrom, 'r' in opts)
         except (ValueError, TypeError) as e:
             print(e.args[0])
             return
         macro = Macro(lines)
         self.shell.define_macro(name, macro)
-        if "q" not in opts:
-            print(
-                "Macro `%s` created. To execute, type its name (without quotes)." % name
-            )
-            print("=== Macro contents: ===")
-            print(macro, end=" ")
+        if not ( 'q' in opts) : 
+            print('Macro `%s` created. To execute, type its name (without quotes).' % name)
+            print('=== Macro contents: ===')
+            print(macro, end=' ')
 
     @magic_arguments.magic_arguments()
     @magic_arguments.argument('output', type=str, default='', nargs='?',
         help="""The name of the variable in which to store output.
         This is a utils.io.CapturedIO object with stdout/err attributes
         for the text of the captured output.
 
         CapturedOutput also has a show() method for displaying the output,
         and __call__ as well, so you can use that to quickly display the
         output.
 
         If unspecified, captured output is discarded.
         """
     )
     @magic_arguments.argument('--no-stderr', action="store_true",
         help="""Don't capture stderr."""
     )
     @magic_arguments.argument('--no-stdout', action="store_true",
         help="""Don't capture stdout."""
     )
     @magic_arguments.argument('--no-display', action="store_true",
         help="""Don't capture IPython's rich display."""
     )
     @cell_magic
     def capture(self, line, cell):
         """run the cell, capturing stdout, stderr, and IPython's rich display() calls."""
         args = magic_arguments.parse_argstring(self.capture, line)
         out = not args.no_stdout
         err = not args.no_stderr
         disp = not args.no_display
         with capture_output(out, err, disp) as io:
             self.shell.run_cell(cell)
         if DisplayHook.semicolon_at_end_of_expression(cell):
             if args.output in self.shell.user_ns:
                 del self.shell.user_ns[args.output]
         elif args.output:
             self.shell.user_ns[args.output] = io
 
     @skip_doctest
     @magic_arguments.magic_arguments()
     @magic_arguments.argument("name", type=str, default="default", nargs="?")
     @magic_arguments.argument(
         "--remove", action="store_true", help="remove the current transformer"
     )
     @magic_arguments.argument(
         "--list", action="store_true", help="list existing transformers name"
     )
     @magic_arguments.argument(
         "--list-all",
         action="store_true",
         help="list existing transformers name and code template",
     )
     @line_cell_magic
     def code_wrap(self, line, cell=None):
         """
         Simple magic to quickly define a code transformer for all IPython's future input.
 
         ``__code__`` and ``__ret__`` are special variable that represent the code to run
         and the value of the last expression of ``__code__`` respectively.
 
         Examples
         --------
 
         .. ipython::
 
             In [1]: %%code_wrap before_after
                ...: print('before')
                ...: __code__
                ...: print('after')
                ...: __ret__
 
 
             In [2]: 1
             before
             after
             Out[2]: 1
 
             In [3]: %code_wrap --list
             before_after
 
             In [4]: %code_wrap --list-all
             before_after :
                 print('before')
                 __code__
                 print('after')
                 __ret__
 
             In [5]: %code_wrap --remove before_after
 
         """
         args = magic_arguments.parse_argstring(self.code_wrap, line)
 
         if args.list:
             for name in self._transformers.keys():
                 print(name)
             return
         if args.list_all:
             for name, _t in self._transformers.items():
                 print(name, ":")
                 print(indent(ast.unparse(_t.template), "    "))
             print()
             return
 
         to_remove = self._transformers.pop(args.name, None)
         if to_remove in self.shell.ast_transformers:
             self.shell.ast_transformers.remove(to_remove)
         if cell is None or args.remove:
             return
 
         _trs = ReplaceCodeTransformer(ast.parse(cell))
 
         self._transformers[args.name] = _trs
         self.shell.ast_transformers.append(_trs)
 
 
 def parse_breakpoint(text, current_file):
     '''Returns (file, line) for file:line and (current_file, line) for line'''
     colon = text.find(':')
     if colon == -1:
         return current_file, int(text)
     else:
         return text[:colon], int(text[colon+1:])
     
 def _format_time(timespan, precision=3):
     """Formats the timespan in a human readable form"""
 
     if timespan >= 60.0:
         # we have more than a minute, format that in a human readable form
         # Idea from http://snipplr.com/view/5713/
         parts = [("d", 60*60*24),("h", 60*60),("min", 60), ("s", 1)]
         time = []
         leftover = timespan
         for suffix, length in parts:
             value = int(leftover / length)
             if value > 0:
                 leftover = leftover % length
                 time.append(u'%s%s' % (str(value), suffix))
             if leftover < 1:
                 break
         return " ".join(time)
 
 
     # Unfortunately characters outside of range(128) can cause problems in
     # certain terminals.
     # See bug: https://bugs.launchpad.net/ipython/+bug/348466
     # Try to prevent crashes by being more secure than it needs to
     # E.g. eclipse is able to print a µ, but has no sys.stdout.encoding set.
     units = ["s", "ms", "us", "ns"]  # the safe value
     if hasattr(sys.stdout, "encoding") and sys.stdout.encoding:
         try:
             "μ".encode(sys.stdout.encoding)
             units = ["s", "ms", "μs", "ns"]
         except:
             pass
     scaling = [1, 1e3, 1e6, 1e9]
         
     if timespan > 0.0:
         order = min(-int(math.floor(math.log10(timespan)) // 3), 3)
     else:
         order = 3
     return "%.*g %s" % (precision, timespan * scaling[order], units[order])

IPython/core/magics/osm.py

@@ -1,855 +1,855
 """Implementation of magic functions for interaction with the OS.
 
 Note: this module is named 'osm' instead of 'os' to avoid a collision with the
 builtin.
 """
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 import io
 import os
 import pathlib
 import re
 import sys
 from pprint import pformat
 
 from IPython.core import magic_arguments
 from IPython.core import oinspect
 from IPython.core import page
 from IPython.core.alias import AliasError, Alias
 from IPython.core.error import UsageError
 from IPython.core.magic import  (
     Magics, compress_dhist, magics_class, line_magic, cell_magic, line_cell_magic
 )
 from IPython.testing.skipdoctest import skip_doctest
 from IPython.utils.openpy import source_to_unicode
 from IPython.utils.process import abbrev_cwd
 from IPython.utils.terminal import set_term_title
 from traitlets import Bool
 from warnings import warn
 
 
 @magics_class
 class OSMagics(Magics):
     """Magics to interact with the underlying OS (shell-type functionality).
     """
 
     cd_force_quiet = Bool(False,
         help="Force %cd magic to be quiet even if -q is not passed."
     ).tag(config=True)
 
     def __init__(self, shell=None, **kwargs):
 
         # Now define isexec in a cross platform manner.
         self.is_posix = False
         self.execre = None
         if os.name == 'posix':
             self.is_posix = True
         else:
             try:
                 winext = os.environ['pathext'].replace(';','|').replace('.','')
             except KeyError:
                 winext = 'exe|com|bat|py'
             try:
                 self.execre = re.compile(r'(.*)\.(%s)$' % winext,re.IGNORECASE)
             except re.error:
                 warn("Seems like your pathext environmental "
                      "variable is malformed. Please check it to "
                      "enable a proper handle of file extensions "
                      "managed for your system")
                 winext = 'exe|com|bat|py'
                 self.execre = re.compile(r'(.*)\.(%s)$' % winext,re.IGNORECASE)
 
         # call up the chain
         super().__init__(shell=shell, **kwargs)
 
 
     def _isexec_POSIX(self, file):
         """
         Test for executable on a POSIX system
         """
         if os.access(file.path, os.X_OK):
             # will fail on maxOS if access is not X_OK
             return file.is_file()
         return False
 
 
     
     def _isexec_WIN(self, file):
         """
         Test for executable file on non POSIX system
         """
         return file.is_file() and self.execre.match(file.name) is not None
 
     def isexec(self, file):
         """
         Test for executable file on non POSIX system
         """
         if self.is_posix:
             return self._isexec_POSIX(file)
         else:
             return self._isexec_WIN(file)
 
 
     @skip_doctest
     @line_magic
     def alias(self, parameter_s=''):
         """Define an alias for a system command.
 
         '%alias alias_name cmd' defines 'alias_name' as an alias for 'cmd'
 
         Then, typing 'alias_name params' will execute the system command 'cmd
         params' (from your underlying operating system).
 
         Aliases have lower precedence than magic functions and Python normal
         variables, so if 'foo' is both a Python variable and an alias, the
         alias can not be executed until 'del foo' removes the Python variable.
 
         You can use the %l specifier in an alias definition to represent the
         whole line when the alias is called.  For example::
 
           In [2]: alias bracket echo "Input in brackets: <%l>"
           In [3]: bracket hello world
           Input in brackets: <hello world>
 
         You can also define aliases with parameters using %s specifiers (one
         per parameter)::
 
           In [1]: alias parts echo first %s second %s
           In [2]: %parts A B
           first A second B
           In [3]: %parts A
           Incorrect number of arguments: 2 expected.
           parts is an alias to: 'echo first %s second %s'
 
         Note that %l and %s are mutually exclusive.  You can only use one or
         the other in your aliases.
 
         Aliases expand Python variables just like system calls using ! or !!
         do: all expressions prefixed with '$' get expanded.  For details of
         the semantic rules, see PEP-215:
         https://peps.python.org/pep-0215/.  This is the library used by
         IPython for variable expansion.  If you want to access a true shell
         variable, an extra $ is necessary to prevent its expansion by
         IPython::
 
           In [6]: alias show echo
           In [7]: PATH='A Python string'
           In [8]: show $PATH
           A Python string
           In [9]: show $$PATH
           /usr/local/lf9560/bin:/usr/local/intel/compiler70/ia32/bin:...
 
         You can use the alias facility to access all of $PATH.  See the %rehashx
         function, which automatically creates aliases for the contents of your
         $PATH.
 
         If called with no parameters, %alias prints the current alias table
         for your system.  For posix systems, the default aliases are 'cat',
         'cp', 'mv', 'rm', 'rmdir', and 'mkdir', and other platform-specific
         aliases are added.  For windows-based systems, the default aliases are
         'copy', 'ddir', 'echo', 'ls', 'ldir', 'mkdir', 'ren', and 'rmdir'.
 
         You can see the definition of alias by adding a question mark in the
         end::
 
           In [1]: cat?
           Repr: <alias cat for 'cat'>"""
 
         par = parameter_s.strip()
         if not par:
             aliases = sorted(self.shell.alias_manager.aliases)
             # stored = self.shell.db.get('stored_aliases', {} )
             # for k, v in stored:
             #     atab.append(k, v[0])
 
             print("Total number of aliases:", len(aliases))
             sys.stdout.flush()
             return aliases
 
         # Now try to define a new one
         try:
             alias,cmd = par.split(None, 1)
         except TypeError:
             print(oinspect.getdoc(self.alias))
             return
         
         try:
             self.shell.alias_manager.define_alias(alias, cmd)
         except AliasError as e:
             print(e)
     # end magic_alias
 
     @line_magic
     def unalias(self, parameter_s=''):
         """Remove an alias"""
 
         aname = parameter_s.strip()
         try:
             self.shell.alias_manager.undefine_alias(aname)
         except ValueError as e:
             print(e)
             return
         
         stored = self.shell.db.get('stored_aliases', {} )
         if aname in stored:
             print("Removing %stored alias",aname)
             del stored[aname]
             self.shell.db['stored_aliases'] = stored
 
     @line_magic
     def rehashx(self, parameter_s=''):
         """Update the alias table with all executable files in $PATH.
 
         rehashx explicitly checks that every entry in $PATH is a file
         with execute access (os.X_OK).
 
         Under Windows, it checks executability as a match against a
         '|'-separated string of extensions, stored in the IPython config
         variable win_exec_ext.  This defaults to 'exe|com|bat'.
 
         This function also resets the root module cache of module completer,
         used on slow filesystems.
         """
         from IPython.core.alias import InvalidAliasError
 
         # for the benefit of module completer in ipy_completers.py
         del self.shell.db['rootmodules_cache']
 
         path = [os.path.abspath(os.path.expanduser(p)) for p in
             os.environ.get('PATH','').split(os.pathsep)]
 
         syscmdlist = []
         savedir = os.getcwd()
 
         # Now walk the paths looking for executables to alias.
         try:
             # write the whole loop for posix/Windows so we don't have an if in
             # the innermost part
             if self.is_posix:
                 for pdir in path:
                     try:
                         os.chdir(pdir)
                     except OSError:
                         continue
 
                     # for python 3.6+ rewrite to: with os.scandir(pdir) as dirlist:
                     dirlist = os.scandir(path=pdir)
                     for ff in dirlist:
                         if self.isexec(ff):
                             fname = ff.name
                             try:
                                 # Removes dots from the name since ipython
                                 # will assume names with dots to be python.
                                 if not self.shell.alias_manager.is_alias(fname):
                                     self.shell.alias_manager.define_alias(
                                         fname.replace('.',''), fname)
                             except InvalidAliasError:
                                 pass
                             else:
                                 syscmdlist.append(fname)
             else:
                 no_alias = Alias.blacklist
                 for pdir in path:
                     try:
                         os.chdir(pdir)
                     except OSError:
                         continue
 
                     # for python 3.6+ rewrite to: with os.scandir(pdir) as dirlist:
                     dirlist = os.scandir(pdir)
                     for ff in dirlist:
                         fname = ff.name
                         base, ext = os.path.splitext(fname)
                         if self.isexec(ff) and base.lower() not in no_alias:
                             if ext.lower() == '.exe':
                                 fname = base
                                 try:
                                     # Removes dots from the name since ipython
                                     # will assume names with dots to be python.
                                     self.shell.alias_manager.define_alias(
                                         base.lower().replace('.',''), fname)
                                 except InvalidAliasError:
                                     pass
                                 syscmdlist.append(fname)
 
             self.shell.db['syscmdlist'] = syscmdlist
         finally:
             os.chdir(savedir)
 
     @skip_doctest
     @line_magic
     def pwd(self, parameter_s=''):
         """Return the current working directory path.
 
         Examples
         --------
         ::
 
           In [9]: pwd
           Out[9]: '/home/tsuser/sprint/ipython'
         """
         try:
             return os.getcwd()
         except FileNotFoundError as e:
             raise UsageError("CWD no longer exists - please use %cd to change directory.") from e
 
     @skip_doctest
     @line_magic
     def cd(self, parameter_s=''):
         """Change the current working directory.
 
         This command automatically maintains an internal list of directories
         you visit during your IPython session, in the variable ``_dh``. The
         command :magic:`%dhist` shows this history nicely formatted. You can
         also do ``cd -<tab>`` to see directory history conveniently.
         Usage:
 
           - ``cd 'dir'``: changes to directory 'dir'.
           - ``cd -``: changes to the last visited directory.
           - ``cd -<n>``: changes to the n-th directory in the directory history.
           - ``cd --foo``: change to directory that matches 'foo' in history
           - ``cd -b <bookmark_name>``: jump to a bookmark set by %bookmark
           - Hitting a tab key after ``cd -b`` allows you to tab-complete
             bookmark names.
 
           .. note::
             ``cd <bookmark_name>`` is enough if there is no directory
             ``<bookmark_name>``, but a bookmark with the name exists.
 
         Options:
 
         -q               Be quiet. Do not print the working directory after the
                          cd command is executed. By default IPython's cd
                          command does print this directory, since the default
                          prompts do not display path information.
 
         .. note::
            Note that ``!cd`` doesn't work for this purpose because the shell
            where ``!command`` runs is immediately discarded after executing
            'command'.
 
         Examples
         --------
         ::
 
           In [10]: cd parent/child
           /home/tsuser/parent/child
         """
 
         try:
             oldcwd = os.getcwd()
         except FileNotFoundError:
             # Happens if the CWD has been deleted.
             oldcwd = None
 
         numcd = re.match(r'(-)(\d+)$',parameter_s)
         # jump in directory history by number
         if numcd:
             nn = int(numcd.group(2))
             try:
                 ps = self.shell.user_ns['_dh'][nn]
             except IndexError:
                 print('The requested directory does not exist in history.')
                 return
             else:
                 opts = {}
         elif parameter_s.startswith('--'):
             ps = None
             fallback = None
             pat = parameter_s[2:]
             dh = self.shell.user_ns['_dh']
             # first search only by basename (last component)
             for ent in reversed(dh):
                 if pat in os.path.basename(ent) and os.path.isdir(ent):
                     ps = ent
                     break
 
                 if fallback is None and pat in ent and os.path.isdir(ent):
                     fallback = ent
 
             # if we have no last part match, pick the first full path match
             if ps is None:
                 ps = fallback
 
             if ps is None:
                 print("No matching entry in directory history")
                 return
             else:
                 opts = {}
 
 
         else:
             opts, ps = self.parse_options(parameter_s, 'qb', mode='string')
         # jump to previous
         if ps == '-':
             try:
                 ps = self.shell.user_ns['_dh'][-2]
             except IndexError as e:
                 raise UsageError('%cd -: No previous directory to change to.') from e
         # jump to bookmark if needed
         else:
             if not os.path.isdir(ps) or 'b' in opts:
                 bkms = self.shell.db.get('bookmarks', {})
 
                 if ps in bkms:
                     target = bkms[ps]
                     print('(bookmark:%s) -> %s' % (ps, target))
                     ps = target
                 else:
                     if 'b' in opts:
                         raise UsageError("Bookmark '%s' not found.  "
                               "Use '%%bookmark -l' to see your bookmarks." % ps)
 
         # at this point ps should point to the target dir
         if ps:
             try:
                 os.chdir(os.path.expanduser(ps))
                 if hasattr(self.shell, 'term_title') and self.shell.term_title:
                     set_term_title(self.shell.term_title_format.format(cwd=abbrev_cwd()))
             except OSError:
                 print(sys.exc_info()[1])
             else:
                 cwd = pathlib.Path.cwd()
                 dhist = self.shell.user_ns['_dh']
                 if oldcwd != cwd:
                     dhist.append(cwd)
                     self.shell.db['dhist'] = compress_dhist(dhist)[-100:]
 
         else:
             os.chdir(self.shell.home_dir)
             if hasattr(self.shell, 'term_title') and self.shell.term_title:
                 set_term_title(self.shell.term_title_format.format(cwd="~"))
             cwd = pathlib.Path.cwd()
             dhist = self.shell.user_ns['_dh']
 
             if oldcwd != cwd:
                 dhist.append(cwd)
-                self.shell.db["dhist"] = compress_dhist(dhist)[-100:]
-        if "q" not in opts and not self.cd_force_quiet and self.shell.user_ns["_dh"]:
-            print(self.shell.user_ns["_dh"][-1])
+                self.shell.db['dhist'] = compress_dhist(dhist)[-100:]
+        if not 'q' in opts and not self.cd_force_quiet and self.shell.user_ns['_dh']:
+            print(self.shell.user_ns['_dh'][-1])
 
     @line_magic
     def env(self, parameter_s=''):
         """Get, set, or list environment variables.
 
         Usage:\\
 
           :``%env``: lists all environment variables/values
           :``%env var``: get value for var
           :``%env var val``: set value for var
           :``%env var=val``: set value for var
           :``%env var=$val``: set value for var, using python expansion if possible
         """
         if parameter_s.strip():
             split = '=' if '=' in parameter_s else ' '
             bits = parameter_s.split(split)
             if len(bits) == 1:
                 key = parameter_s.strip()
                 if key in os.environ:
                     return os.environ[key]
                 else:
                     err = "Environment does not have key: {0}".format(key)
                     raise UsageError(err)
             if len(bits) > 1:
                 return self.set_env(parameter_s)
         env = dict(os.environ)
         # hide likely secrets when printing the whole environment
         for key in list(env):
             if any(s in key.lower() for s in ('key', 'token', 'secret')):
                 env[key] = '<hidden>'
 
         return env
 
     @line_magic
     def set_env(self, parameter_s):
         """Set environment variables.  Assumptions are that either "val" is a
         name in the user namespace, or val is something that evaluates to a
         string.
 
         Usage:\\
           :``%set_env var val``: set value for var
           :``%set_env var=val``: set value for var
           :``%set_env var=$val``: set value for var, using python expansion if possible
         """
         split = '=' if '=' in parameter_s else ' '
         bits = parameter_s.split(split, 1)
         if not parameter_s.strip() or len(bits)<2:
             raise UsageError("usage is 'set_env var=val'")
         var = bits[0].strip()
         val = bits[1].strip()
         if re.match(r'.*\s.*', var):
             # an environment variable with whitespace is almost certainly
             # not what the user intended.  what's more likely is the wrong
             # split was chosen, ie for "set_env cmd_args A=B", we chose
             # '=' for the split and should have chosen ' '.  to get around
             # this, users should just assign directly to os.environ or use
             # standard magic {var} expansion.
             err = "refusing to set env var with whitespace: '{0}'"
             err = err.format(val)
             raise UsageError(err)
         os.environ[var] = val
         print('env: {0}={1}'.format(var,val))
 
     @line_magic
     def pushd(self, parameter_s=''):
         """Place the current dir on stack and change directory.
 
         Usage:\\
           %pushd ['dirname']
         """
 
         dir_s = self.shell.dir_stack
         tgt = os.path.expanduser(parameter_s)
         cwd = os.getcwd().replace(self.shell.home_dir,'~')
         if tgt:
             self.cd(parameter_s)
         dir_s.insert(0,cwd)
         return self.shell.run_line_magic('dirs', '')
 
     @line_magic
     def popd(self, parameter_s=''):
         """Change to directory popped off the top of the stack.
         """
         if not self.shell.dir_stack:
             raise UsageError("%popd on empty stack")
         top = self.shell.dir_stack.pop(0)
         self.cd(top)
         print("popd ->",top)
 
     @line_magic
     def dirs(self, parameter_s=''):
         """Return the current directory stack."""
 
         return self.shell.dir_stack
 
     @line_magic
     def dhist(self, parameter_s=''):
         """Print your history of visited directories.
 
         %dhist       -> print full history\\
         %dhist n     -> print last n entries only\\
         %dhist n1 n2 -> print entries between n1 and n2 (n2 not included)\\
 
         This history is automatically maintained by the %cd command, and
         always available as the global list variable _dh. You can use %cd -<n>
         to go to directory number <n>.
 
         Note that most of time, you should view directory history by entering
         cd -<TAB>.
 
         """
 
         dh = self.shell.user_ns['_dh']
         if parameter_s:
             try:
                 args = map(int,parameter_s.split())
             except:
                 self.arg_err(self.dhist)
                 return
             if len(args) == 1:
                 ini,fin = max(len(dh)-(args[0]),0),len(dh)
             elif len(args) == 2:
                 ini,fin = args
                 fin = min(fin, len(dh))
             else:
                 self.arg_err(self.dhist)
                 return
         else:
             ini,fin = 0,len(dh)
         print('Directory history (kept in _dh)')
         for i in range(ini, fin):
             print("%d: %s" % (i, dh[i]))
 
     @skip_doctest
     @line_magic
     def sc(self, parameter_s=''):
         """Shell capture - run shell command and capture output (DEPRECATED use !).
 
         DEPRECATED. Suboptimal, retained for backwards compatibility.
 
         You should use the form 'var = !command' instead. Example:
 
          "%sc -l myfiles = ls ~" should now be written as
 
          "myfiles = !ls ~"
 
         myfiles.s, myfiles.l and myfiles.n still apply as documented
         below.
 
         --
         %sc [options] varname=command
 
         IPython will run the given command using commands.getoutput(), and
         will then update the user's interactive namespace with a variable
         called varname, containing the value of the call.  Your command can
         contain shell wildcards, pipes, etc.
 
         The '=' sign in the syntax is mandatory, and the variable name you
         supply must follow Python's standard conventions for valid names.
 
         (A special format without variable name exists for internal use)
 
         Options:
 
           -l: list output.  Split the output on newlines into a list before
           assigning it to the given variable.  By default the output is stored
           as a single string.
 
           -v: verbose.  Print the contents of the variable.
 
         In most cases you should not need to split as a list, because the
         returned value is a special type of string which can automatically
         provide its contents either as a list (split on newlines) or as a
         space-separated string.  These are convenient, respectively, either
         for sequential processing or to be passed to a shell command.
 
         For example::
 
             # Capture into variable a
             In [1]: sc a=ls *py
 
             # a is a string with embedded newlines
             In [2]: a
             Out[2]: 'setup.py\\nwin32_manual_post_install.py'
 
             # which can be seen as a list:
             In [3]: a.l
             Out[3]: ['setup.py', 'win32_manual_post_install.py']
 
             # or as a whitespace-separated string:
             In [4]: a.s
             Out[4]: 'setup.py win32_manual_post_install.py'
 
             # a.s is useful to pass as a single command line:
             In [5]: !wc -l $a.s
               146 setup.py
               130 win32_manual_post_install.py
               276 total
 
             # while the list form is useful to loop over:
             In [6]: for f in a.l:
                ...:      !wc -l $f
                ...:
             146 setup.py
             130 win32_manual_post_install.py
 
         Similarly, the lists returned by the -l option are also special, in
         the sense that you can equally invoke the .s attribute on them to
         automatically get a whitespace-separated string from their contents::
 
             In [7]: sc -l b=ls *py
 
             In [8]: b
             Out[8]: ['setup.py', 'win32_manual_post_install.py']
 
             In [9]: b.s
             Out[9]: 'setup.py win32_manual_post_install.py'
 
         In summary, both the lists and strings used for output capture have
         the following special attributes::
 
             .l (or .list) : value as list.
             .n (or .nlstr): value as newline-separated string.
             .s (or .spstr): value as space-separated string.
         """
 
         opts,args = self.parse_options(parameter_s, 'lv')
         # Try to get a variable name and command to run
         try:
             # the variable name must be obtained from the parse_options
             # output, which uses shlex.split to strip options out.
             var,_ = args.split('=', 1)
             var = var.strip()
             # But the command has to be extracted from the original input
             # parameter_s, not on what parse_options returns, to avoid the
             # quote stripping which shlex.split performs on it.
             _,cmd = parameter_s.split('=', 1)
         except ValueError:
             var,cmd = '',''
         # If all looks ok, proceed
         split = 'l' in opts
         out = self.shell.getoutput(cmd, split=split)
         if 'v' in opts:
             print('%s ==\n%s' % (var, pformat(out)))
         if var:
             self.shell.user_ns.update({var:out})
         else:
             return out
 
     @line_cell_magic
     def sx(self, line='', cell=None):
         """Shell execute - run shell command and capture output (!! is short-hand).
 
         %sx command
 
         IPython will run the given command using commands.getoutput(), and
         return the result formatted as a list (split on '\\n').  Since the
         output is _returned_, it will be stored in ipython's regular output
         cache Out[N] and in the '_N' automatic variables.
 
         Notes:
 
         1) If an input line begins with '!!', then %sx is automatically
         invoked.  That is, while::
 
           !ls
 
         causes ipython to simply issue system('ls'), typing::
 
           !!ls
 
         is a shorthand equivalent to::
 
           %sx ls
 
         2) %sx differs from %sc in that %sx automatically splits into a list,
         like '%sc -l'.  The reason for this is to make it as easy as possible
         to process line-oriented shell output via further python commands.
         %sc is meant to provide much finer control, but requires more
         typing.
 
         3) Just like %sc -l, this is a list with special attributes:
         ::
 
           .l (or .list) : value as list.
           .n (or .nlstr): value as newline-separated string.
           .s (or .spstr): value as whitespace-separated string.
 
         This is very useful when trying to use such lists as arguments to
         system commands."""
         
         if cell is None:
             # line magic
             return self.shell.getoutput(line)
         else:
             opts,args = self.parse_options(line, '', 'out=')
             output = self.shell.getoutput(cell)
             out_name = opts.get('out', opts.get('o'))
             if out_name:
                 self.shell.user_ns[out_name] = output
             else:
                 return output
 
     system = line_cell_magic('system')(sx)
     bang = cell_magic('!')(sx)
 
     @line_magic
     def bookmark(self, parameter_s=''):
         """Manage IPython's bookmark system.
 
         %bookmark <name>       - set bookmark to current dir
         %bookmark <name> <dir> - set bookmark to <dir>
         %bookmark -l           - list all bookmarks
         %bookmark -d <name>    - remove bookmark
         %bookmark -r           - remove all bookmarks
 
         You can later on access a bookmarked folder with::
 
           %cd -b <name>
 
         or simply '%cd <name>' if there is no directory called <name> AND
         there is such a bookmark defined.
 
         Your bookmarks persist through IPython sessions, but they are
         associated with each profile."""
 
         opts,args = self.parse_options(parameter_s,'drl',mode='list')
         if len(args) > 2:
             raise UsageError("%bookmark: too many arguments")
 
         bkms = self.shell.db.get('bookmarks',{})
 
         if 'd' in opts:
             try:
                 todel = args[0]
             except IndexError as e:
                 raise UsageError(
                     "%bookmark -d: must provide a bookmark to delete") from e
             else:
                 try:
                     del bkms[todel]
                 except KeyError as e:
                     raise UsageError(
                         "%%bookmark -d: Can't delete bookmark '%s'" % todel) from e
 
         elif 'r' in opts:
             bkms = {}
         elif 'l' in opts:
             bks = sorted(bkms)
             if bks:
                 size = max(map(len, bks))
             else:
                 size = 0
             fmt = '%-'+str(size)+'s -> %s'
             print('Current bookmarks:')
             for bk in bks:
                 print(fmt % (bk, bkms[bk]))
         else:
             if not args:
                 raise UsageError("%bookmark: You must specify the bookmark name")
             elif len(args)==1:
                 bkms[args[0]] = os.getcwd()
             elif len(args)==2:
                 bkms[args[0]] = args[1]
         self.shell.db['bookmarks'] = bkms
 
     @line_magic
     def pycat(self, parameter_s=''):
         """Show a syntax-highlighted file through a pager.
 
         This magic is similar to the cat utility, but it will assume the file
         to be Python source and will show it with syntax highlighting.
 
         This magic command can either take a local filename, an url,
         an history range (see %history) or a macro as argument.
 
         If no parameter is given, prints out history of current session up to
         this point. ::
 
         %pycat myscript.py
         %pycat 7-27
         %pycat myMacro
         %pycat http://www.example.com/myscript.py
         """
         try:
             cont = self.shell.find_user_code(parameter_s, skip_encoding_cookie=False)
         except (ValueError, IOError):
             print("Error: no such file, variable, URL, history range or macro")
             return
 
         page.page(self.shell.pycolorize(source_to_unicode(cont)))
 
     @magic_arguments.magic_arguments()
     @magic_arguments.argument(
         '-a', '--append', action='store_true', default=False,
         help='Append contents of the cell to an existing file. '
              'The file will be created if it does not exist.'
     )
     @magic_arguments.argument(
         'filename', type=str,
         help='file to write'
     )
     @cell_magic
     def writefile(self, line, cell):
         """Write the contents of the cell to a file.
 
         The file will be overwritten unless the -a (--append) flag is specified.
         """
         args = magic_arguments.parse_argstring(self.writefile, line)
         if re.match(r'^(\'.*\')|(".*")$', args.filename):
             filename = os.path.expanduser(args.filename[1:-1])
         else:
             filename = os.path.expanduser(args.filename)
             
         if os.path.exists(filename):
             if args.append:
                 print("Appending to %s" % filename)
             else:
                 print("Overwriting %s" % filename)
         else:
             print("Writing %s" % filename)
         
         mode = 'a' if args.append else 'w'
         with io.open(filename, mode, encoding='utf-8') as f:
             f.write(cell)

IPython/core/oinspect.py

@@ -1,1209 +1,1210
 """Tools for inspecting Python objects.
 
 Uses syntax highlighting for presenting the various information elements.
 
 Similar in spirit to the inspect module, but all calls take a name argument to
 reference the name under which an object is being read.
 """
 
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 __all__ = ['Inspector','InspectColors']
 
 # stdlib modules
 from dataclasses import dataclass
 from inspect import signature
 from textwrap import dedent
 import ast
 import html
 import inspect
 import io as stdlib_io
 import linecache
 import os
 import types
 import warnings
 
 
 from typing import (
     cast,
     Any,
     Optional,
     Dict,
     Union,
     List,
     TypedDict,
     TypeAlias,
     Tuple,
 )
 
 import traitlets
 
 # IPython's own
 from IPython.core import page
 from IPython.lib.pretty import pretty
 from IPython.testing.skipdoctest import skip_doctest
 from IPython.utils import PyColorize, openpy
 from IPython.utils.dir2 import safe_hasattr
 from IPython.utils.path import compress_user
 from IPython.utils.text import indent
 from IPython.utils.wildcard import list_namespace, typestr2type
 from IPython.utils.coloransi import TermColors
 from IPython.utils.colorable import Colorable
 from IPython.utils.decorators import undoc
 
 from pygments import highlight
 from pygments.lexers import PythonLexer
 from pygments.formatters import HtmlFormatter
 
 HOOK_NAME = "__custom_documentations__"
 
 
 UnformattedBundle: TypeAlias = Dict[str, List[Tuple[str, str]]]  # List of (title, body)
 Bundle: TypeAlias = Dict[str, str]
 
 
 @dataclass
 class OInfo:
     ismagic: bool
     isalias: bool
     found: bool
     namespace: Optional[str]
     parent: Any
     obj: Any
 
     def get(self, field):
         """Get a field from the object for backward compatibility with before 8.12
 
         see https://github.com/h5py/h5py/issues/2253
         """
         # We need to deprecate this at some point, but the warning will show in completion.
         # Let's comment this for now and uncomment end of 2023 ish
         #        warnings.warn(
         #            f"OInfo dataclass with fields access since IPython 8.12 please use OInfo.{field} instead."
         #            "OInfo used to be a dict but a dataclass provide static fields verification with mypy."
         #            "This warning and backward compatibility `get()` method were added in 8.13.",
         #            DeprecationWarning,
         #            stacklevel=2,
         #        )
         return getattr(self, field)
 
 
 def pylight(code):
     return highlight(code, PythonLexer(), HtmlFormatter(noclasses=True))
 
 # builtin docstrings to ignore
 _func_call_docstring = types.FunctionType.__call__.__doc__
 _object_init_docstring = object.__init__.__doc__
 _builtin_type_docstrings = {
     inspect.getdoc(t) for t in (types.ModuleType, types.MethodType,
                                 types.FunctionType, property)
 }
 
 _builtin_func_type = type(all)
 _builtin_meth_type = type(str.upper)  # Bound methods have the same type as builtin functions
 #****************************************************************************
 # Builtin color schemes
 
 Colors = TermColors  # just a shorthand
 
 InspectColors = PyColorize.ANSICodeColors
 
 #****************************************************************************
 # Auxiliary functions and objects
 
 
 class InfoDict(TypedDict):
     type_name: Optional[str]
     base_class: Optional[str]
     string_form: Optional[str]
     namespace: Optional[str]
     length: Optional[str]
     file: Optional[str]
     definition: Optional[str]
     docstring: Optional[str]
     source: Optional[str]
     init_definition: Optional[str]
     class_docstring: Optional[str]
     init_docstring: Optional[str]
     call_def: Optional[str]
     call_docstring: Optional[str]
     subclasses: Optional[str]
     # These won't be printed but will be used to determine how to
     # format the object
     ismagic: bool
     isalias: bool
     isclass: bool
     found: bool
     name: str
 
 
 _info_fields = list(InfoDict.__annotations__.keys())
 
 
 def __getattr__(name):
     if name == "info_fields":
         warnings.warn(
             "IPython.core.oinspect's `info_fields` is considered for deprecation and may be removed in the Future. ",
             DeprecationWarning,
             stacklevel=2,
         )
         return _info_fields
 
     raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
 
 
 @dataclass
 class InspectorHookData:
     """Data passed to the mime hook"""
 
     obj: Any
     info: Optional[OInfo]
     info_dict: InfoDict
     detail_level: int
     omit_sections: list[str]
 
 
 @undoc
 def object_info(
     *,
     name: str,
     found: bool,
     isclass: bool = False,
     isalias: bool = False,
     ismagic: bool = False,
     **kw,
 ) -> InfoDict:
     """Make an object info dict with all fields present."""
     infodict = kw
     infodict = {k: None for k in _info_fields if k not in infodict}
     infodict["name"] = name  # type: ignore
     infodict["found"] = found  # type: ignore
     infodict["isclass"] = isclass  # type: ignore
     infodict["isalias"] = isalias  # type: ignore
     infodict["ismagic"] = ismagic  # type: ignore
 
     return InfoDict(**infodict)  # type:ignore
 
 
 def get_encoding(obj):
     """Get encoding for python source file defining obj
 
     Returns None if obj is not defined in a sourcefile.
     """
     ofile = find_file(obj)
     # run contents of file through pager starting at line where the object
     # is defined, as long as the file isn't binary and is actually on the
     # filesystem.
     if ofile is None:
         return None
     elif ofile.endswith(('.so', '.dll', '.pyd')):
         return None
     elif not os.path.isfile(ofile):
         return None
     else:
         # Print only text files, not extension binaries.  Note that
         # getsourcelines returns lineno with 1-offset and page() uses
         # 0-offset, so we must adjust.
         with stdlib_io.open(ofile, 'rb') as buffer:   # Tweaked to use io.open for Python 2
             encoding, _lines = openpy.detect_encoding(buffer.readline)
         return encoding
 
 
 def getdoc(obj) -> Union[str, None]:
     """Stable wrapper around inspect.getdoc.
 
     This can't crash because of attribute problems.
 
     It also attempts to call a getdoc() method on the given object.  This
     allows objects which provide their docstrings via non-standard mechanisms
     (like Pyro proxies) to still be inspected by ipython's ? system.
     """
     # Allow objects to offer customized documentation via a getdoc method:
     try:
         ds = obj.getdoc()
     except Exception:
         pass
     else:
         if isinstance(ds, str):
             return inspect.cleandoc(ds)
     docstr = inspect.getdoc(obj)
     return docstr
 
 
 def getsource(obj, oname='') -> Union[str,None]:
     """Wrapper around inspect.getsource.
 
     This can be modified by other projects to provide customized source
     extraction.
 
     Parameters
     ----------
     obj : object
         an object whose source code we will attempt to extract
     oname : str
         (optional) a name under which the object is known
 
     Returns
     -------
     src : unicode or None
 
     """
 
     if isinstance(obj, property):
         sources = []
         for attrname in ['fget', 'fset', 'fdel']:
             fn = getattr(obj, attrname)
             if fn is not None:
+                encoding = get_encoding(fn)
                 oname_prefix = ('%s.' % oname) if oname else ''
                 sources.append(''.join(('# ', oname_prefix, attrname)))
                 if inspect.isfunction(fn):
                     _src = getsource(fn)
                     if _src:
                         # assert _src is not None, "please mypy"
                         sources.append(dedent(_src))
                 else:
                     # Default str/repr only prints function name,
                     # pretty.pretty prints module name too.
                     sources.append(
                         '%s%s = %s\n' % (oname_prefix, attrname, pretty(fn))
                     )
         if sources:
             return '\n'.join(sources)
         else:
             return None
 
     else:
         # Get source for non-property objects.
 
         obj = _get_wrapped(obj)
 
         try:
             src = inspect.getsource(obj)
         except TypeError:
             # The object itself provided no meaningful source, try looking for
             # its class definition instead.
             try:
                 src = inspect.getsource(obj.__class__)
             except (OSError, TypeError):
                 return None
         except OSError:
             return None
 
         return src
 
 
 def is_simple_callable(obj):
     """True if obj is a function ()"""
     return (inspect.isfunction(obj) or inspect.ismethod(obj) or \
             isinstance(obj, _builtin_func_type) or isinstance(obj, _builtin_meth_type))
 
 def _get_wrapped(obj):
     """Get the original object if wrapped in one or more @decorators
 
     Some objects automatically construct similar objects on any unrecognised
     attribute access (e.g. unittest.mock.call). To protect against infinite loops,
     this will arbitrarily cut off after 100 levels of obj.__wrapped__
     attribute access. --TK, Jan 2016
     """
     orig_obj = obj
     i = 0
     while safe_hasattr(obj, '__wrapped__'):
         obj = obj.__wrapped__
         i += 1
         if i > 100:
             # __wrapped__ is probably a lie, so return the thing we started with
             return orig_obj
     return obj
 
 def find_file(obj) -> Optional[str]:
     """Find the absolute path to the file where an object was defined.
 
     This is essentially a robust wrapper around `inspect.getabsfile`.
 
     Returns None if no file can be found.
 
     Parameters
     ----------
     obj : any Python object
 
     Returns
     -------
     fname : str
         The absolute path to the file where the object was defined.
     """
     obj = _get_wrapped(obj)
 
     fname: Optional[str] = None
     try:
         fname = inspect.getabsfile(obj)
     except TypeError:
         # For an instance, the file that matters is where its class was
         # declared.
         try:
             fname = inspect.getabsfile(obj.__class__)
         except (OSError, TypeError):
             # Can happen for builtins
             pass
     except OSError:
         pass
 
     return fname
 
 
 def find_source_lines(obj):
     """Find the line number in a file where an object was defined.
 
     This is essentially a robust wrapper around `inspect.getsourcelines`.
 
     Returns None if no file can be found.
 
     Parameters
     ----------
     obj : any Python object
 
     Returns
     -------
     lineno : int
         The line number where the object definition starts.
     """
     obj = _get_wrapped(obj)
 
     try:
         lineno = inspect.getsourcelines(obj)[1]
     except TypeError:
         # For instances, try the class object like getsource() does
         try:
             lineno = inspect.getsourcelines(obj.__class__)[1]
         except (OSError, TypeError):
             return None
     except OSError:
         return None
 
     return lineno
 
 class Inspector(Colorable):
 
     mime_hooks = traitlets.Dict(
         config=True,
         help="dictionary of mime to callable to add information into help mimebundle dict",
     ).tag(config=True)
 
     def __init__(
         self,
         color_table=InspectColors,
         code_color_table=PyColorize.ANSICodeColors,
         scheme=None,
         str_detail_level=0,
         parent=None,
         config=None,
     ):
         super(Inspector, self).__init__(parent=parent, config=config)
         self.color_table = color_table
         self.parser = PyColorize.Parser(out='str', parent=self, style=scheme)
         self.format = self.parser.format
         self.str_detail_level = str_detail_level
         self.set_active_scheme(scheme)
 
     def _getdef(self,obj,oname='') -> Union[str,None]:
         """Return the call signature for any callable object.
 
         If any exception is generated, None is returned instead and the
         exception is suppressed."""
         if not callable(obj):
             return None
         try:
             return _render_signature(signature(obj), oname)
         except:
             return None
 
     def __head(self,h) -> str:
         """Return a header string with proper colors."""
         return '%s%s%s' % (self.color_table.active_colors.header,h,
                            self.color_table.active_colors.normal)
 
     def set_active_scheme(self, scheme):
         if scheme is not None:
             self.color_table.set_active_scheme(scheme)
             self.parser.color_table.set_active_scheme(scheme)
 
     def noinfo(self, msg, oname):
         """Generic message when no information is found."""
         print('No %s found' % msg, end=' ')
         if oname:
             print('for %s' % oname)
         else:
             print()
 
     def pdef(self, obj, oname=''):
         """Print the call signature for any callable object.
 
         If the object is a class, print the constructor information."""
 
         if not callable(obj):
             print('Object is not callable.')
             return
 
         header = ''
 
         if inspect.isclass(obj):
             header = self.__head('Class constructor information:\n')
 
 
         output = self._getdef(obj,oname)
         if output is None:
             self.noinfo('definition header',oname)
         else:
             print(header,self.format(output), end=' ')
 
     # In Python 3, all classes are new-style, so they all have __init__.
     @skip_doctest
     def pdoc(self, obj, oname='', formatter=None):
         """Print the docstring for any object.
 
         Optional:
         -formatter: a function to run the docstring through for specially
         formatted docstrings.
 
         Examples
         --------
         In [1]: class NoInit:
            ...:     pass
 
         In [2]: class NoDoc:
            ...:     def __init__(self):
            ...:         pass
 
         In [3]: %pdoc NoDoc
         No documentation found for NoDoc
 
         In [4]: %pdoc NoInit
         No documentation found for NoInit
 
         In [5]: obj = NoInit()
 
         In [6]: %pdoc obj
         No documentation found for obj
 
         In [5]: obj2 = NoDoc()
 
         In [6]: %pdoc obj2
         No documentation found for obj2
         """
 
         head = self.__head  # For convenience
         lines = []
         ds = getdoc(obj)
         if formatter:
             ds = formatter(ds).get('plain/text', ds)
         if ds:
             lines.append(head("Class docstring:"))
             lines.append(indent(ds))
         if inspect.isclass(obj) and hasattr(obj, '__init__'):
             init_ds = getdoc(obj.__init__)
             if init_ds is not None:
                 lines.append(head("Init docstring:"))
                 lines.append(indent(init_ds))
         elif hasattr(obj,'__call__'):
             call_ds = getdoc(obj.__call__)
             if call_ds:
                 lines.append(head("Call docstring:"))
                 lines.append(indent(call_ds))
 
         if not lines:
             self.noinfo('documentation',oname)
         else:
             page.page('\n'.join(lines))
 
     def psource(self, obj, oname=''):
         """Print the source code for an object."""
 
         # Flush the source cache because inspect can return out-of-date source
         linecache.checkcache()
         try:
             src = getsource(obj, oname=oname)
         except Exception:
             src = None
 
         if src is None:
             self.noinfo('source', oname)
         else:
             page.page(self.format(src))
 
     def pfile(self, obj, oname=''):
         """Show the whole file where an object was defined."""
 
         lineno = find_source_lines(obj)
         if lineno is None:
             self.noinfo('file', oname)
             return
 
         ofile = find_file(obj)
         # run contents of file through pager starting at line where the object
         # is defined, as long as the file isn't binary and is actually on the
         # filesystem.
         if ofile is None:
             print("Could not find file for object")
         elif ofile.endswith((".so", ".dll", ".pyd")):
             print("File %r is binary, not printing." % ofile)
         elif not os.path.isfile(ofile):
             print('File %r does not exist, not printing.' % ofile)
         else:
             # Print only text files, not extension binaries.  Note that
             # getsourcelines returns lineno with 1-offset and page() uses
             # 0-offset, so we must adjust.
             page.page(self.format(openpy.read_py_file(ofile, skip_encoding_cookie=False)), lineno - 1)
 
 
     def _mime_format(self, text:str, formatter=None) -> dict:
         """Return a mime bundle representation of the input text.
 
         - if `formatter` is None, the returned mime bundle has
            a ``text/plain`` field, with the input text.
            a ``text/html`` field with a ``<pre>`` tag containing the input text.
 
         - if ``formatter`` is not None, it must be a callable transforming the
           input text into a mime bundle. Default values for ``text/plain`` and
           ``text/html`` representations are the ones described above.
 
         Note:
 
         Formatters returning strings are supported but this behavior is deprecated.
 
         """
         defaults = {
             "text/plain": text,
             "text/html": f"<pre>{html.escape(text)}</pre>",
         }
 
         if formatter is None:
             return defaults
         else:
             formatted = formatter(text)
 
             if not isinstance(formatted, dict):
                 # Handle the deprecated behavior of a formatter returning
                 # a string instead of a mime bundle.
                 return {"text/plain": formatted, "text/html": f"<pre>{formatted}</pre>"}
 
             else:
                 return dict(defaults, **formatted)
 
     def format_mime(self, bundle: UnformattedBundle) -> Bundle:
         """Format a mimebundle being created by _make_info_unformatted into a real mimebundle"""
         # Format text/plain mimetype
         assert isinstance(bundle["text/plain"], list)
         for item in bundle["text/plain"]:
             assert isinstance(item, tuple)
 
         new_b: Bundle = {}
         lines = []
         _len = max(len(h) for h, _ in bundle["text/plain"])
 
         for head, body in bundle["text/plain"]:
             body = body.strip("\n")
             delim = "\n" if "\n" in body else " "
             lines.append(
                 f"{self.__head(head+':')}{(_len - len(head))*' '}{delim}{body}"
             )
 
         new_b["text/plain"] = "\n".join(lines)
 
         if "text/html" in bundle:
             assert isinstance(bundle["text/html"], list)
             for item in bundle["text/html"]:
                 assert isinstance(item, tuple)
             # Format the text/html mimetype
             if isinstance(bundle["text/html"], (list, tuple)):
                 # bundle['text/html'] is a list of (head, formatted body) pairs
                 new_b["text/html"] = "\n".join(
                     (f"<h1>{head}</h1>\n{body}" for (head, body) in bundle["text/html"])
                 )
 
         for k in bundle.keys():
             if k in ("text/html", "text/plain"):
                 continue
             else:
                 new_b[k] = bundle[k]  # type:ignore
         return new_b
 
     def _append_info_field(
         self,
         bundle: UnformattedBundle,
         title: str,
         key: str,
         info,
         omit_sections: List[str],
         formatter,
     ):
         """Append an info value to the unformatted mimebundle being constructed by _make_info_unformatted"""
         if title in omit_sections or key in omit_sections:
             return
         field = info[key]
         if field is not None:
             formatted_field = self._mime_format(field, formatter)
             bundle["text/plain"].append((title, formatted_field["text/plain"]))
             bundle["text/html"].append((title, formatted_field["text/html"]))
 
     def _make_info_unformatted(
         self, obj, info, formatter, detail_level, omit_sections
     ) -> UnformattedBundle:
         """Assemble the mimebundle as unformatted lists of information"""
         bundle: UnformattedBundle = {
             "text/plain": [],
             "text/html": [],
         }
 
         # A convenience function to simplify calls below
         def append_field(
             bundle: UnformattedBundle, title: str, key: str, formatter=None
         ):
             self._append_info_field(
                 bundle,
                 title=title,
                 key=key,
                 info=info,
                 omit_sections=omit_sections,
                 formatter=formatter,
             )
 
         def code_formatter(text) -> Bundle:
             return {
                 'text/plain': self.format(text),
                 'text/html': pylight(text)
             }
 
         if info["isalias"]:
             append_field(bundle, "Repr", "string_form")
 
         elif info['ismagic']:
             if detail_level > 0:
                 append_field(bundle, "Source", "source", code_formatter)
             else:
                 append_field(bundle, "Docstring", "docstring", formatter)
             append_field(bundle, "File", "file")
 
         elif info['isclass'] or is_simple_callable(obj):
             # Functions, methods, classes
             append_field(bundle, "Signature", "definition", code_formatter)
             append_field(bundle, "Init signature", "init_definition", code_formatter)
             append_field(bundle, "Docstring", "docstring", formatter)
             if detail_level > 0 and info["source"]:
                 append_field(bundle, "Source", "source", code_formatter)
             else:
                 append_field(bundle, "Init docstring", "init_docstring", formatter)
 
             append_field(bundle, "File", "file")
             append_field(bundle, "Type", "type_name")
             append_field(bundle, "Subclasses", "subclasses")
 
         else:
             # General Python objects
             append_field(bundle, "Signature", "definition", code_formatter)
             append_field(bundle, "Call signature", "call_def", code_formatter)
             append_field(bundle, "Type", "type_name")
             append_field(bundle, "String form", "string_form")
 
             # Namespace
             if info["namespace"] != "Interactive":
                 append_field(bundle, "Namespace", "namespace")
 
             append_field(bundle, "Length", "length")
             append_field(bundle, "File", "file")
 
             # Source or docstring, depending on detail level and whether
             # source found.
             if detail_level > 0 and info["source"]:
                 append_field(bundle, "Source", "source", code_formatter)
             else:
                 append_field(bundle, "Docstring", "docstring", formatter)
 
             append_field(bundle, "Class docstring", "class_docstring", formatter)
             append_field(bundle, "Init docstring", "init_docstring", formatter)
             append_field(bundle, "Call docstring", "call_docstring", formatter)
         return bundle
 
 
     def _get_info(
         self,
         obj: Any,
         oname: str = "",
         formatter=None,
         info: Optional[OInfo] = None,
         detail_level: int = 0,
         omit_sections: Union[List[str], Tuple[()]] = (),
     ) -> Bundle:
         """Retrieve an info dict and format it.
 
         Parameters
         ----------
         obj : any
             Object to inspect and return info from
         oname : str (default: ''):
             Name of the variable pointing to `obj`.
         formatter : callable
         info
             already computed information
         detail_level : integer
             Granularity of detail level, if set to 1, give more information.
         omit_sections : list[str]
             Titles or keys to omit from output (can be set, tuple, etc., anything supporting `in`)
         """
 
         info_dict = self.info(obj, oname=oname, info=info, detail_level=detail_level)
         omit_sections = list(omit_sections)
 
         bundle = self._make_info_unformatted(
             obj,
             info_dict,
             formatter,
             detail_level=detail_level,
             omit_sections=omit_sections,
         )
         if self.mime_hooks:
             hook_data = InspectorHookData(
                 obj=obj,
                 info=info,
                 info_dict=info_dict,
                 detail_level=detail_level,
                 omit_sections=omit_sections,
             )
             for key, hook in self.mime_hooks.items():  # type:ignore
                 required_parameters = [
                     parameter
                     for parameter in inspect.signature(hook).parameters.values()
                     if parameter.default != inspect.Parameter.default
                 ]
                 if len(required_parameters) == 1:
                     res = hook(hook_data)
                 else:
                     warnings.warn(
                         "MIME hook format changed in IPython 8.22; hooks should now accept"
                         " a single parameter (InspectorHookData); support for hooks requiring"
                         " two-parameters (obj and info) will be removed in a future version",
                         DeprecationWarning,
                         stacklevel=2,
                     )
                     res = hook(obj, info)
                 if res is not None:
                     bundle[key] = res
         return self.format_mime(bundle)
 
     def pinfo(
         self,
         obj,
         oname="",
         formatter=None,
         info: Optional[OInfo] = None,
         detail_level=0,
         enable_html_pager=True,
         omit_sections=(),
     ):
         """Show detailed information about an object.
 
         Optional arguments:
 
         - oname: name of the variable pointing to the object.
 
         - formatter: callable (optional)
               A special formatter for docstrings.
 
               The formatter is a callable that takes a string as an input
               and returns either a formatted string or a mime type bundle
               in the form of a dictionary.
 
               Although the support of custom formatter returning a string
               instead of a mime type bundle is deprecated.
 
         - info: a structure with some information fields which may have been
           precomputed already.
 
         - detail_level: if set to 1, more information is given.
 
         - omit_sections: set of section keys and titles to omit
         """
         assert info is not None
         info_b: Bundle = self._get_info(
             obj, oname, formatter, info, detail_level, omit_sections=omit_sections
         )
         if not enable_html_pager:
             del info_b["text/html"]
         page.page(info_b)
 
     def _info(self, obj, oname="", info=None, detail_level=0):
         """
         Inspector.info() was likely improperly marked as deprecated
         while only a parameter was deprecated. We "un-deprecate" it.
         """
 
         warnings.warn(
             "The `Inspector.info()` method has been un-deprecated as of 8.0 "
             "and the `formatter=` keyword removed. `Inspector._info` is now "
             "an alias, and you can just call `.info()` directly.",
             DeprecationWarning,
             stacklevel=2,
         )
         return self.info(obj, oname=oname, info=info, detail_level=detail_level)
 
     def info(self, obj, oname="", info=None, detail_level=0) -> InfoDict:
         """Compute a dict with detailed information about an object.
 
         Parameters
         ----------
         obj : any
             An object to find information about
         oname : str (default: '')
             Name of the variable pointing to `obj`.
         info : (default: None)
             A struct (dict like with attr access) with some information fields
             which may have been precomputed already.
         detail_level : int (default:0)
             If set to 1, more information is given.
 
         Returns
         -------
         An object info dict with known fields from `info_fields` (see `InfoDict`).
         """
 
         if info is None:
             ismagic = False
             isalias = False
             ospace = ''
         else:
             ismagic = info.ismagic
             isalias = info.isalias
             ospace = info.namespace
 
         # Get docstring, special-casing aliases:
         att_name = oname.split(".")[-1]
         parents_docs = None
         prelude = ""
         if info and info.parent is not None and hasattr(info.parent, HOOK_NAME):
             parents_docs_dict = getattr(info.parent, HOOK_NAME)
             parents_docs = parents_docs_dict.get(att_name, None)
         out: InfoDict = cast(
             InfoDict,
             {
                 **{field: None for field in _info_fields},
                 **{
                     "name": oname,
                     "found": True,
                     "isalias": isalias,
                     "ismagic": ismagic,
                     "subclasses": None,
                 },
             },
         )
 
         if parents_docs:
             ds = parents_docs
         elif isalias:
             if not callable(obj):
                 try:
                     ds = "Alias to the system command:\n  %s" % obj[1]
                 except:
                     ds = "Alias: " + str(obj)
             else:
                 ds = "Alias to " + str(obj)
                 if obj.__doc__:
                     ds += "\nDocstring:\n" + obj.__doc__
         else:
             ds_or_None = getdoc(obj)
             if ds_or_None is None:
                 ds = '<no docstring>'
             else:
                 ds = ds_or_None
 
         ds = prelude + ds
 
         # store output in a dict, we initialize it here and fill it as we go
 
         string_max = 200 # max size of strings to show (snipped if longer)
         shalf = int((string_max - 5) / 2)
 
         if ismagic:
             out['type_name'] = 'Magic function'
         elif isalias:
             out['type_name'] = 'System alias'
         else:
             out['type_name'] = type(obj).__name__
 
         try:
             bclass = obj.__class__
             out['base_class'] = str(bclass)
         except:
             pass
 
         # String form, but snip if too long in ? form (full in ??)
         if detail_level >= self.str_detail_level:
             try:
                 ostr = str(obj)
                 if not detail_level and len(ostr) > string_max:
                     ostr = ostr[:shalf] + ' <...> ' + ostr[-shalf:]
                     # TODO: `'string_form'.expandtabs()` seems wrong, but
                     # it was (nearly) like this since the first commit ever.
                     ostr = ("\n" + " " * len("string_form".expandtabs())).join(
                         q.strip() for q in ostr.split("\n")
                     )
                 out["string_form"] = ostr
             except:
                 pass
 
         if ospace:
             out['namespace'] = ospace
 
         # Length (for strings and lists)
         try:
             out['length'] = str(len(obj))
         except Exception:
             pass
 
         # Filename where object was defined
         binary_file = False
         fname = find_file(obj)
         if fname is None:
             # if anything goes wrong, we don't want to show source, so it's as
             # if the file was binary
             binary_file = True
         else:
             if fname.endswith(('.so', '.dll', '.pyd')):
                 binary_file = True
             elif fname.endswith('<string>'):
                 fname = 'Dynamically generated function. No source code available.'
             out['file'] = compress_user(fname)
 
         # Original source code for a callable, class or property.
         if detail_level:
             # Flush the source cache because inspect can return out-of-date
             # source
             linecache.checkcache()
             try:
                 if isinstance(obj, property) or not binary_file:
                     src = getsource(obj, oname)
                     if src is not None:
                         src = src.rstrip()
                     out['source'] = src
 
             except Exception:
                 pass
 
         # Add docstring only if no source is to be shown (avoid repetitions).
         if ds and not self._source_contains_docstring(out.get('source'), ds):
             out['docstring'] = ds
 
         # Constructor docstring for classes
         if inspect.isclass(obj):
             out['isclass'] = True
 
             # get the init signature:
             try:
                 init_def = self._getdef(obj, oname)
             except AttributeError:
                 init_def = None
 
             # get the __init__ docstring
             try:
                 obj_init = obj.__init__
             except AttributeError:
                 init_ds = None
             else:
                 if init_def is None:
                     # Get signature from init if top-level sig failed.
                     # Can happen for built-in types (list, etc.).
                     try:
                         init_def = self._getdef(obj_init, oname)
                     except AttributeError:
                         pass
                 init_ds = getdoc(obj_init)
                 # Skip Python's auto-generated docstrings
                 if init_ds == _object_init_docstring:
                     init_ds = None
 
             if init_def:
                 out['init_definition'] = init_def
 
             if init_ds:
                 out['init_docstring'] = init_ds
 
             names = [sub.__name__ for sub in type.__subclasses__(obj)]
             if len(names) < 10:
                 all_names = ', '.join(names)
             else:
                 all_names = ', '.join(names[:10]+['...'])
             out['subclasses'] = all_names
         # and class docstring for instances:
         else:
             # reconstruct the function definition and print it:
             defln = self._getdef(obj, oname)
             if defln:
                 out['definition'] = defln
 
             # First, check whether the instance docstring is identical to the
             # class one, and print it separately if they don't coincide.  In
             # most cases they will, but it's nice to print all the info for
             # objects which use instance-customized docstrings.
             if ds:
                 try:
                     cls = getattr(obj,'__class__')
                 except:
                     class_ds = None
                 else:
                     class_ds = getdoc(cls)
                 # Skip Python's auto-generated docstrings
                 if class_ds in _builtin_type_docstrings:
                     class_ds = None
                 if class_ds and ds != class_ds:
                     out['class_docstring'] = class_ds
 
             # Next, try to show constructor docstrings
             try:
                 init_ds = getdoc(obj.__init__)
                 # Skip Python's auto-generated docstrings
                 if init_ds == _object_init_docstring:
                     init_ds = None
             except AttributeError:
                 init_ds = None
             if init_ds:
                 out['init_docstring'] = init_ds
 
             # Call form docstring for callable instances
             if safe_hasattr(obj, '__call__') and not is_simple_callable(obj):
                 call_def = self._getdef(obj.__call__, oname)
                 if call_def and (call_def != out.get('definition')):
                     # it may never be the case that call def and definition differ,
                     # but don't include the same signature twice
                     out['call_def'] = call_def
                 call_ds = getdoc(obj.__call__)
                 # Skip Python's auto-generated docstrings
                 if call_ds == _func_call_docstring:
                     call_ds = None
                 if call_ds:
                     out['call_docstring'] = call_ds
 
         return out
 
     @staticmethod
     def _source_contains_docstring(src, doc):
         """
         Check whether the source *src* contains the docstring *doc*.
 
         This is is helper function to skip displaying the docstring if the
         source already contains it, avoiding repetition of information.
         """
         try:
             (def_node,) = ast.parse(dedent(src)).body
             return ast.get_docstring(def_node) == doc  # type: ignore[arg-type]
         except Exception:
             # The source can become invalid or even non-existent (because it
             # is re-fetched from the source file) so the above code fail in
             # arbitrary ways.
             return False
 
     def psearch(self,pattern,ns_table,ns_search=[],
                 ignore_case=False,show_all=False, *, list_types=False):
         """Search namespaces with wildcards for objects.
 
         Arguments:
 
         - pattern: string containing shell-like wildcards to use in namespace
           searches and optionally a type specification to narrow the search to
           objects of that type.
 
         - ns_table: dict of name->namespaces for search.
 
         Optional arguments:
 
           - ns_search: list of namespace names to include in search.
 
           - ignore_case(False): make the search case-insensitive.
 
           - show_all(False): show all names, including those starting with
             underscores.
 
           - list_types(False): list all available object types for object matching.
         """
         # print('ps pattern:<%r>' % pattern)  # dbg
 
         # defaults
         type_pattern = 'all'
         filter = ''
 
         # list all object types
         if list_types:
             page.page('\n'.join(sorted(typestr2type)))
             return
 
         cmds = pattern.split()
         len_cmds  =  len(cmds)
         if len_cmds == 1:
             # Only filter pattern given
             filter = cmds[0]
         elif len_cmds == 2:
             # Both filter and type specified
             filter,type_pattern = cmds
         else:
             raise ValueError('invalid argument string for psearch: <%s>' %
                              pattern)
 
         # filter search namespaces
         for name in ns_search:
             if name not in ns_table:
                 raise ValueError('invalid namespace <%s>. Valid names: %s' %
                                  (name,ns_table.keys()))
 
         # print('type_pattern:',type_pattern)  # dbg
         search_result, namespaces_seen = set(), set()
         for ns_name in ns_search:
             ns = ns_table[ns_name]
             # Normally, locals and globals are the same, so we just check one.
             if id(ns) in namespaces_seen:
                 continue
             namespaces_seen.add(id(ns))
             tmp_res = list_namespace(ns, type_pattern, filter,
                                     ignore_case=ignore_case, show_all=show_all)
             search_result.update(tmp_res)
 
         page.page('\n'.join(sorted(search_result)))
 
 
 def _render_signature(obj_signature, obj_name) -> str:
     """
     This was mostly taken from inspect.Signature.__str__.
     Look there for the comments.
     The only change is to add linebreaks when this gets too long.
     """
     result = []
     pos_only = False
     kw_only = True
     for param in obj_signature.parameters.values():
         if param.kind == inspect.Parameter.POSITIONAL_ONLY:
             pos_only = True
         elif pos_only:
             result.append('/')
             pos_only = False
 
         if param.kind == inspect.Parameter.VAR_POSITIONAL:
             kw_only = False
         elif param.kind == inspect.Parameter.KEYWORD_ONLY and kw_only:
             result.append('*')
             kw_only = False
 
         result.append(str(param))
 
     if pos_only:
         result.append('/')
 
     # add up name, parameters, braces (2), and commas
     if len(obj_name) + sum(len(r) + 2 for r in result) > 75:
         # This doesn’t fit behind “Signature: ” in an inspect window.
         rendered = '{}(\n{})'.format(obj_name, ''.join(
             '    {},\n'.format(r) for r in result)
         )
     else:
         rendered = '{}({})'.format(obj_name, ', '.join(result))
 
     if obj_signature.return_annotation is not inspect._empty:
         anno = inspect.formatannotation(obj_signature.return_annotation)
         rendered += ' -> {}'.format(anno)
 
     return rendered

IPython/core/page.py

@@ -1,347 +1,348
 # encoding: utf-8
 """
 Paging capabilities for IPython.core
 
 Notes
 -----
 
 For now this uses IPython hooks, so it can't be in IPython.utils.  If we can get
 rid of that dependency, we could move it there.
 -----
 """
 
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 
 import os
 import io
 import re
 import sys
 import tempfile
 import subprocess
 
 from io import UnsupportedOperation
 from pathlib import Path
 
 from IPython import get_ipython
 from IPython.display import display
 from IPython.core.error import TryNext
 from IPython.utils.data import chop
 from IPython.utils.process import system
 from IPython.utils.terminal import get_terminal_size
 from IPython.utils import py3compat
 
 
 def display_page(strng, start=0, screen_lines=25):
     """Just display, no paging. screen_lines is ignored."""
     if isinstance(strng, dict):
         data = strng
     else:
         if start:
             strng = u'\n'.join(strng.splitlines()[start:])
         data = { 'text/plain': strng }
     display(data, raw=True)
 
 
 def as_hook(page_func):
     """Wrap a pager func to strip the `self` arg
 
     so it can be called as a hook.
     """
     return lambda self, *args, **kwargs: page_func(*args, **kwargs)
 
 
 esc_re = re.compile(r"(\x1b[^m]+m)")
 
 def page_dumb(strng, start=0, screen_lines=25):
     """Very dumb 'pager' in Python, for when nothing else works.
 
     Only moves forward, same interface as page(), except for pager_cmd and
     mode.
     """
     if isinstance(strng, dict):
         strng = strng.get('text/plain', '')
     out_ln  = strng.splitlines()[start:]
     screens = chop(out_ln,screen_lines-1)
     if len(screens) == 1:
         print(os.linesep.join(screens[0]))
     else:
         last_escape = ""
         for scr in screens[0:-1]:
             hunk = os.linesep.join(scr)
             print(last_escape + hunk)
             if not page_more():
                 return
             esc_list = esc_re.findall(hunk)
             if len(esc_list) > 0:
                 last_escape = esc_list[-1]
         print(last_escape + os.linesep.join(screens[-1]))
 
 def _detect_screen_size(screen_lines_def):
     """Attempt to work out the number of lines on the screen.
 
     This is called by page(). It can raise an error (e.g. when run in the
     test suite), so it's separated out so it can easily be called in a try block.
     """
     TERM = os.environ.get('TERM',None)
     if not((TERM=='xterm' or TERM=='xterm-color') and sys.platform != 'sunos5'):
         # curses causes problems on many terminals other than xterm, and
         # some termios calls lock up on Sun OS5.
         return screen_lines_def
     
     try:
         import termios
         import curses
     except ImportError:
         return screen_lines_def
     
     # There is a bug in curses, where *sometimes* it fails to properly
     # initialize, and then after the endwin() call is made, the
     # terminal is left in an unusable state.  Rather than trying to
     # check every time for this (by requesting and comparing termios
     # flags each time), we just save the initial terminal state and
     # unconditionally reset it every time.  It's cheaper than making
     # the checks.
     try:
         term_flags = termios.tcgetattr(sys.stdout)
     except termios.error as err:
         # can fail on Linux 2.6, pager_page will catch the TypeError
         raise TypeError('termios error: {0}'.format(err)) from err
 
     try:
         scr = curses.initscr()
     except AttributeError:
         # Curses on Solaris may not be complete, so we can't use it there
         return screen_lines_def
     
     screen_lines_real,screen_cols = scr.getmaxyx()
     curses.endwin()
 
     # Restore terminal state in case endwin() didn't.
     termios.tcsetattr(sys.stdout,termios.TCSANOW,term_flags)
     # Now we have what we needed: the screen size in rows/columns
     return screen_lines_real
     # print('***Screen size:',screen_lines_real,'lines x',
     #       screen_cols,'columns.')  # dbg
 
 def pager_page(strng, start=0, screen_lines=0, pager_cmd=None) -> None:
     """Display a string, piping through a pager after a certain length.
 
     strng can be a mime-bundle dict, supplying multiple representations,
     keyed by mime-type.
 
     The screen_lines parameter specifies the number of *usable* lines of your
     terminal screen (total lines minus lines you need to reserve to show other
     information).
 
     If you set screen_lines to a number <=0, page() will try to auto-determine
     your screen size and will only use up to (screen_size+screen_lines) for
     printing, paging after that. That is, if you want auto-detection but need
     to reserve the bottom 3 lines of the screen, use screen_lines = -3, and for
     auto-detection without any lines reserved simply use screen_lines = 0.
 
     If a string won't fit in the allowed lines, it is sent through the
     specified pager command. If none given, look for PAGER in the environment,
     and ultimately default to less.
 
     If no system pager works, the string is sent through a 'dumb pager'
     written in python, very simplistic.
     """
     
     # for compatibility with mime-bundle form:
     if isinstance(strng, dict):
         strng = strng['text/plain']
 
     # Ugly kludge, but calling curses.initscr() flat out crashes in emacs
     TERM = os.environ.get('TERM','dumb')
     if TERM in ['dumb','emacs'] and os.name != 'nt':
         print(strng)
         return
     # chop off the topmost part of the string we don't want to see
     str_lines = strng.splitlines()[start:]
     str_toprint = os.linesep.join(str_lines)
     num_newlines = len(str_lines)
     len_str = len(str_toprint)
 
     # Dumb heuristics to guesstimate number of on-screen lines the string
     # takes.  Very basic, but good enough for docstrings in reasonable
     # terminals. If someone later feels like refining it, it's not hard.
     numlines = max(num_newlines,int(len_str/80)+1)
 
     screen_lines_def = get_terminal_size()[1]
 
     # auto-determine screen size
     if screen_lines <= 0:
         try:
             screen_lines += _detect_screen_size(screen_lines_def)
         except (TypeError, UnsupportedOperation):
             print(str_toprint)
             return
 
     # print('numlines',numlines,'screenlines',screen_lines)  # dbg
     if numlines <= screen_lines :
         # print('*** normal print')  # dbg
         print(str_toprint)
     else:
         # Try to open pager and default to internal one if that fails.
         # All failure modes are tagged as 'retval=1', to match the return
         # value of a failed system command.  If any intermediate attempt
         # sets retval to 1, at the end we resort to our own page_dumb() pager.
         pager_cmd = get_pager_cmd(pager_cmd)
         pager_cmd += ' ' + get_pager_start(pager_cmd,start)
         if os.name == 'nt':
             if pager_cmd.startswith('type'):
                 # The default WinXP 'type' command is failing on complex strings.
                 retval = 1
             else:
                 fd, tmpname = tempfile.mkstemp('.txt')
                 tmppath = Path(tmpname)
                 try:
                     os.close(fd)
                     with tmppath.open("wt", encoding="utf-8") as tmpfile:
                         tmpfile.write(strng)
                         cmd = "%s < %s" % (pager_cmd, tmppath)
                     # tmpfile needs to be closed for windows
                     if os.system(cmd):
                         retval = 1
                     else:
                         retval = None
                 finally:
                     Path.unlink(tmppath)
         else:
             try:
                 retval = None
                 # Emulate os.popen, but redirect stderr
                 proc = subprocess.Popen(
                     pager_cmd,
                     shell=True,
                     stdin=subprocess.PIPE,
                     stderr=subprocess.DEVNULL,
                 )
                 pager = os._wrap_close(
                     io.TextIOWrapper(proc.stdin, encoding="utf-8"), proc
                 )
                 try:
+                    pager_encoding = pager.encoding or sys.stdout.encoding
                     pager.write(strng)
                 finally:
                     retval = pager.close()
             except IOError as msg:  # broken pipe when user quits
                 if msg.args == (32, 'Broken pipe'):
                     retval = None
                 else:
                     retval = 1
             except OSError:
                 # Other strange problems, sometimes seen in Win2k/cygwin
                 retval = 1
         if retval is not None:
             page_dumb(strng,screen_lines=screen_lines)
 
 
 def page(data, start: int = 0, screen_lines: int = 0, pager_cmd=None):
     """Display content in a pager, piping through a pager after a certain length.
 
     data can be a mime-bundle dict, supplying multiple representations,
     keyed by mime-type, or text.
 
     Pager is dispatched via the `show_in_pager` IPython hook.
     If no hook is registered, `pager_page` will be used.
     """
     # Some routines may auto-compute start offsets incorrectly and pass a
     # negative value.  Offset to 0 for robustness.
     start = max(0, start)
 
     # first, try the hook
     ip = get_ipython()
     if ip:
         try:
             ip.hooks.show_in_pager(data, start=start, screen_lines=screen_lines)
             return
         except TryNext:
             pass
     
     # fallback on default pager
     return pager_page(data, start, screen_lines, pager_cmd)
 
 
 def page_file(fname, start=0, pager_cmd=None):
     """Page a file, using an optional pager command and starting line.
     """
 
     pager_cmd = get_pager_cmd(pager_cmd)
     pager_cmd += ' ' + get_pager_start(pager_cmd,start)
 
     try:
         if os.environ['TERM'] in ['emacs','dumb']:
             raise EnvironmentError
         system(pager_cmd + ' ' + fname)
     except:
         try:
             if start > 0:
                 start -= 1
             page(open(fname, encoding="utf-8").read(), start)
         except:
             print('Unable to show file',repr(fname))
 
 
 def get_pager_cmd(pager_cmd=None):
     """Return a pager command.
 
     Makes some attempts at finding an OS-correct one.
     """
     if os.name == 'posix':
         default_pager_cmd = 'less -R'  # -R for color control sequences
     elif os.name in ['nt','dos']:
         default_pager_cmd = 'type'
 
     if pager_cmd is None:
         try:
             pager_cmd = os.environ['PAGER']
         except:
             pager_cmd = default_pager_cmd
     
     if pager_cmd == 'less' and '-r' not in os.environ.get('LESS', '').lower():
         pager_cmd += ' -R'
     
     return pager_cmd
 
 
 def get_pager_start(pager, start):
     """Return the string for paging files with an offset.
 
     This is the '+N' argument which less and more (under Unix) accept.
     """
 
     if pager in ['less','more']:
         if start:
             start_string = '+' + str(start)
         else:
             start_string = ''
     else:
         start_string = ''
     return start_string
 
 
 # (X)emacs on win32 doesn't like to be bypassed with msvcrt.getch()
 if os.name == 'nt' and os.environ.get('TERM','dumb') != 'emacs':
     import msvcrt
     def page_more():
         """ Smart pausing between pages
 
         @return:    True if need print more lines, False if quit
         """
         sys.stdout.write('---Return to continue, q to quit--- ')
         ans = msvcrt.getwch()
         if ans in ("q", "Q"):
             result = False
         else:
             result = True
         sys.stdout.write("\b"*37 + " "*37 + "\b"*37)
         return result
 else:
     def page_more():
         ans = py3compat.input('---Return to continue, q to quit--- ')
         if ans.lower().startswith('q'):
             return False
         else:
             return True

IPython/core/payloadpage.py

@@ -1,40 +1,41
 """A payload based version of page."""
 
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
+import warnings
 from IPython.core.getipython import get_ipython
 
 # see https://github.com/ipython/ipykernel/issues/1304
 # this should be moved to ipykernel and removed in the long run.
 
 
 def page(strng, start=0, screen_lines=0, pager_cmd=None):
     """Print a string, piping through a pager.
 
     This version ignores the screen_lines and pager_cmd arguments and uses
     IPython's payload system instead.
 
     Parameters
     ----------
     strng : str or mime-dict
         Text to page, or a mime-type keyed dict of already formatted data.
     start : int
         Starting line at which to place the display.
     """
 
     # Some routines may auto-compute start offsets incorrectly and pass a
     # negative value.  Offset to 0 for robustness.
     start = max(0, start)
     shell = get_ipython()
     
     if isinstance(strng, dict):
         data = strng
     else:
         data = {'text/plain' : strng}
     payload = dict(
         source='page',
         data=data,
         start=start,
         )
     shell.payload_manager.write_payload(payload)

IPython/core/tests/refbug.py

@@ -1,46 +1,46
 """Minimal script to reproduce our nasty reference counting bug.
 
 The problem is related to https://github.com/ipython/ipython/issues/141
 
 The original fix for that appeared to work, but John D. Hunter found a
 matplotlib example which, when run twice in a row, would break.  The problem
 were references held by open figures to internals of Tkinter.
 
 This code reproduces the problem that John saw, without matplotlib.
 
 This script is meant to be called by other parts of the test suite that call it
 via %run as if it were executed interactively by the user. As of 2011-05-29,
 test_run.py calls it.
 """
 
 #-----------------------------------------------------------------------------
 # Module imports
 #-----------------------------------------------------------------------------
 
 from IPython import get_ipython
 
 #-----------------------------------------------------------------------------
 # Globals
 #-----------------------------------------------------------------------------
 
 # This needs to be here because nose and other test runners will import
 # this module. Importing this module has potential side effects that we
 # want to prevent.
 if __name__ == '__main__':
 
     ip = get_ipython()
 
-    if "_refbug_cache" not in ip.user_ns:
-        ip.user_ns["_refbug_cache"] = []
+    if not '_refbug_cache' in ip.user_ns:
+        ip.user_ns['_refbug_cache'] = []
 
 
     aglobal = 'Hello'
     def f():
         return aglobal
 
     cache = ip.user_ns['_refbug_cache']
     cache.append(f)
 
     def call_f():
         for func in cache:
             print('lowercased:',func().lower())

IPython/core/tests/test_exceptiongroup_tb.py

@@ -1,103 +1,108
+import unittest
+import re
 from IPython.utils.capture import capture_output
+import sys
 import pytest
+from tempfile import TemporaryDirectory
+from IPython.testing import tools as tt
 
 
 def _exceptiongroup_common(
     outer_chain: str,
     inner_chain: str,
     native: bool,
 ) -> None:
     pre_raise = "exceptiongroup." if not native else ""
     filestr = f"""
     {"import exceptiongroup" if not native else ""}
     import pytest
 
     def f(): raise ValueError("From f()")
     def g(): raise BaseException("From g()")
 
     def inner(inner_chain):
         excs = []
         for callback in [f, g]:
             try:
                 callback()
             except BaseException as err:
                 excs.append(err)
         if excs:
             if inner_chain == "none":
                 raise {pre_raise}BaseExceptionGroup("Oops", excs)
             try:
                 raise SyntaxError()
             except SyntaxError as e:
                 if inner_chain == "from":
                     raise {pre_raise}BaseExceptionGroup("Oops", excs) from e
                 else:
                     raise {pre_raise}BaseExceptionGroup("Oops", excs)
 
     def outer(outer_chain, inner_chain):
         try:
             inner(inner_chain)
         except BaseExceptionGroup as e:
             if outer_chain == "none":
                 raise
             if outer_chain == "from":
                 raise IndexError() from e
             else:
                 raise IndexError
 
 
     outer("{outer_chain}", "{inner_chain}")
     """
     with capture_output() as cap:
         ip.run_cell(filestr)
 
     match_lines = []
     if inner_chain == "another":
         match_lines += [
             "During handling of the above exception, another exception occurred:",
         ]
     elif inner_chain == "from":
         match_lines += [
             "The above exception was the direct cause of the following exception:",
         ]
 
     match_lines += [
         "  + Exception Group Traceback (most recent call last):",
         "  | BaseExceptionGroup: Oops (2 sub-exceptions)",
         "    | ValueError: From f()",
         "    | BaseException: From g()",
     ]
 
     if outer_chain == "another":
         match_lines += [
             "During handling of the above exception, another exception occurred:",
             "IndexError",
         ]
     elif outer_chain == "from":
         match_lines += [
             "The above exception was the direct cause of the following exception:",
             "IndexError",
         ]
 
     error_lines = cap.stderr.split("\n")
 
     err_index = match_index = 0
     for expected in match_lines:
         for i, actual in enumerate(error_lines):
             if actual == expected:
                 error_lines = error_lines[i + 1 :]
                 break
         else:
             assert False, f"{expected} not found in cap.stderr"
 
 
 @pytest.mark.parametrize("outer_chain", ["none", "from", "another"])
 @pytest.mark.parametrize("inner_chain", ["none", "from", "another"])
 def test_native_exceptiongroup(outer_chain, inner_chain) -> None:
     _exceptiongroup_common(outer_chain, inner_chain, native=True)
 
 
 @pytest.mark.parametrize("outer_chain", ["none", "from", "another"])
 @pytest.mark.parametrize("inner_chain", ["none", "from", "another"])
 def test_native_exceptiongroup(outer_chain, inner_chain) -> None:
     pytest.importorskip("exceptiongroup")
     _exceptiongroup_common(outer_chain, inner_chain, native=False)

IPython/core/tests/test_formatters.py

@@ -1,544 +1,545
 """Tests for the Formatters."""
 
 from math import pi
 
 try:
     import numpy
 except:
     numpy = None
 import pytest
 
 from IPython import get_ipython
 from traitlets.config import Config
 from IPython.core.formatters import (
     PlainTextFormatter, HTMLFormatter, PDFFormatter, _mod_name_key,
+    DisplayFormatter, JSONFormatter,
 )
 from IPython.utils.io import capture_output
 
 class A(object):
     def __repr__(self):
         return 'A()'
 
 class B(A):
     def __repr__(self):
         return 'B()'
 
 class C:
     pass
 
 class BadRepr(object):
     def __repr__(self):
         raise ValueError("bad repr")
 
 class BadPretty(object):
     _repr_pretty_ = None
 
 class GoodPretty(object):
     def _repr_pretty_(self, pp, cycle):
         pp.text('foo')
 
     def __repr__(self):
         return 'GoodPretty()'
 
 def foo_printer(obj, pp, cycle):
     pp.text('foo')
 
 def test_pretty():
     f = PlainTextFormatter()
     f.for_type(A, foo_printer)
     assert f(A()) == "foo"
     assert f(B()) == "B()"
     assert f(GoodPretty()) == "foo"
     # Just don't raise an exception for the following:
     f(BadPretty())
 
     f.pprint = False
     assert f(A()) == "A()"
     assert f(B()) == "B()"
     assert f(GoodPretty()) == "GoodPretty()"
 
 
 def test_deferred():
     f = PlainTextFormatter()
 
 def test_precision():
     """test various values for float_precision."""
     f = PlainTextFormatter()
     assert f(pi) == repr(pi)
     f.float_precision = 0
     if numpy:
         po = numpy.get_printoptions()
         assert po["precision"] == 0
     assert f(pi) == "3"
     f.float_precision = 2
     if numpy:
         po = numpy.get_printoptions()
         assert po["precision"] == 2
     assert f(pi) == "3.14"
     f.float_precision = "%g"
     if numpy:
         po = numpy.get_printoptions()
         assert po["precision"] == 2
     assert f(pi) == "3.14159"
     f.float_precision = "%e"
     assert f(pi) == "3.141593e+00"
     f.float_precision = ""
     if numpy:
         po = numpy.get_printoptions()
         assert po["precision"] == 8
     assert f(pi) == repr(pi)
 
 
 def test_bad_precision():
     """test various invalid values for float_precision."""
     f = PlainTextFormatter()
     def set_fp(p):
         f.float_precision = p
 
     pytest.raises(ValueError, set_fp, "%")
     pytest.raises(ValueError, set_fp, "%.3f%i")
     pytest.raises(ValueError, set_fp, "foo")
     pytest.raises(ValueError, set_fp, -1)
 
 def test_for_type():
     f = PlainTextFormatter()
 
     # initial return, None
     assert f.for_type(C, foo_printer) is None
     # no func queries
     assert f.for_type(C) is foo_printer
     # shouldn't change anything
     assert f.for_type(C) is foo_printer
     # None should do the same
     assert f.for_type(C, None) is foo_printer
     assert f.for_type(C, None) is foo_printer
 
 def test_for_type_string():
     f = PlainTextFormatter()
 
     type_str = '%s.%s' % (C.__module__, 'C')
 
     # initial return, None
     assert f.for_type(type_str, foo_printer) is None
     # no func queries
     assert f.for_type(type_str) is foo_printer
     assert _mod_name_key(C) in f.deferred_printers
     assert f.for_type(C) is foo_printer
     assert _mod_name_key(C) not in f.deferred_printers
     assert C in f.type_printers
 
 def test_for_type_by_name():
     f = PlainTextFormatter()
 
     mod = C.__module__
 
     # initial return, None
     assert f.for_type_by_name(mod, "C", foo_printer) is None
     # no func queries
     assert f.for_type_by_name(mod, "C") is foo_printer
     # shouldn't change anything
     assert f.for_type_by_name(mod, "C") is foo_printer
     # None should do the same
     assert f.for_type_by_name(mod, "C", None) is foo_printer
     assert f.for_type_by_name(mod, "C", None) is foo_printer
 
 
 def test_lookup():
     f = PlainTextFormatter()
 
     f.for_type(C, foo_printer)
     assert f.lookup(C()) is foo_printer
     with pytest.raises(KeyError):
         f.lookup(A())
 
 def test_lookup_string():
     f = PlainTextFormatter()
     type_str = '%s.%s' % (C.__module__, 'C')
 
     f.for_type(type_str, foo_printer)
     assert f.lookup(C()) is foo_printer
     # should move from deferred to imported dict
     assert _mod_name_key(C) not in f.deferred_printers
     assert C in f.type_printers
 
 def test_lookup_by_type():
     f = PlainTextFormatter()
     f.for_type(C, foo_printer)
     assert f.lookup_by_type(C) is foo_printer
     with pytest.raises(KeyError):
         f.lookup_by_type(A)
 
 def test_lookup_by_type_string():
     f = PlainTextFormatter()
     type_str = '%s.%s' % (C.__module__, 'C')
     f.for_type(type_str, foo_printer)
 
     # verify insertion
     assert _mod_name_key(C) in f.deferred_printers
     assert C not in f.type_printers
 
     assert f.lookup_by_type(type_str) is foo_printer
     # lookup by string doesn't cause import
     assert _mod_name_key(C) in f.deferred_printers
     assert C not in f.type_printers
 
     assert f.lookup_by_type(C) is foo_printer
     # should move from deferred to imported dict
     assert _mod_name_key(C) not in f.deferred_printers
     assert C in f.type_printers
 
 def test_in_formatter():
     f = PlainTextFormatter()
     f.for_type(C, foo_printer)
     type_str = '%s.%s' % (C.__module__, 'C')
     assert C in f
     assert type_str in f
 
 def test_string_in_formatter():
     f = PlainTextFormatter()
     type_str = '%s.%s' % (C.__module__, 'C')
     f.for_type(type_str, foo_printer)
     assert type_str in f
     assert C in f
 
 def test_pop():
     f = PlainTextFormatter()
     f.for_type(C, foo_printer)
     assert f.lookup_by_type(C) is foo_printer
     assert f.pop(C, None) is foo_printer
     f.for_type(C, foo_printer)
     assert f.pop(C) is foo_printer
     with pytest.raises(KeyError):
         f.lookup_by_type(C)
     with pytest.raises(KeyError):
         f.pop(C)
     with pytest.raises(KeyError):
         f.pop(A)
     assert f.pop(A, None) is None
 
 def test_pop_string():
     f = PlainTextFormatter()
     type_str = '%s.%s' % (C.__module__, 'C')
 
     with pytest.raises(KeyError):
         f.pop(type_str)
 
     f.for_type(type_str, foo_printer)
     f.pop(type_str)
     with pytest.raises(KeyError):
         f.lookup_by_type(C)
     with pytest.raises(KeyError):
         f.pop(type_str)
 
     f.for_type(C, foo_printer)
     assert f.pop(type_str, None) is foo_printer
     with pytest.raises(KeyError):
         f.lookup_by_type(C)
     with pytest.raises(KeyError):
         f.pop(type_str)
     assert f.pop(type_str, None) is None
 
 
 def test_error_method():
     f = HTMLFormatter()
     class BadHTML(object):
         def _repr_html_(self):
             raise ValueError("Bad HTML")
     bad = BadHTML()
     with capture_output() as captured:
         result = f(bad)
     assert result is None
     assert "Traceback" in captured.stdout
     assert "Bad HTML" in captured.stdout
     assert "_repr_html_" in captured.stdout
 
 def test_nowarn_notimplemented():
     f = HTMLFormatter()
     class HTMLNotImplemented(object):
         def _repr_html_(self):
             raise NotImplementedError
     h = HTMLNotImplemented()
     with capture_output() as captured:
         result = f(h)
     assert result is None
     assert "" == captured.stderr
     assert "" == captured.stdout
 
 
 def test_warn_error_for_type():
     f = HTMLFormatter()
     f.for_type(int, lambda i: name_error)
     with capture_output() as captured:
         result = f(5)
     assert result is None
     assert "Traceback" in captured.stdout
     assert "NameError" in captured.stdout
     assert "name_error" in captured.stdout
 
 def test_error_pretty_method():
     f = PlainTextFormatter()
     class BadPretty(object):
         def _repr_pretty_(self):
             return "hello"
     bad = BadPretty()
     with capture_output() as captured:
         result = f(bad)
     assert result is None
     assert "Traceback" in captured.stdout
     assert "_repr_pretty_" in captured.stdout
     assert "given" in captured.stdout
     assert "argument" in captured.stdout
 
 
 def test_bad_repr_traceback():
     f = PlainTextFormatter()
     bad = BadRepr()
     with capture_output() as captured:
         result = f(bad)
     # catches error, returns None
     assert result is None
     assert "Traceback" in captured.stdout
     assert "__repr__" in captured.stdout
     assert "ValueError" in captured.stdout
 
 
 class MakePDF(object):
     def _repr_pdf_(self):
         return 'PDF'
 
 def test_pdf_formatter():
     pdf = MakePDF()
     f = PDFFormatter()
     assert f(pdf) == "PDF"
 
 
 def test_print_method_bound():
     f = HTMLFormatter()
     class MyHTML(object):
         def _repr_html_(self):
             return "hello"
     with capture_output() as captured:
         result = f(MyHTML)
     assert result is None
     assert "FormatterWarning" not in captured.stderr
 
     with capture_output() as captured:
         result = f(MyHTML())
     assert result == "hello"
     assert captured.stderr == ""
 
 
 def test_print_method_weird():
 
     class TextMagicHat(object):
         def __getattr__(self, key):
             return key
 
     f = HTMLFormatter()
 
     text_hat = TextMagicHat()
     assert text_hat._repr_html_ == "_repr_html_"
     with capture_output() as captured:
         result = f(text_hat)
 
     assert result is None
     assert "FormatterWarning" not in captured.stderr
 
     class CallableMagicHat(object):
         def __getattr__(self, key):
             return lambda : key
 
     call_hat = CallableMagicHat()
     with capture_output() as captured:
         result = f(call_hat)
 
     assert result is None
 
     class BadReprArgs(object):
         def _repr_html_(self, extra, args):
             return "html"
 
     bad = BadReprArgs()
     with capture_output() as captured:
         result = f(bad)
 
     assert result is None
     assert "FormatterWarning" not in captured.stderr
 
 
 def test_format_config():
     """config objects don't pretend to support fancy reprs with lazy attrs"""
     f = HTMLFormatter()
     cfg = Config()
     with capture_output() as captured:
         result = f(cfg)
     assert result is None
     assert captured.stderr == ""
 
     with capture_output() as captured:
         result = f(Config)
     assert result is None
     assert captured.stderr == ""
 
 
 def test_pretty_max_seq_length():
     f = PlainTextFormatter(max_seq_length=1)
     lis = list(range(3))
     text = f(lis)
     assert text == "[0, ...]"
     f.max_seq_length = 0
     text = f(lis)
     assert text == "[0, 1, 2]"
     text = f(list(range(1024)))
     lines = text.splitlines()
     assert len(lines) == 1024
 
 
 def test_ipython_display_formatter():
     """Objects with _ipython_display_ defined bypass other formatters"""
     f = get_ipython().display_formatter
     catcher = []
     class SelfDisplaying(object):
         def _ipython_display_(self):
             catcher.append(self)
 
     class NotSelfDisplaying(object):
         def __repr__(self):
             return "NotSelfDisplaying"
 
         def _ipython_display_(self):
             raise NotImplementedError
 
     save_enabled = f.ipython_display_formatter.enabled
     f.ipython_display_formatter.enabled = True
 
     yes = SelfDisplaying()
     no = NotSelfDisplaying()
 
     d, md = f.format(no)
     assert d == {"text/plain": repr(no)}
     assert md == {}
     assert catcher == []
 
     d, md = f.format(yes)
     assert d == {}
     assert md == {}
     assert catcher == [yes]
 
     f.ipython_display_formatter.enabled = save_enabled
 
 
 def test_repr_mime():
     class HasReprMime(object):
         def _repr_mimebundle_(self, include=None, exclude=None):
             return {
                 'application/json+test.v2': {
                     'x': 'y'
                 },
                 'plain/text' : '<HasReprMime>',
                 'image/png' : 'i-overwrite'
             }
 
         def _repr_png_(self):
             return 'should-be-overwritten'
         def _repr_html_(self):
             return '<b>hi!</b>'
 
     f = get_ipython().display_formatter
     html_f = f.formatters['text/html']
     save_enabled = html_f.enabled
     html_f.enabled = True
     obj = HasReprMime()
     d, md = f.format(obj)
     html_f.enabled = save_enabled
 
     assert sorted(d) == [
         "application/json+test.v2",
         "image/png",
         "plain/text",
         "text/html",
         "text/plain",
     ]
     assert md == {}
 
     d, md = f.format(obj, include={"image/png"})
     assert list(d.keys()) == [
         "image/png"
     ], "Include should filter out even things from repr_mimebundle"
 
     assert d["image/png"] == "i-overwrite", "_repr_mimebundle_ take precedence"
 
 
 def test_pass_correct_include_exclude():
     class Tester(object):
 
         def __init__(self, include=None, exclude=None):
             self.include = include
             self.exclude = exclude
 
         def _repr_mimebundle_(self, include, exclude, **kwargs):
             if include and (include != self.include):
                 raise ValueError('include got modified: display() may be broken.')
             if exclude and (exclude != self.exclude):
                 raise ValueError('exclude got modified: display() may be broken.')
 
             return None
 
     include = {'a', 'b', 'c'}
     exclude = {'c', 'e' , 'f'}
 
     f = get_ipython().display_formatter
     f.format(Tester(include=include, exclude=exclude), include=include, exclude=exclude)
     f.format(Tester(exclude=exclude), exclude=exclude)
     f.format(Tester(include=include), include=include)
 
 
 def test_repr_mime_meta():
     class HasReprMimeMeta(object):
         def _repr_mimebundle_(self, include=None, exclude=None):
             data = {
                 'image/png': 'base64-image-data',
             }
             metadata = {
                 'image/png': {
                     'width': 5,
                     'height': 10,
                 }
             }
             return (data, metadata)
 
     f = get_ipython().display_formatter
     obj = HasReprMimeMeta()
     d, md = f.format(obj)
     assert sorted(d) == ["image/png", "text/plain"]
     assert md == {
         "image/png": {
             "width": 5,
             "height": 10,
         }
     }
 
 
 def test_repr_mime_failure():
     class BadReprMime(object):
         def _repr_mimebundle_(self, include=None, exclude=None):
             raise RuntimeError
 
     f = get_ipython().display_formatter
     obj = BadReprMime()
     d, md = f.format(obj)
     assert "text/plain" in d
 
 
 def test_custom_repr_namedtuple_partialmethod():
     from functools import partialmethod
     from typing import NamedTuple
 
     class Foo(NamedTuple): ...
 
     Foo.__repr__ = partialmethod(lambda obj: "Hello World")
     foo = Foo()
 
     f = PlainTextFormatter()
     assert f.pprint
     assert f(foo) == "Hello World"

IPython/core/tests/test_handlers.py

@@ -1,74 +1,75
 """Tests for input handlers.
 """
 #-----------------------------------------------------------------------------
 # Module imports
 #-----------------------------------------------------------------------------
 
 # our own packages
 from IPython.core import autocall
+from IPython.testing import tools as tt
 import pytest
 from collections.abc import Callable
 
 #-----------------------------------------------------------------------------
 # Globals
 #-----------------------------------------------------------------------------
 
 # Test functions
 #-----------------------------------------------------------------------------
 
 class CallableIndexable(object):
     def __getitem__(self, idx): return True
     def __call__(self, *args, **kws): return True
 
 
 class Autocallable(autocall.IPyAutocall):
     def __call__(self):
         return "called"
 
 
 @pytest.mark.parametrize(
     "autocall, input, output",
     [
         # For many of the below, we're also checking that leading whitespace
         # turns off the esc char, which it should unless there is a continuation
         # line.
         ("1", '"no change"', '"no change"'),  # normal
         ("1", "lsmagic", "get_ipython().run_line_magic('lsmagic', '')"),  # magic
         # Only explicit escapes or instances of IPyAutocallable should get
         # expanded
         ("0", 'len "abc"', 'len "abc"'),
         ("0", "autocallable", "autocallable()"),
         # Don't add extra brackets (gh-1117)
         ("0", "autocallable()", "autocallable()"),
         ("1", 'len "abc"', 'len("abc")'),
         ("1", 'len "abc";', 'len("abc");'),  # ; is special -- moves out of parens
         # Autocall is turned off if first arg is [] and the object
         # is both callable and indexable.  Like so:
         ("1", "len [1,2]", "len([1,2])"),  # len doesn't support __getitem__...
         ("1", "call_idx [1]", "call_idx [1]"),  # call_idx *does*..
         ("1", "call_idx 1", "call_idx(1)"),
         ("1", "len", "len"),  # only at 2 does it auto-call on single args
         ("2", 'len "abc"', 'len("abc")'),
         ("2", 'len "abc";', 'len("abc");'),
         ("2", "len [1,2]", "len([1,2])"),
         ("2", "call_idx [1]", "call_idx [1]"),
         ("2", "call_idx 1", "call_idx(1)"),
         # T his is what's different:
         ("2", "len", "len()"),  # only at 2 does it auto-call on single args
         ("0", "Callable[[int], None]", "Callable[[int], None]"),
         ("1", "Callable[[int], None]", "Callable[[int], None]"),
         ("1", "Callable[[int], None]", "Callable[[int], None]"),
     ],
 )
 def test_handlers_I(autocall, input, output):
     autocallable = Autocallable()
     ip.user_ns["autocallable"] = autocallable
 
     call_idx = CallableIndexable()
     ip.user_ns["call_idx"] = call_idx
 
     ip.user_ns["Callable"] = Callable
 
     ip.run_line_magic("autocall", autocall)
     assert ip.prefilter_manager.prefilter_lines(input) == output
     ip.run_line_magic("autocall", "1")

IPython/core/tests/test_interactiveshell.py

@@ -1,1220 +1,1221
 # -*- coding: utf-8 -*-
 """Tests for the key interactiveshell module.
 
 Historically the main classes in interactiveshell have been under-tested.  This
 module should grow as many single-method tests as possible to trap many of the
 recurring bugs we seem to encounter with high-level interaction.
 """
 
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 import asyncio
 import ast
 import os
 import signal
 import shutil
 import sys
 import tempfile
 import unittest
 import pytest
 from unittest import mock
 
 from os.path import join
 
 from IPython.core.error import InputRejected
+from IPython.core.inputtransformer import InputTransformer
 from IPython.core import interactiveshell
 from IPython.core.oinspect import OInfo
 from IPython.testing.decorators import (
     skipif,
     skip_win32,
     onlyif_unicode_paths,
     onlyif_cmds_exist,
     skip_if_not_osx,
 )
 from IPython.testing import tools as tt
 from IPython.utils.process import find_cmd
 
 #-----------------------------------------------------------------------------
 # Globals
 #-----------------------------------------------------------------------------
 # This is used by every single test, no point repeating it ad nauseam
 
 #-----------------------------------------------------------------------------
 # Tests
 #-----------------------------------------------------------------------------
 
 class DerivedInterrupt(KeyboardInterrupt):
     pass
 
 class InteractiveShellTestCase(unittest.TestCase):
     def test_naked_string_cells(self):
         """Test that cells with only naked strings are fully executed"""
         # First, single-line inputs
         ip.run_cell('"a"\n')
         self.assertEqual(ip.user_ns['_'], 'a')
         # And also multi-line cells
         ip.run_cell('"""a\nb"""\n')
         self.assertEqual(ip.user_ns['_'], 'a\nb')
 
     def test_run_empty_cell(self):
         """Just make sure we don't get a horrible error with a blank
         cell of input. Yes, I did overlook that."""
         old_xc = ip.execution_count
         res = ip.run_cell('')
         self.assertEqual(ip.execution_count, old_xc)
         self.assertEqual(res.execution_count, None)
 
     def test_run_cell_multiline(self):
         """Multi-block, multi-line cells must execute correctly.
         """
         src = '\n'.join(["x=1",
                          "y=2",
                          "if 1:",
                          "    x += 1",
                          "    y += 1",])
         res = ip.run_cell(src)
         self.assertEqual(ip.user_ns['x'], 2)
         self.assertEqual(ip.user_ns['y'], 3)
         self.assertEqual(res.success, True)
         self.assertEqual(res.result, None)
 
     def test_multiline_string_cells(self):
         "Code sprinkled with multiline strings should execute (GH-306)"
         ip.run_cell('tmp=0')
         self.assertEqual(ip.user_ns['tmp'], 0)
         res = ip.run_cell('tmp=1;"""a\nb"""\n')
         self.assertEqual(ip.user_ns['tmp'], 1)
         self.assertEqual(res.success, True)
         self.assertEqual(res.result, "a\nb")
 
     def test_dont_cache_with_semicolon(self):
         "Ending a line with semicolon should not cache the returned object (GH-307)"
         oldlen = len(ip.user_ns['Out'])
         for cell in ['1;', '1;1;']:
             res = ip.run_cell(cell, store_history=True)
             newlen = len(ip.user_ns['Out'])
             self.assertEqual(oldlen, newlen)
             self.assertIsNone(res.result)
         i = 0
         #also test the default caching behavior
         for cell in ['1', '1;1']:
             ip.run_cell(cell, store_history=True)
             newlen = len(ip.user_ns['Out'])
             i += 1
             self.assertEqual(oldlen+i, newlen)
 
     def test_syntax_error(self):
         res = ip.run_cell("raise = 3")
         self.assertIsInstance(res.error_before_exec, SyntaxError)
 
     def test_open_standard_input_stream(self):
         res = ip.run_cell("open(0)")
         self.assertIsInstance(res.error_in_exec, ValueError)
 
     def test_open_standard_output_stream(self):
         res = ip.run_cell("open(1)")
         self.assertIsInstance(res.error_in_exec, ValueError)
 
     def test_open_standard_error_stream(self):
         res = ip.run_cell("open(2)")
         self.assertIsInstance(res.error_in_exec, ValueError)
 
     def test_In_variable(self):
         "Verify that In variable grows with user input (GH-284)"
         oldlen = len(ip.user_ns['In'])
         ip.run_cell('1;', store_history=True)
         newlen = len(ip.user_ns['In'])
         self.assertEqual(oldlen+1, newlen)
         self.assertEqual(ip.user_ns['In'][-1],'1;')
 
     def test_magic_names_in_string(self):
         ip.run_cell('a = """\n%exit\n"""')
         self.assertEqual(ip.user_ns['a'], '\n%exit\n')
 
     def test_trailing_newline(self):
         """test that running !(command) does not raise a SyntaxError"""
         ip.run_cell('!(true)\n', False)
         ip.run_cell('!(true)\n\n\n', False)
 
     def test_gh_597(self):
         """Pretty-printing lists of objects with non-ascii reprs may cause
         problems."""
         class Spam(object):
             def __repr__(self):
                 return "\xe9"*50
         import IPython.core.formatters
         f = IPython.core.formatters.PlainTextFormatter()
         f([Spam(), Spam()])
 
     def test_future_flags(self):
         """Check that future flags are used for parsing code (gh-777)"""
         ip.run_cell('from __future__ import barry_as_FLUFL')
         try:
             ip.run_cell('prfunc_return_val = 1 <> 2')
             assert 'prfunc_return_val' in ip.user_ns
         finally:
             # Reset compiler flags so we don't mess up other tests.
             ip.compile.reset_compiler_flags()
 
     def test_can_pickle(self):
         "Can we pickle objects defined interactively (GH-29)"
         ip = get_ipython()
         ip.reset()
         ip.run_cell(("class Mylist(list):\n"
                      "    def __init__(self,x=[]):\n"
                      "        list.__init__(self,x)"))
         ip.run_cell("w=Mylist([1,2,3])")
 
         from pickle import dumps
 
         # We need to swap in our main module - this is only necessary
         # inside the test framework, because IPython puts the interactive module
         # in place (but the test framework undoes this).
         _main = sys.modules['__main__']
         sys.modules['__main__'] = ip.user_module
         try:
             res = dumps(ip.user_ns["w"])
         finally:
             sys.modules['__main__'] = _main
         self.assertTrue(isinstance(res, bytes))
 
     def test_global_ns(self):
         "Code in functions must be able to access variables outside them."
         ip = get_ipython()
         ip.run_cell("a = 10")
         ip.run_cell(("def f(x):\n"
                      "    return x + a"))
         ip.run_cell("b = f(12)")
         self.assertEqual(ip.user_ns["b"], 22)
 
     def test_bad_custom_tb(self):
         """Check that InteractiveShell is protected from bad custom exception handlers"""
         ip.set_custom_exc((IOError,), lambda etype,value,tb: 1/0)
         self.assertEqual(ip.custom_exceptions, (IOError,))
         with tt.AssertPrints("Custom TB Handler failed", channel='stderr'):
             ip.run_cell(u'raise IOError("foo")')
         self.assertEqual(ip.custom_exceptions, ())
 
     def test_bad_custom_tb_return(self):
         """Check that InteractiveShell is protected from bad return types in custom exception handlers"""
         ip.set_custom_exc((NameError,),lambda etype,value,tb, tb_offset=None: 1)
         self.assertEqual(ip.custom_exceptions, (NameError,))
         with tt.AssertPrints("Custom TB Handler failed", channel='stderr'):
             ip.run_cell(u'a=abracadabra')
         self.assertEqual(ip.custom_exceptions, ())
 
     def test_drop_by_id(self):
         myvars = {"a":object(), "b":object(), "c": object()}
         ip.push(myvars, interactive=False)
         for name in myvars:
             assert name in ip.user_ns, name
             assert name in ip.user_ns_hidden, name
         ip.user_ns['b'] = 12
         ip.drop_by_id(myvars)
         for name in ["a", "c"]:
             assert name not in ip.user_ns, name
             assert name not in ip.user_ns_hidden, name
         assert ip.user_ns['b'] == 12
         ip.reset()
 
     def test_var_expand(self):
         ip.user_ns['f'] = u'Ca\xf1o'
         self.assertEqual(ip.var_expand(u'echo $f'), u'echo Ca\xf1o')
         self.assertEqual(ip.var_expand(u'echo {f}'), u'echo Ca\xf1o')
         self.assertEqual(ip.var_expand(u'echo {f[:-1]}'), u'echo Ca\xf1')
         self.assertEqual(ip.var_expand(u'echo {1*2}'), u'echo 2')
 
         self.assertEqual(ip.var_expand(u"grep x | awk '{print $1}'"), u"grep x | awk '{print $1}'")
 
         ip.user_ns['f'] = b'Ca\xc3\xb1o'
         # This should not raise any exception:
         ip.var_expand(u'echo $f')
 
     def test_var_expand_local(self):
         """Test local variable expansion in !system and %magic calls"""
         # !system
         ip.run_cell(
             "def test():\n"
             '    lvar = "ttt"\n'
             "    ret = !echo {lvar}\n"
             "    return ret[0]\n"
         )
         res = ip.user_ns["test"]()
         self.assertIn("ttt", res)
 
         # %magic
         ip.run_cell(
             "def makemacro():\n"
             '    macroname = "macro_var_expand_locals"\n'
             "    %macro {macroname} codestr\n"
         )
         ip.user_ns["codestr"] = "str(12)"
         ip.run_cell("makemacro()")
         self.assertIn("macro_var_expand_locals", ip.user_ns)
 
     def test_var_expand_self(self):
         """Test variable expansion with the name 'self', which was failing.
 
         See https://github.com/ipython/ipython/issues/1878#issuecomment-7698218
         """
         ip.run_cell(
             "class cTest:\n"
             '  classvar="see me"\n'
             "  def test(self):\n"
             "    res = !echo Variable: {self.classvar}\n"
             "    return res[0]\n"
         )
         self.assertIn("see me", ip.user_ns["cTest"]().test())
 
     def test_bad_var_expand(self):
         """var_expand on invalid formats shouldn't raise"""
         # SyntaxError
         self.assertEqual(ip.var_expand(u"{'a':5}"), u"{'a':5}")
         # NameError
         self.assertEqual(ip.var_expand(u"{asdf}"), u"{asdf}")
         # ZeroDivisionError
         self.assertEqual(ip.var_expand(u"{1/0}"), u"{1/0}")
 
     def test_silent_postexec(self):
         """run_cell(silent=True) doesn't invoke pre/post_run_cell callbacks"""
         pre_explicit = mock.Mock()
         pre_always = mock.Mock()
         post_explicit = mock.Mock()
         post_always = mock.Mock()
         all_mocks = [pre_explicit, pre_always, post_explicit, post_always]
 
         ip.events.register('pre_run_cell', pre_explicit)
         ip.events.register('pre_execute', pre_always)
         ip.events.register('post_run_cell', post_explicit)
         ip.events.register('post_execute', post_always)
 
         try:
             ip.run_cell("1", silent=True)
             assert pre_always.called
             assert not pre_explicit.called
             assert post_always.called
             assert not post_explicit.called
             # double-check that non-silent exec did what we expected
             # silent to avoid
             ip.run_cell("1")
             assert pre_explicit.called
             assert post_explicit.called
             info, = pre_explicit.call_args[0]
             result, = post_explicit.call_args[0]
             self.assertEqual(info, result.info)
             # check that post hooks are always called
             [m.reset_mock() for m in all_mocks]
             ip.run_cell("syntax error")
             assert pre_always.called
             assert pre_explicit.called
             assert post_always.called
             assert post_explicit.called
             info, = pre_explicit.call_args[0]
             result, = post_explicit.call_args[0]
             self.assertEqual(info, result.info)
         finally:
             # remove post-exec
             ip.events.unregister('pre_run_cell', pre_explicit)
             ip.events.unregister('pre_execute', pre_always)
             ip.events.unregister('post_run_cell', post_explicit)
             ip.events.unregister('post_execute', post_always)
 
     def test_silent_noadvance(self):
         """run_cell(silent=True) doesn't advance execution_count"""
         ec = ip.execution_count
         # silent should force store_history=False
         ip.run_cell("1", store_history=True, silent=True)
 
         self.assertEqual(ec, ip.execution_count)
         # double-check that non-silent exec did what we expected
         # silent to avoid
         ip.run_cell("1", store_history=True)
         self.assertEqual(ec+1, ip.execution_count)
 
     def test_silent_nodisplayhook(self):
         """run_cell(silent=True) doesn't trigger displayhook"""
         d = dict(called=False)
 
         trap = ip.display_trap
         save_hook = trap.hook
 
         def failing_hook(*args, **kwargs):
             d['called'] = True
 
         try:
             trap.hook = failing_hook
             res = ip.run_cell("1", silent=True)
             self.assertFalse(d['called'])
             self.assertIsNone(res.result)
             # double-check that non-silent exec did what we expected
             # silent to avoid
             ip.run_cell("1")
             self.assertTrue(d['called'])
         finally:
             trap.hook = save_hook
 
     def test_ofind_line_magic(self):
         from IPython.core.magic import register_line_magic
 
         @register_line_magic
         def lmagic(line):
             "A line magic"
 
         # Get info on line magic
         lfind = ip._ofind("lmagic")
         info = OInfo(
             found=True,
             isalias=False,
             ismagic=True,
             namespace="IPython internal",
             obj=lmagic,
             parent=None,
         )
         self.assertEqual(lfind, info)
 
     def test_ofind_cell_magic(self):
         from IPython.core.magic import register_cell_magic
 
         @register_cell_magic
         def cmagic(line, cell):
             "A cell magic"
 
         # Get info on cell magic
         find = ip._ofind("cmagic")
         info = OInfo(
             found=True,
             isalias=False,
             ismagic=True,
             namespace="IPython internal",
             obj=cmagic,
             parent=None,
         )
         self.assertEqual(find, info)
 
     def test_ofind_property_with_error(self):
         class A(object):
             @property
             def foo(self):
                 raise NotImplementedError()  # pragma: no cover
 
         a = A()
 
         found = ip._ofind("a.foo", [("locals", locals())])
         info = OInfo(
             found=True,
             isalias=False,
             ismagic=False,
             namespace="locals",
             obj=A.foo,
             parent=a,
         )
         self.assertEqual(found, info)
 
     def test_ofind_multiple_attribute_lookups(self):
         class A(object):
             @property
             def foo(self):
                 raise NotImplementedError()  # pragma: no cover
 
         a = A()
         a.a = A()
         a.a.a = A()
 
         found = ip._ofind("a.a.a.foo", [("locals", locals())])
         info = OInfo(
             found=True,
             isalias=False,
             ismagic=False,
             namespace="locals",
             obj=A.foo,
             parent=a.a.a,
         )
         self.assertEqual(found, info)
 
     def test_ofind_slotted_attributes(self):
         class A(object):
             __slots__ = ['foo']
             def __init__(self):
                 self.foo = 'bar'
 
         a = A()
         found = ip._ofind("a.foo", [("locals", locals())])
         info = OInfo(
             found=True,
             isalias=False,
             ismagic=False,
             namespace="locals",
             obj=a.foo,
             parent=a,
         )
         self.assertEqual(found, info)
 
         found = ip._ofind("a.bar", [("locals", locals())])
         expected = OInfo(
             found=False,
             isalias=False,
             ismagic=False,
             namespace=None,
             obj=None,
             parent=a,
         )
         assert found == expected
 
     def test_ofind_prefers_property_to_instance_level_attribute(self):
         class A(object):
             @property
             def foo(self):
                 return 'bar'
         a = A()
         a.__dict__["foo"] = "baz"
         self.assertEqual(a.foo, "bar")
         found = ip._ofind("a.foo", [("locals", locals())])
         self.assertIs(found.obj, A.foo)
 
     def test_custom_syntaxerror_exception(self):
         called = []
         def my_handler(shell, etype, value, tb, tb_offset=None):
             called.append(etype)
             shell.showtraceback((etype, value, tb), tb_offset=tb_offset)
 
         ip.set_custom_exc((SyntaxError,), my_handler)
         try:
             ip.run_cell("1f")
             # Check that this was called, and only once.
             self.assertEqual(called, [SyntaxError])
         finally:
             # Reset the custom exception hook
             ip.set_custom_exc((), None)
 
     def test_custom_exception(self):
         called = []
         def my_handler(shell, etype, value, tb, tb_offset=None):
             called.append(etype)
             shell.showtraceback((etype, value, tb), tb_offset=tb_offset)
 
         ip.set_custom_exc((ValueError,), my_handler)
         try:
             res = ip.run_cell("raise ValueError('test')")
             # Check that this was called, and only once.
             self.assertEqual(called, [ValueError])
             # Check that the error is on the result object
             self.assertIsInstance(res.error_in_exec, ValueError)
         finally:
             # Reset the custom exception hook
             ip.set_custom_exc((), None)
 
     @mock.patch("builtins.print")
     def test_showtraceback_with_surrogates(self, mocked_print):
         values = []
 
         def mock_print_func(value, sep=" ", end="\n", file=sys.stdout, flush=False):
             values.append(value)
             if value == chr(0xD8FF):
                 raise UnicodeEncodeError("utf-8", chr(0xD8FF), 0, 1, "")
 
         # mock builtins.print
         mocked_print.side_effect = mock_print_func
 
         # ip._showtraceback() is replaced in globalipapp.py.
         # Call original method to test.
         interactiveshell.InteractiveShell._showtraceback(ip, None, None, chr(0xD8FF))
 
         self.assertEqual(mocked_print.call_count, 2)
         self.assertEqual(values, [chr(0xD8FF), "\\ud8ff"])
 
     def test_mktempfile(self):
         filename = ip.mktempfile()
         # Check that we can open the file again on Windows
         with open(filename, "w", encoding="utf-8") as f:
             f.write("abc")
 
         filename = ip.mktempfile(data="blah")
         with open(filename, "r", encoding="utf-8") as f:
             self.assertEqual(f.read(), "blah")
 
     def test_new_main_mod(self):
         # Smoketest to check that this accepts a unicode module name
         name = u'jiefmw'
         mod = ip.new_main_mod(u'%s.py' % name, name)
         self.assertEqual(mod.__name__, name)
 
     def test_get_exception_only(self):
         try:
             raise KeyboardInterrupt
         except KeyboardInterrupt:
             msg = ip.get_exception_only()
         self.assertEqual(msg, 'KeyboardInterrupt\n')
 
         try:
             raise DerivedInterrupt("foo")
         except KeyboardInterrupt:
             msg = ip.get_exception_only()
         self.assertEqual(msg, 'IPython.core.tests.test_interactiveshell.DerivedInterrupt: foo\n')
 
     def test_inspect_text(self):
         ip.run_cell('a = 5')
         text = ip.object_inspect_text('a')
         self.assertIsInstance(text, str)
 
     def test_last_execution_result(self):
         """ Check that last execution result gets set correctly (GH-10702) """
         result = ip.run_cell('a = 5; a')
         self.assertTrue(ip.last_execution_succeeded)
         self.assertEqual(ip.last_execution_result.result, 5)
 
         result = ip.run_cell('a = x_invalid_id_x')
         self.assertFalse(ip.last_execution_succeeded)
         self.assertFalse(ip.last_execution_result.success)
         self.assertIsInstance(ip.last_execution_result.error_in_exec, NameError)
 
     def test_reset_aliasing(self):
         """ Check that standard posix aliases work after %reset. """
         if os.name != 'posix':
             return
 
         ip.reset()
         for cmd in ('clear', 'more', 'less', 'man'):
             res = ip.run_cell('%' + cmd)
             self.assertEqual(res.success, True)
 
 
 @pytest.mark.skipif(
     sys.implementation.name == "pypy"
     and ((7, 3, 13) < sys.implementation.version < (7, 3, 16)),
     reason="Unicode issues with scandir on PyPy, see https://github.com/pypy/pypy/issues/4860",
 )
 class TestSafeExecfileNonAsciiPath(unittest.TestCase):
     @onlyif_unicode_paths
     def setUp(self):
         self.BASETESTDIR = tempfile.mkdtemp()
         self.TESTDIR = join(self.BASETESTDIR, u"åäö")
         os.mkdir(self.TESTDIR)
         with open(
             join(self.TESTDIR, "åäötestscript.py"), "w", encoding="utf-8"
         ) as sfile:
             sfile.write("pass\n")
         self.oldpath = os.getcwd()
         os.chdir(self.TESTDIR)
         self.fname = u"åäötestscript.py"
 
     def tearDown(self):
         os.chdir(self.oldpath)
         shutil.rmtree(self.BASETESTDIR)
 
     @onlyif_unicode_paths
     def test_1(self):
         """Test safe_execfile with non-ascii path
         """
         ip.safe_execfile(self.fname, {}, raise_exceptions=True)
 
 class ExitCodeChecks(tt.TempFileMixin):
 
     def setUp(self):
         self.system = ip.system_raw
 
     def test_exit_code_ok(self):
         self.system('exit 0')
         self.assertEqual(ip.user_ns['_exit_code'], 0)
 
     def test_exit_code_error(self):
         self.system('exit 1')
         self.assertEqual(ip.user_ns['_exit_code'], 1)
 
     @skipif(not hasattr(signal, 'SIGALRM'))
     def test_exit_code_signal(self):
         self.mktmp("import signal, time\n"
                    "signal.setitimer(signal.ITIMER_REAL, 0.1)\n"
                    "time.sleep(1)\n")
         self.system("%s %s" % (sys.executable, self.fname))
         self.assertEqual(ip.user_ns['_exit_code'], -signal.SIGALRM)
 
     @onlyif_cmds_exist("csh")
     def test_exit_code_signal_csh(self):  # pragma: no cover
         SHELL = os.environ.get("SHELL", None)
         os.environ["SHELL"] = find_cmd("csh")
         try:
             self.test_exit_code_signal()
         finally:
             if SHELL is not None:
                 os.environ['SHELL'] = SHELL
             else:
                 del os.environ['SHELL']
 
 
 class TestSystemRaw(ExitCodeChecks):
 
     def setUp(self):
         super().setUp()
         self.system = ip.system_raw
 
     @onlyif_unicode_paths
     def test_1(self):
         """Test system_raw with non-ascii cmd
         """
         cmd = u'''python -c "'åäö'"   '''
         ip.system_raw(cmd)
 
     @mock.patch('subprocess.call', side_effect=KeyboardInterrupt)
     @mock.patch('os.system', side_effect=KeyboardInterrupt)
     def test_control_c(self, *mocks):
         try:
             self.system("sleep 1 # won't happen")
         except KeyboardInterrupt:  # pragma: no cove
             self.fail(
                 "system call should intercept "
                 "keyboard interrupt from subprocess.call"
             )
         self.assertEqual(ip.user_ns["_exit_code"], -signal.SIGINT)
 
 
 @pytest.mark.parametrize("magic_cmd", ["pip", "conda", "cd"])
 def test_magic_warnings(magic_cmd):
     if sys.platform == "win32":
         to_mock = "os.system"
         expected_arg, expected_kwargs = magic_cmd, dict()
     else:
         to_mock = "subprocess.call"
         expected_arg, expected_kwargs = magic_cmd, dict(
             shell=True, executable=os.environ.get("SHELL", None)
         )
 
     with mock.patch(to_mock, return_value=0) as mock_sub:
         with pytest.warns(Warning, match=r"You executed the system command"):
             ip.system_raw(magic_cmd)
         mock_sub.assert_called_once_with(expected_arg, **expected_kwargs)
 
 
 # TODO: Exit codes are currently ignored on Windows.
 class TestSystemPipedExitCode(ExitCodeChecks):
 
     def setUp(self):
         super().setUp()
         self.system = ip.system_piped
 
     @skip_win32
     def test_exit_code_ok(self):
         ExitCodeChecks.test_exit_code_ok(self)
 
     @skip_win32
     def test_exit_code_error(self):
         ExitCodeChecks.test_exit_code_error(self)
 
     @skip_win32
     def test_exit_code_signal(self):
         ExitCodeChecks.test_exit_code_signal(self)
 
 class TestModules(tt.TempFileMixin):
     def test_extraneous_loads(self):
         """Test we're not loading modules on startup that we shouldn't.
         """
         self.mktmp("import sys\n"
                    "print('numpy' in sys.modules)\n"
                    "print('ipyparallel' in sys.modules)\n"
                    "print('ipykernel' in sys.modules)\n"
                    )
         out = "False\nFalse\nFalse\n"
         tt.ipexec_validate(self.fname, out)
 
 class Negator(ast.NodeTransformer):
     """Negates all number literals in an AST."""
 
     def visit_Num(self, node):
         node.value = -node.value
         return node
 
     def visit_Constant(self, node):
         if isinstance(node.value, int):
             return self.visit_Num(node)
         return node
 
 class TestAstTransform(unittest.TestCase):
     def setUp(self):
         self.negator = Negator()
         ip.ast_transformers.append(self.negator)
 
     def tearDown(self):
         ip.ast_transformers.remove(self.negator)
 
     def test_non_int_const(self):
         with tt.AssertPrints("hello"):
             ip.run_cell('print("hello")')
 
     def test_run_cell(self):
         with tt.AssertPrints("-34"):
             ip.run_cell("print(12 + 22)")
 
         # A named reference to a number shouldn't be transformed.
         ip.user_ns["n"] = 55
         with tt.AssertNotPrints("-55"):
             ip.run_cell("print(n)")
 
     def test_timeit(self):
         called = set()
         def f(x):
             called.add(x)
         ip.push({'f':f})
 
         with tt.AssertPrints("std. dev. of"):
             ip.run_line_magic("timeit", "-n1 f(1)")
         self.assertEqual(called, {-1})
         called.clear()
 
         with tt.AssertPrints("std. dev. of"):
             ip.run_cell_magic("timeit", "-n1 f(2)", "f(3)")
         self.assertEqual(called, {-2, -3})
 
     def test_time(self):
         called = []
         def f(x):
             called.append(x)
         ip.push({'f':f})
 
         # Test with an expression
         with tt.AssertPrints("Wall time: "):
             ip.run_line_magic("time", "f(5+9)")
         self.assertEqual(called, [-14])
         called[:] = []
 
         # Test with a statement (different code path)
         with tt.AssertPrints("Wall time: "):
             ip.run_line_magic("time", "a = f(-3 + -2)")
         self.assertEqual(called, [5])
 
     def test_macro(self):
         ip.push({'a':10})
         # The AST transformation makes this do a+=-1
         ip.define_macro("amacro", "a+=1\nprint(a)")
 
         with tt.AssertPrints("9"):
             ip.run_cell("amacro")
         with tt.AssertPrints("8"):
             ip.run_cell("amacro")
 
 class TestMiscTransform(unittest.TestCase):
 
 
     def test_transform_only_once(self):
         cleanup = 0
         line_t = 0
         def count_cleanup(lines):
             nonlocal cleanup
             cleanup += 1
             return lines
 
         def count_line_t(lines):
             nonlocal line_t
             line_t += 1
             return lines
 
         ip.input_transformer_manager.cleanup_transforms.append(count_cleanup)
         ip.input_transformer_manager.line_transforms.append(count_line_t)
 
         ip.run_cell('1')
 
         assert cleanup == 1
         assert line_t == 1
 
 class IntegerWrapper(ast.NodeTransformer):
     """Wraps all integers in a call to Integer()"""
 
     # for Python 3.7 and earlier
 
     # for Python 3.7 and earlier
     def visit_Num(self, node):
         if isinstance(node.n, int):
             return ast.Call(func=ast.Name(id='Integer', ctx=ast.Load()),
                             args=[node], keywords=[])
         return node
 
     # For Python 3.8+
     def visit_Constant(self, node):
         if isinstance(node.value, int):
             return self.visit_Num(node)
         return node
 
 
 class TestAstTransform2(unittest.TestCase):
     def setUp(self):
         self.intwrapper = IntegerWrapper()
         ip.ast_transformers.append(self.intwrapper)
 
         self.calls = []
         def Integer(*args):
             self.calls.append(args)
             return args
         ip.push({"Integer": Integer})
 
     def tearDown(self):
         ip.ast_transformers.remove(self.intwrapper)
         del ip.user_ns['Integer']
 
     def test_run_cell(self):
         ip.run_cell("n = 2")
         self.assertEqual(self.calls, [(2,)])
 
         # This shouldn't throw an error
         ip.run_cell("o = 2.0")
         self.assertEqual(ip.user_ns['o'], 2.0)
 
     def test_run_cell_non_int(self):
         ip.run_cell("n = 'a'")
         assert self.calls == []
 
     def test_timeit(self):
         called = set()
         def f(x):
             called.add(x)
         ip.push({'f':f})
 
         with tt.AssertPrints("std. dev. of"):
             ip.run_line_magic("timeit", "-n1 f(1)")
         self.assertEqual(called, {(1,)})
         called.clear()
 
         with tt.AssertPrints("std. dev. of"):
             ip.run_cell_magic("timeit", "-n1 f(2)", "f(3)")
         self.assertEqual(called, {(2,), (3,)})
 
 class ErrorTransformer(ast.NodeTransformer):
     """Throws an error when it sees a number."""
 
     def visit_Constant(self, node):
         if isinstance(node.value, int):
             raise ValueError("test")
         return node
 
 
 class TestAstTransformError(unittest.TestCase):
     def test_unregistering(self):
         err_transformer = ErrorTransformer()
         ip.ast_transformers.append(err_transformer)
 
         with self.assertWarnsRegex(UserWarning, "It will be unregistered"):
             ip.run_cell("1 + 2")
 
         # This should have been removed.
         self.assertNotIn(err_transformer, ip.ast_transformers)
 
 
 class StringRejector(ast.NodeTransformer):
     """Throws an InputRejected when it sees a string literal.
 
     Used to verify that NodeTransformers can signal that a piece of code should
     not be executed by throwing an InputRejected.
     """
 
     def visit_Constant(self, node):
         if isinstance(node.value, str):
             raise InputRejected("test")
         return node
 
 
 class TestAstTransformInputRejection(unittest.TestCase):
 
     def setUp(self):
         self.transformer = StringRejector()
         ip.ast_transformers.append(self.transformer)
 
     def tearDown(self):
         ip.ast_transformers.remove(self.transformer)
 
     def test_input_rejection(self):
         """Check that NodeTransformers can reject input."""
 
         expect_exception_tb = tt.AssertPrints("InputRejected: test")
         expect_no_cell_output = tt.AssertNotPrints("'unsafe'", suppress=False)
 
         # Run the same check twice to verify that the transformer is not
         # disabled after raising.
         with expect_exception_tb, expect_no_cell_output:
             ip.run_cell("'unsafe'")
 
         with expect_exception_tb, expect_no_cell_output:
             res = ip.run_cell("'unsafe'")
 
         self.assertIsInstance(res.error_before_exec, InputRejected)
 
 def test__IPYTHON__():
     # This shouldn't raise a NameError, that's all
     __IPYTHON__
 
 
 class DummyRepr(object):
     def __repr__(self):
         return "DummyRepr"
 
     def _repr_html_(self):
         return "<b>dummy</b>"
 
     def _repr_javascript_(self):
         return "console.log('hi');", {'key': 'value'}
 
 
 def test_user_variables():
     # enable all formatters
     ip.display_formatter.active_types = ip.display_formatter.format_types
 
     ip.user_ns['dummy'] = d = DummyRepr()
     keys = {'dummy', 'doesnotexist'}
     r = ip.user_expressions({ key:key for key in keys})
 
     assert keys == set(r.keys())
     dummy = r["dummy"]
     assert {"status", "data", "metadata"} == set(dummy.keys())
     assert dummy["status"] == "ok"
     data = dummy["data"]
     metadata = dummy["metadata"]
     assert data.get("text/html") == d._repr_html_()
     js, jsmd = d._repr_javascript_()
     assert data.get("application/javascript") == js
     assert metadata.get("application/javascript") == jsmd
 
     dne = r["doesnotexist"]
     assert dne["status"] == "error"
     assert dne["ename"] == "NameError"
 
     # back to text only
     ip.display_formatter.active_types = ['text/plain']
 
 def test_user_expression():
     # enable all formatters
     ip.display_formatter.active_types = ip.display_formatter.format_types
     query = {
         'a' : '1 + 2',
         'b' : '1/0',
     }
     r = ip.user_expressions(query)
     import pprint
     pprint.pprint(r)
     assert set(r.keys()) == set(query.keys())
     a = r["a"]
     assert {"status", "data", "metadata"} == set(a.keys())
     assert a["status"] == "ok"
     data = a["data"]
     metadata = a["metadata"]
     assert data.get("text/plain") == "3"
 
     b = r["b"]
     assert b["status"] == "error"
     assert b["ename"] == "ZeroDivisionError"
 
     # back to text only
     ip.display_formatter.active_types = ['text/plain']
 
 
 class TestSyntaxErrorTransformer(unittest.TestCase):
     """Check that SyntaxError raised by an input transformer is handled by run_cell()"""
 
     @staticmethod
     def transformer(lines):
         for line in lines:
             pos = line.find('syntaxerror')
             if pos >= 0:
                 e = SyntaxError('input contains "syntaxerror"')
                 e.text = line
                 e.offset = pos + 1
                 raise e
         return lines
 
     def setUp(self):
         ip.input_transformers_post.append(self.transformer)
 
     def tearDown(self):
         ip.input_transformers_post.remove(self.transformer)
 
     def test_syntaxerror_input_transformer(self):
         with tt.AssertPrints('1234'):
             ip.run_cell('1234')
         with tt.AssertPrints('SyntaxError: invalid syntax'):
             ip.run_cell('1 2 3')   # plain python syntax error
         with tt.AssertPrints('SyntaxError: input contains "syntaxerror"'):
             ip.run_cell('2345  # syntaxerror')  # input transformer syntax error
         with tt.AssertPrints('3456'):
             ip.run_cell('3456')
 
 
 class TestWarningSuppression(unittest.TestCase):
     def test_warning_suppression(self):
         ip.run_cell("import warnings")
         try:
             with self.assertWarnsRegex(UserWarning, "asdf"):
                 ip.run_cell("warnings.warn('asdf')")
             # Here's the real test -- if we run that again, we should get the
             # warning again. Traditionally, each warning was only issued once per
             # IPython session (approximately), even if the user typed in new and
             # different code that should have also triggered the warning, leading
             # to much confusion.
             with self.assertWarnsRegex(UserWarning, "asdf"):
                 ip.run_cell("warnings.warn('asdf')")
         finally:
             ip.run_cell("del warnings")
 
 
     def test_deprecation_warning(self):
         ip.run_cell("""
 import warnings
 def wrn():
     warnings.warn(
         "I AM  A WARNING",
         DeprecationWarning
     )
         """)
         try:
             with self.assertWarnsRegex(DeprecationWarning, "I AM  A WARNING"):
                 ip.run_cell("wrn()")
         finally:
             ip.run_cell("del warnings")
             ip.run_cell("del wrn")
 
 
 class TestImportNoDeprecate(tt.TempFileMixin):
 
     def setUp(self):
         """Make a valid python temp file."""
         self.mktmp("""
 import warnings
 def wrn():
     warnings.warn(
         "I AM  A WARNING",
         DeprecationWarning
     )
 """)
         super().setUp()
 
     def test_no_dep(self):
         """
         No deprecation warning should be raised from imported functions
         """
         ip.run_cell("from {} import wrn".format(self.fname))
 
         with tt.AssertNotPrints("I AM  A WARNING"):
             ip.run_cell("wrn()")
         ip.run_cell("del wrn")
 
 
 def test_custom_exc_count():
     hook = mock.Mock(return_value=None)
     ip.set_custom_exc((SyntaxError,), hook)
     before = ip.execution_count
     ip.run_cell("def foo()", store_history=True)
     # restore default excepthook
     ip.set_custom_exc((), None)
     assert hook.call_count == 1
     assert ip.execution_count == before + 1
 
 
 def test_run_cell_async():
     ip.run_cell("import asyncio")
     coro = ip.run_cell_async("await asyncio.sleep(0.01)\n5")
     assert asyncio.iscoroutine(coro)
     loop = asyncio.new_event_loop()
     result = loop.run_until_complete(coro)
     assert isinstance(result, interactiveshell.ExecutionResult)
     assert result.result == 5
 
 
 def test_run_cell_await():
     ip.run_cell("import asyncio")
     result = ip.run_cell("await asyncio.sleep(0.01); 10")
     assert ip.user_ns["_"] == 10
 
 
 def test_run_cell_asyncio_run():
     ip.run_cell("import asyncio")
     result = ip.run_cell("await asyncio.sleep(0.01); 1")
     assert ip.user_ns["_"] == 1
     result = ip.run_cell("asyncio.run(asyncio.sleep(0.01)); 2")
     assert ip.user_ns["_"] == 2
     result = ip.run_cell("await asyncio.sleep(0.01); 3")
     assert ip.user_ns["_"] == 3
 
 
 def test_should_run_async():
     assert not ip.should_run_async("a = 5", transformed_cell="a = 5")
     assert ip.should_run_async("await x", transformed_cell="await x")
     assert ip.should_run_async(
         "import asyncio; await asyncio.sleep(1)",
         transformed_cell="import asyncio; await asyncio.sleep(1)",
     )
 
 
 def test_set_custom_completer():
     num_completers = len(ip.Completer.matchers)
 
     def foo(*args, **kwargs):
         return "I'm a completer!"
 
     ip.set_custom_completer(foo, 0)
 
     # check that we've really added a new completer
     assert len(ip.Completer.matchers) == num_completers + 1
 
     # check that the first completer is the function we defined
     assert ip.Completer.matchers[0]() == "I'm a completer!"
 
     # clean up
     ip.Completer.custom_matchers.pop()
 
 
 class TestShowTracebackAttack(unittest.TestCase):
     """Test that the interactive shell is resilient against the client attack of
     manipulating the showtracebacks method. These attacks shouldn't result in an
     unhandled exception in the kernel."""
 
     def setUp(self):
         self.orig_showtraceback = interactiveshell.InteractiveShell.showtraceback
 
     def tearDown(self):
         interactiveshell.InteractiveShell.showtraceback = self.orig_showtraceback
 
     def test_set_show_tracebacks_none(self):
         """Test the case of the client setting showtracebacks to None"""
 
         result = ip.run_cell(
             """
             import IPython.core.interactiveshell
             IPython.core.interactiveshell.InteractiveShell.showtraceback = None
 
             assert False, "This should not raise an exception"
         """
         )
         print(result)
 
         assert result.result is None
         assert isinstance(result.error_in_exec, TypeError)
         assert str(result.error_in_exec) == "'NoneType' object is not callable"
 
     def test_set_show_tracebacks_noop(self):
         """Test the case of the client setting showtracebacks to a no op lambda"""
 
         result = ip.run_cell(
             """
             import IPython.core.interactiveshell
             IPython.core.interactiveshell.InteractiveShell.showtraceback = lambda *args, **kwargs: None
 
             assert False, "This should not raise an exception"
         """
         )
         print(result)
 
         assert result.result is None
         assert isinstance(result.error_in_exec, AssertionError)
         assert str(result.error_in_exec) == "This should not raise an exception"
 
 
 @skip_if_not_osx
 def test_enable_gui_osx():
     simple_prompt = ip.simple_prompt
     ip.simple_prompt = False
 
     ip.enable_gui("osx")
     assert ip.active_eventloop == "osx"
     ip.enable_gui()
 
     # The following line fails for IPython <= 8.25.0
     ip.enable_gui("macosx")
     assert ip.active_eventloop == "osx"
     ip.enable_gui()
 
     ip.simple_prompt = simple_prompt

IPython/core/tests/test_iplib.py

@@ -1,245 +1,246
 """Tests for the key interactiveshell module, where the main ipython class is defined.
 """
 
 import stack_data
+import sys
 
 SV_VERSION = tuple([int(x) for x in stack_data.__version__.split(".")[0:2]])
 
 
 def test_reset():
     """reset must clear most namespaces."""
 
     # Check that reset runs without error
     ip.reset()
 
     # Once we've reset it (to clear of any junk that might have been there from
     # other tests, we can count how many variables are in the user's namespace
     nvars_user_ns = len(ip.user_ns)
     nvars_hidden = len(ip.user_ns_hidden)
 
     # Now add a few variables to user_ns, and check that reset clears them
     ip.user_ns['x'] = 1
     ip.user_ns['y'] = 1
     ip.reset()
     
     # Finally, check that all namespaces have only as many variables as we
     # expect to find in them:
     assert len(ip.user_ns) == nvars_user_ns
     assert len(ip.user_ns_hidden) == nvars_hidden
 
 
 # Tests for reporting of exceptions in various modes, handling of SystemExit,
 # and %tb functionality.  This is really a mix of testing ultraTB and interactiveshell.
 
 def doctest_tb_plain():
     """
     In [18]: xmode plain
     Exception reporting mode: Plain
 
     In [19]: run simpleerr.py
     Traceback (most recent call last):
       File ...:...
         bar(mode)
       File ...:... in bar
         div0()
       File ...:... in div0
         x/y
     ZeroDivisionError: ...
     """
 
 
 def doctest_tb_context():
     """
     In [3]: xmode context
     Exception reporting mode: Context
 
     In [4]: run simpleerr.py
     ---------------------------------------------------------------------------
     ZeroDivisionError                         Traceback (most recent call last)
     <BLANKLINE>
     ...
          30     except IndexError:
          31         mode = 'div'
     ---> 33     bar(mode)
     <BLANKLINE>
     ... in bar(mode)
          15     "bar"
          16     if mode=='div':
     ---> 17         div0()
          18     elif mode=='exit':
          19         try:
     <BLANKLINE>
     ... in div0()
           6     x = 1
           7     y = 0
     ----> 8     x/y
     <BLANKLINE>
     ZeroDivisionError: ..."""
 
 
 def doctest_tb_verbose():
     """
     In [5]: xmode verbose
     Exception reporting mode: Verbose
 
     In [6]: run simpleerr.py
     ---------------------------------------------------------------------------
     ZeroDivisionError                         Traceback (most recent call last)
     <BLANKLINE>
     ...
          30     except IndexError:
          31         mode = 'div'
     ---> 33     bar(mode)
             mode = 'div'
     <BLANKLINE>
     ... in bar(mode='div')
          15     "bar"
          16     if mode=='div':
     ---> 17         div0()
          18     elif mode=='exit':
          19         try:
     <BLANKLINE>
     ... in div0()
           6     x = 1
           7     y = 0
     ----> 8     x/y
             x = 1
             y = 0
     <BLANKLINE>
     ZeroDivisionError: ...
     """
 
 
 def doctest_tb_sysexit():
     """
     In [17]: %xmode plain
     Exception reporting mode: Plain
 
     In [18]: %run simpleerr.py exit
     An exception has occurred, use %tb to see the full traceback.
     SystemExit: (1, 'Mode = exit')
 
     In [19]: %run simpleerr.py exit 2
     An exception has occurred, use %tb to see the full traceback.
     SystemExit: (2, 'Mode = exit')
 
     In [20]: %tb
     Traceback (most recent call last):
       File ...:... in execfile
         exec(compiler(f.read(), fname, "exec"), glob, loc)
       File ...:...
         bar(mode)
       File ...:... in bar
         sysexit(stat, mode)
       File ...:... in sysexit
         raise SystemExit(stat, f"Mode = {mode}")
     SystemExit: (2, 'Mode = exit')
 
     In [21]: %xmode context
     Exception reporting mode: Context
 
     In [22]: %tb
     ---------------------------------------------------------------------------
     SystemExit                                Traceback (most recent call last)
     File ..., in execfile(fname, glob, loc, compiler)
          ... with open(fname, "rb") as f:
          ...     compiler = compiler or compile
     ---> ...     exec(compiler(f.read(), fname, "exec"), glob, loc)
     ...
          30     except IndexError:
          31         mode = 'div'
     ---> 33     bar(mode)
     <BLANKLINE>
     ...bar(mode)
          21         except:
          22             stat = 1
     ---> 23         sysexit(stat, mode)
          24     else:
          25         raise ValueError('Unknown mode')
     <BLANKLINE>
     ...sysexit(stat, mode)
          10 def sysexit(stat, mode):
     ---> 11     raise SystemExit(stat, f"Mode = {mode}")
     <BLANKLINE>
     SystemExit: (2, 'Mode = exit')
     """
 
 
 if SV_VERSION < (0, 6):
 
     def doctest_tb_sysexit_verbose_stack_data_05():
         """
         In [18]: %run simpleerr.py exit
         An exception has occurred, use %tb to see the full traceback.
         SystemExit: (1, 'Mode = exit')
 
         In [19]: %run simpleerr.py exit 2
         An exception has occurred, use %tb to see the full traceback.
         SystemExit: (2, 'Mode = exit')
 
         In [23]: %xmode verbose
         Exception reporting mode: Verbose
 
         In [24]: %tb
         ---------------------------------------------------------------------------
         SystemExit                                Traceback (most recent call last)
         <BLANKLINE>
         ...
             30     except IndexError:
             31         mode = 'div'
         ---> 33     bar(mode)
                 mode = 'exit'
         <BLANKLINE>
         ... in bar(mode='exit')
             ...     except:
             ...         stat = 1
         ---> ...     sysexit(stat, mode)
                 mode = 'exit'
                 stat = 2
             ...     else:
             ...         raise ValueError('Unknown mode')
         <BLANKLINE>
         ... in sysexit(stat=2, mode='exit')
             10 def sysexit(stat, mode):
         ---> 11     raise SystemExit(stat, f"Mode = {mode}")
                 stat = 2
         <BLANKLINE>
         SystemExit: (2, 'Mode = exit')
         """
 
 
 def test_run_cell():
     import textwrap
 
     ip.run_cell("a = 10\na+=1")
     ip.run_cell("assert a == 11\nassert 1")
 
     assert ip.user_ns["a"] == 11
     complex = textwrap.dedent(
         """
     if 1:
         print "hello"
         if 1:
             print "world"
         
     if 2:
         print "foo"
 
     if 3:
         print "bar"
 
     if 4:
         print "bar"
     
     """
     )
     # Simply verifies that this kind of input is run
     ip.run_cell(complex)
     
 
 def test_db():
     """Test the internal database used for variable persistence."""
     ip.db["__unittest_"] = 12
     assert ip.db["__unittest_"] == 12
     del ip.db["__unittest_"]
     assert "__unittest_" not in ip.db

IPython/core/tests/test_magic.py

@@ -1,1591 +1,1556
 # -*- coding: utf-8 -*-
 """Tests for various magic functions."""
 
 import gc
 import io
 import os
 import re
 import shlex
 import sys
 import warnings
 from importlib import invalidate_caches
 from io import StringIO
 from pathlib import Path
 from textwrap import dedent
 from unittest import TestCase, mock
 
 import pytest
 
 from IPython import get_ipython
 from IPython.core import magic
 from IPython.core.error import UsageError
 from IPython.core.magic import (
     Magics,
     cell_magic,
     line_magic,
     magics_class,
     register_cell_magic,
     register_line_magic,
 )
 from IPython.core.magics import code, execution, logging, osm, script
 from IPython.testing import decorators as dec
 from IPython.testing import tools as tt
 from IPython.utils.io import capture_output
 from IPython.utils.process import find_cmd
 from IPython.utils.tempdir import TemporaryDirectory, TemporaryWorkingDirectory
 from IPython.utils.syspathcontext import prepended_to_syspath
 
-# import needed by doctest
-from .test_debugger import PdbTestInput  # noqa: F401
-
-_ip = get_ipython()
+from .test_debugger import PdbTestInput
 
+from tempfile import NamedTemporaryFile
 
 @magic.magics_class
-class DummyMagics(magic.Magics):
-    pass
-
+class DummyMagics(magic.Magics): pass
 
 def test_extract_code_ranges():
     instr = "1 3 5-6 7-9 10:15 17: :10 10- -13 :"
     expected = [
         (0, 1),
         (2, 3),
         (4, 6),
         (6, 9),
         (9, 14),
         (16, None),
         (None, 9),
         (9, None),
         (None, 13),
         (None, None),
     ]
     actual = list(code.extract_code_ranges(instr))
     assert actual == expected
 
-
 def test_extract_symbols():
     source = """import foo\na = 10\ndef b():\n    return 42\n\n\nclass A: pass\n\n\n"""
     symbols_args = ["a", "b", "A", "A,b", "A,a", "z"]
-    expected = [
-        ([], ["a"]),
-        (["def b():\n    return 42\n"], []),
-        (["class A: pass\n"], []),
-        (["class A: pass\n", "def b():\n    return 42\n"], []),
-        (["class A: pass\n"], ["a"]),
-        ([], ["z"]),
-    ]
+    expected = [([], ['a']),
+                (["def b():\n    return 42\n"], []),
+                (["class A: pass\n"], []),
+                (["class A: pass\n", "def b():\n    return 42\n"], []),
+                (["class A: pass\n"], ['a']),
+                ([], ['z'])]
     for symbols, exp in zip(symbols_args, expected):
         assert code.extract_symbols(source, symbols) == exp
 
 
 def test_extract_symbols_raises_exception_with_non_python_code():
-    source = "=begin A Ruby program :)=end\n" "def hello\n" "puts 'Hello world'\n" "end"
+    source = ("=begin A Ruby program :)=end\n"
+              "def hello\n"
+              "puts 'Hello world'\n"
+              "end")
     with pytest.raises(SyntaxError):
         code.extract_symbols(source, "hello")
 
 
 def test_magic_not_found():
     # magic not found raises UsageError
     with pytest.raises(UsageError):
         _ip.run_line_magic("doesntexist", "")
 
     # ensure result isn't success when a magic isn't found
-    result = _ip.run_cell("%doesntexist")
+    result = _ip.run_cell('%doesntexist')
     assert isinstance(result.error_in_exec, UsageError)
 
 
 def test_cell_magic_not_found():
     # magic not found raises UsageError
     with pytest.raises(UsageError):
-        _ip.run_cell_magic("doesntexist", "line", "cell")
+        _ip.run_cell_magic('doesntexist', 'line', 'cell')
 
     # ensure result isn't success when a magic isn't found
-    result = _ip.run_cell("%%doesntexist")
+    result = _ip.run_cell('%%doesntexist')
     assert isinstance(result.error_in_exec, UsageError)
 
 
 def test_magic_error_status():
     def fail(shell):
-        1 / 0
-
+        1/0
     _ip.register_magic_function(fail)
-    result = _ip.run_cell("%fail")
+    result = _ip.run_cell('%fail')
     assert isinstance(result.error_in_exec, ZeroDivisionError)
 
 
 def test_config():
-    """test that config magic does not raise
+    """ test that config magic does not raise
     can happen if Configurable init is moved too early into
     Magics.__init__ as then a Config object will be registered as a
     magic.
     """
     ## should not raise.
     _ip.run_line_magic("config", "")
 
 
 def test_config_available_configs():
-    """test that config magic prints available configs in unique and
-    sorted order."""
+    """ test that config magic prints available configs in unique and
+    sorted order. """
     with capture_output() as captured:
         _ip.run_line_magic("config", "")
 
     stdout = captured.stdout
-    config_classes = stdout.strip().split("\n")[1:]
+    config_classes = stdout.strip().split('\n')[1:]
     assert config_classes == sorted(set(config_classes))
 
-
 def test_config_print_class():
-    """test that config with a classname prints the class's options."""
+    """ test that config with a classname prints the class's options. """
     with capture_output() as captured:
         _ip.run_line_magic("config", "TerminalInteractiveShell")
 
     stdout = captured.stdout
     assert re.match(
         "TerminalInteractiveShell.* options", stdout.splitlines()[0]
     ), f"{stdout}\n\n1st line of stdout not like 'TerminalInteractiveShell.* options'"
 
 
 def test_rehashx():
     # clear up everything
     _ip.alias_manager.clear_aliases()
-    del _ip.db["syscmdlist"]
+    del _ip.db['syscmdlist']
 
     _ip.run_line_magic("rehashx", "")
     # Practically ALL ipython development systems will have more than 10 aliases
 
     assert len(_ip.alias_manager.aliases) > 10
     for name, cmd in _ip.alias_manager.aliases:
         # we must strip dots from alias names
         assert "." not in name
 
     # rehashx must fill up syscmdlist
-    scoms = _ip.db["syscmdlist"]
+    scoms = _ip.db['syscmdlist']
     assert len(scoms) > 10
 
 
 def test_magic_parse_options():
     """Test that we don't mangle paths when parsing magic options."""
     ip = get_ipython()
-    path = "c:\\x"
+    path = 'c:\\x'
     m = DummyMagics(ip)
-    opts = m.parse_options("-f %s" % path, "f:")[0]
+    opts = m.parse_options('-f %s' % path,'f:')[0]
     # argv splitting is os-dependent
-    if os.name == "posix":
-        expected = "c:x"
+    if os.name == 'posix':
+        expected = 'c:x'
     else:
         expected = path
     assert opts["f"] == expected
 
 
 def test_magic_parse_long_options():
     """Magic.parse_options can handle --foo=bar long options"""
     ip = get_ipython()
     m = DummyMagics(ip)
     opts, _ = m.parse_options("--foo --bar=bubble", "a", "foo", "bar=")
     assert "foo" in opts
     assert "bar" in opts
     assert opts["bar"] == "bubble"
 
 
 def doctest_hist_f():
     """Test %hist -f with temporary filename.
 
     In [9]: import tempfile
 
     In [10]: tfile = tempfile.mktemp('.py','tmp-ipython-')
 
     In [11]: %hist -nl -f $tfile 3
 
     In [13]: import os; os.unlink(tfile)
     """
 
 
 def doctest_hist_op():
     """Test %hist -op
 
     In [1]: class b(float):
        ...:     pass
        ...:
 
     In [2]: class s(object):
        ...:     def __str__(self):
        ...:         return 's'
        ...:
 
     In [3]:
 
     In [4]: class r(b):
        ...:     def __repr__(self):
        ...:         return 'r'
        ...:
 
     In [5]: class sr(s,r): pass
        ...:
 
     In [6]:
 
     In [7]: bb=b()
 
     In [8]: ss=s()
 
     In [9]: rr=r()
 
     In [10]: ssrr=sr()
 
     In [11]: 4.5
     Out[11]: 4.5
 
     In [12]: str(ss)
     Out[12]: 's'
 
     In [13]:
 
     In [14]: %hist -op
     >>> class b:
     ...     pass
     ...
     >>> class s(b):
     ...     def __str__(self):
     ...         return 's'
     ...
     >>>
     >>> class r(b):
     ...     def __repr__(self):
     ...         return 'r'
     ...
     >>> class sr(s,r): pass
     >>>
     >>> bb=b()
     >>> ss=s()
     >>> rr=r()
     >>> ssrr=sr()
     >>> 4.5
     4.5
     >>> str(ss)
     's'
     >>>
     """
 
-
 def test_hist_pof():
     ip = get_ipython()
     ip.run_cell("1+2", store_history=True)
-    # raise Exception(ip.history_manager.session_number)
-    # raise Exception(list(ip.history_manager._get_range_session()))
+    #raise Exception(ip.history_manager.session_number)
+    #raise Exception(list(ip.history_manager._get_range_session()))
     with TemporaryDirectory() as td:
-        tf = os.path.join(td, "hist.py")
-        ip.run_line_magic("history", "-pof %s" % tf)
+        tf = os.path.join(td, 'hist.py')
+        ip.run_line_magic('history', '-pof %s' % tf)
         assert os.path.isfile(tf)
 
 
 def test_macro():
     ip = get_ipython()
-    ip.history_manager.reset()  # Clear any existing history.
+    ip.history_manager.reset()   # Clear any existing history.
     cmds = ["a=1", "def b():\n  return a**2", "print(a,b())"]
     for i, cmd in enumerate(cmds, start=1):
         ip.history_manager.store_inputs(i, cmd)
     ip.run_line_magic("macro", "test 1-3")
     assert ip.user_ns["test"].value == "\n".join(cmds) + "\n"
 
     # List macros
     assert "test" in ip.run_line_magic("macro", "")
 
 
 def test_macro_run():
     """Test that we can run a multi-line macro successfully."""
     ip = get_ipython()
     ip.history_manager.reset()
     cmds = ["a=10", "a+=1", "print(a)", "%macro test 2-3"]
     for cmd in cmds:
         ip.run_cell(cmd, store_history=True)
     assert ip.user_ns["test"].value == "a+=1\nprint(a)\n"
     with tt.AssertPrints("12"):
         ip.run_cell("test")
     with tt.AssertPrints("13"):
         ip.run_cell("test")
 
 
 def test_magic_magic():
     """Test %magic"""
     ip = get_ipython()
     with capture_output() as captured:
         ip.run_line_magic("magic", "")
 
     stdout = captured.stdout
     assert "%magic" in stdout
     assert "IPython" in stdout
     assert "Available" in stdout
 
 
 @dec.skipif_not_numpy
 def test_numpy_reset_array_undec():
     "Test '%reset array' functionality"
     _ip.ex("import numpy as np")
     _ip.ex("a = np.empty(2)")
     assert "a" in _ip.user_ns
     _ip.run_line_magic("reset", "-f array")
     assert "a" not in _ip.user_ns
 
 
 def test_reset_out():
     "Test '%reset out' magic"
     _ip.run_cell("parrot = 'dead'", store_history=True)
     # test '%reset -f out', make an Out prompt
     _ip.run_cell("parrot", store_history=True)
     assert "dead" in [_ip.user_ns[x] for x in ("_", "__", "___")]
     _ip.run_line_magic("reset", "-f out")
     assert "dead" not in [_ip.user_ns[x] for x in ("_", "__", "___")]
     assert len(_ip.user_ns["Out"]) == 0
 
 
 def test_reset_in():
     "Test '%reset in' magic"
     # test '%reset -f in'
     _ip.run_cell("parrot", store_history=True)
     assert "parrot" in [_ip.user_ns[x] for x in ("_i", "_ii", "_iii")]
     _ip.run_line_magic("reset", "-f in")
     assert "parrot" not in [_ip.user_ns[x] for x in ("_i", "_ii", "_iii")]
     assert len(set(_ip.user_ns["In"])) == 1
 
 
 def test_reset_dhist():
     "Test '%reset dhist' magic"
     _ip.run_cell("tmp = [d for d in _dh]")  # copy before clearing
     _ip.run_line_magic("cd", os.path.dirname(pytest.__file__))
     _ip.run_line_magic("cd", "-")
     assert len(_ip.user_ns["_dh"]) > 0
     _ip.run_line_magic("reset", "-f dhist")
     assert len(_ip.user_ns["_dh"]) == 0
     _ip.run_cell("_dh = [d for d in tmp]")  # restore
 
 
 def test_reset_in_length():
     "Test that '%reset in' preserves In[] length"
     _ip.run_cell("print 'foo'")
     _ip.run_cell("reset -f in")
     assert len(_ip.user_ns["In"]) == _ip.displayhook.prompt_count + 1
 
 
 class TestResetErrors(TestCase):
+
     def test_reset_redefine(self):
+
         @magics_class
         class KernelMagics(Magics):
-            @line_magic
-            def less(self, shell):
-                pass
+              @line_magic
+              def less(self, shell): pass
 
         _ip.register_magics(KernelMagics)
 
         with self.assertLogs() as cm:
             # hack, we want to just capture logs, but assertLogs fails if not
             # logs get produce.
             # so log one things we ignore.
             import logging as log_mod
-
             log = log_mod.getLogger()
-            log.info("Nothing")
+            log.info('Nothing')
             # end hack.
             _ip.run_cell("reset -f")
 
         assert len(cm.output) == 1
         for out in cm.output:
             assert "Invalid alias" not in out
 
-
 def test_tb_syntaxerror():
     """test %tb after a SyntaxError"""
     ip = get_ipython()
     ip.run_cell("for")
 
     # trap and validate stdout
     save_stdout = sys.stdout
     try:
         sys.stdout = StringIO()
         ip.run_cell("%tb")
         out = sys.stdout.getvalue()
     finally:
         sys.stdout = save_stdout
     # trim output, and only check the last line
     last_line = out.rstrip().splitlines()[-1].strip()
     assert last_line == "SyntaxError: invalid syntax"
 
 
 def test_time():
     ip = get_ipython()
 
     with tt.AssertPrints("Wall time: "):
         ip.run_cell("%time None")
 
-    ip.run_cell("def f(kmjy):\n" "    %time print (2*kmjy)")
+    ip.run_cell("def f(kmjy):\n"
+                "    %time print (2*kmjy)")
 
     with tt.AssertPrints("Wall time: "):
         with tt.AssertPrints("hihi", suppress=False):
             ip.run_cell("f('hi')")
 
 
 # ';' at the end of %time prevents instruction value to be printed.
 # This tests fix for #13837.
 def test_time_no_output_with_semicolon():
     ip = get_ipython()
 
     # Test %time cases
     with tt.AssertPrints(" 123456"):
         with tt.AssertPrints("Wall time: ", suppress=False):
             with tt.AssertPrints("CPU times: ", suppress=False):
                 ip.run_cell("%time 123000+456")
 
     with tt.AssertNotPrints(" 123456"):
         with tt.AssertPrints("Wall time: ", suppress=False):
             with tt.AssertPrints("CPU times: ", suppress=False):
                 ip.run_cell("%time 123000+456;")
 
     with tt.AssertPrints(" 123456"):
         with tt.AssertPrints("Wall time: ", suppress=False):
             with tt.AssertPrints("CPU times: ", suppress=False):
                 ip.run_cell("%time 123000+456 # Comment")
 
     with tt.AssertNotPrints(" 123456"):
         with tt.AssertPrints("Wall time: ", suppress=False):
             with tt.AssertPrints("CPU times: ", suppress=False):
                 ip.run_cell("%time 123000+456; # Comment")
 
     with tt.AssertPrints(" 123456"):
         with tt.AssertPrints("Wall time: ", suppress=False):
             with tt.AssertPrints("CPU times: ", suppress=False):
                 ip.run_cell("%time 123000+456 # ;Comment")
 
     # Test %%time cases
     with tt.AssertPrints("123456"):
         with tt.AssertPrints("Wall time: ", suppress=False):
             with tt.AssertPrints("CPU times: ", suppress=False):
                 ip.run_cell("%%time\n123000+456\n\n\n")
 
     with tt.AssertNotPrints("123456"):
         with tt.AssertPrints("Wall time: ", suppress=False):
             with tt.AssertPrints("CPU times: ", suppress=False):
                 ip.run_cell("%%time\n123000+456;\n\n\n")
 
     with tt.AssertPrints("123456"):
         with tt.AssertPrints("Wall time: ", suppress=False):
             with tt.AssertPrints("CPU times: ", suppress=False):
                 ip.run_cell("%%time\n123000+456  # Comment\n\n\n")
 
     with tt.AssertNotPrints("123456"):
         with tt.AssertPrints("Wall time: ", suppress=False):
             with tt.AssertPrints("CPU times: ", suppress=False):
                 ip.run_cell("%%time\n123000+456;  # Comment\n\n\n")
 
     with tt.AssertPrints("123456"):
         with tt.AssertPrints("Wall time: ", suppress=False):
             with tt.AssertPrints("CPU times: ", suppress=False):
                 ip.run_cell("%%time\n123000+456  # ;Comment\n\n\n")
 
 
 def test_time_last_not_expression():
-    _ip.run_cell("%%time\n" "var_1 = 1\n" "var_2 = 2\n")
-    assert _ip.user_ns["var_1"] == 1
-    del _ip.user_ns["var_1"]
-    assert _ip.user_ns["var_2"] == 2
-    del _ip.user_ns["var_2"]
+    ip.run_cell("%%time\n"
+                "var_1 = 1\n"
+                "var_2 = 2\n")
+    assert ip.user_ns['var_1'] == 1
+    del ip.user_ns['var_1']
+    assert ip.user_ns['var_2'] == 2
+    del ip.user_ns['var_2']
 
 
 @dec.skip_win32
 def test_time2():
     ip = get_ipython()
 
     with tt.AssertPrints("CPU times: user "):
         ip.run_cell("%time None")
 
-
 def test_time3():
     """Erroneous magic function calls, issue gh-3334"""
     ip = get_ipython()
-    ip.user_ns.pop("run", None)
-
-    with tt.AssertNotPrints("not found", channel="stderr"):
-        ip.run_cell("%%time\n" "run = 0\n" "run += 1")
+    ip.user_ns.pop('run', None)
 
+    with tt.AssertNotPrints("not found", channel='stderr'):
+        ip.run_cell("%%time\n"
+                    "run = 0\n"
+                    "run += 1")
 
 def test_multiline_time():
     """Make sure last statement from time return a value."""
     ip = get_ipython()
-    ip.user_ns.pop("run", None)
+    ip.user_ns.pop('run', None)
 
     ip.run_cell(
         dedent(
             """\
         %%time
         a = "ho"
         b = "hey"
         a+b
         """
         )
     )
     assert ip.user_ns_hidden["_"] == "hohey"
 
 
 def test_time_local_ns():
     """
     Test that local_ns is actually global_ns when running a cell magic
     """
     ip = get_ipython()
     ip.run_cell("%%time\n" "myvar = 1")
     assert ip.user_ns["myvar"] == 1
     del ip.user_ns["myvar"]
 
 
 def test_time_microseconds_display():
     """Ensure ASCII is used when necessary"""
     with mock.patch("sys.stdout", io.TextIOWrapper(StringIO(), encoding="utf-8")):
         assert execution._format_time(0.000001) == "1 \u03bcs"
     with mock.patch("sys.stdout", io.TextIOWrapper(StringIO(), encoding="ascii")):
         assert execution._format_time(0.000001) == "1 us"
 
 
 # Test %%capture magic. Added to test issue #13926
 def test_capture():
     ip = get_ipython()
 
     # Test %%capture nominal case
     ip.run_cell("%%capture abc\n1+2")
     with tt.AssertPrints("True", suppress=False):
         ip.run_cell("'abc' in locals()")
     with tt.AssertPrints("True", suppress=False):
         ip.run_cell("'outputs' in dir(abc)")
     with tt.AssertPrints("3", suppress=False):
         ip.run_cell("abc.outputs[0]")
 
     # Test %%capture with ';' at end of expression
     ip.run_cell("%%capture abc\n7+8;")
     with tt.AssertPrints("False", suppress=False):
         ip.run_cell("'abc' in locals()")
 
 
 def test_doctest_mode():
     "Toggle doctest_mode twice, it should be a no-op and run without error"
     _ip.run_line_magic("doctest_mode", "")
     _ip.run_line_magic("doctest_mode", "")
 
 
 def test_parse_options():
     """Tests for basic options parsing in magics."""
     # These are only the most minimal of tests, more should be added later.  At
     # the very least we check that basic text/unicode calls work OK.
     m = DummyMagics(_ip)
     assert m.parse_options("foo", "")[1] == "foo"
     assert m.parse_options("foo", "")[1] == "foo"
 
 
 def test_parse_options_preserve_non_option_string():
     """Test to assert preservation of non-option part of magic-block, while parsing magic options."""
     m = DummyMagics(_ip)
     opts, stmt = m.parse_options(
         " -n1  -r 13 _ = 314 + foo", "n:r:", preserve_non_opts=True
     )
     assert opts == {"n": "1", "r": "13"}
     assert stmt == "_ = 314 + foo"
 
 
 def test_run_magic_preserve_code_block():
     """Test to assert preservation of non-option part of magic-block, while running magic."""
     _ip.user_ns["spaces"] = []
     _ip.run_line_magic(
         "timeit", "-n1 -r1 spaces.append([s.count(' ') for s in ['document']])"
     )
     assert _ip.user_ns["spaces"] == [[0]]
 
 
 def test_dirops():
     """Test various directory handling operations."""
     # curpath = lambda :os.path.splitdrive(os.getcwd())[1].replace('\\','/')
     curpath = os.getcwd
     startdir = os.getcwd()
     ipdir = os.path.realpath(_ip.ipython_dir)
     try:
         _ip.run_line_magic("cd", '"%s"' % ipdir)
         assert curpath() == ipdir
         _ip.run_line_magic("cd", "-")
         assert curpath() == startdir
         _ip.run_line_magic("pushd", '"%s"' % ipdir)
         assert curpath() == ipdir
         _ip.run_line_magic("popd", "")
         assert curpath() == startdir
     finally:
         os.chdir(startdir)
 
 
 def test_cd_force_quiet():
     """Test OSMagics.cd_force_quiet option"""
     _ip.config.OSMagics.cd_force_quiet = True
     osmagics = osm.OSMagics(shell=_ip)
 
     startdir = os.getcwd()
     ipdir = os.path.realpath(_ip.ipython_dir)
 
     try:
         with tt.AssertNotPrints(ipdir):
             osmagics.cd('"%s"' % ipdir)
         with tt.AssertNotPrints(startdir):
-            osmagics.cd("-")
+            osmagics.cd('-')
     finally:
         os.chdir(startdir)
 
 
 def test_xmode():
     # Calling xmode three times should be a no-op
     xmode = _ip.InteractiveTB.mode
     for i in range(4):
         _ip.run_line_magic("xmode", "")
     assert _ip.InteractiveTB.mode == xmode
 
-
 def test_reset_hard():
     monitor = []
-
     class A(object):
         def __del__(self):
             monitor.append(1)
-
         def __repr__(self):
             return "<A instance>"
 
     _ip.user_ns["a"] = A()
     _ip.run_cell("a")
 
     assert monitor == []
     _ip.run_line_magic("reset", "-f")
     assert monitor == [1]
 
-
 class TestXdel(tt.TempFileMixin):
     def test_xdel(self):
         """Test that references from %run are cleared by xdel."""
-        src = (
-            "class A(object):\n"
-            "    monitor = []\n"
-            "    def __del__(self):\n"
-            "        self.monitor.append(1)\n"
-            "a = A()\n"
-        )
+        src = ("class A(object):\n"
+               "    monitor = []\n"
+               "    def __del__(self):\n"
+               "        self.monitor.append(1)\n"
+               "a = A()\n")
         self.mktmp(src)
         # %run creates some hidden references...
         _ip.run_line_magic("run", "%s" % self.fname)
         # ... as does the displayhook.
         _ip.run_cell("a")
 
         monitor = _ip.user_ns["A"].monitor
         assert monitor == []
 
         _ip.run_line_magic("xdel", "a")
 
         # Check that a's __del__ method has been called.
         gc.collect(0)
         assert monitor == [1]
 
-
 def doctest_who():
     """doctest for %who
 
     In [1]: %reset -sf
 
     In [2]: alpha = 123
 
     In [3]: beta = 'beta'
 
     In [4]: %who int
     alpha
 
     In [5]: %who str
     beta
 
     In [6]: %whos
     Variable   Type    Data/Info
     ----------------------------
     alpha      int     123
     beta       str     beta
 
     In [7]: %who_ls
     Out[7]: ['alpha', 'beta']
     """
 
-
 def test_whos():
     """Check that whos is protected against objects where repr() fails."""
-
     class A(object):
         def __repr__(self):
             raise Exception()
-
-    _ip.user_ns["a"] = A()
+    _ip.user_ns['a'] = A()
     _ip.run_line_magic("whos", "")
 
-
 def doctest_precision():
     """doctest for %precision
 
     In [1]: f = get_ipython().display_formatter.formatters['text/plain']
 
     In [2]: %precision 5
     Out[2]: '%.5f'
 
     In [3]: f.float_format
     Out[3]: '%.5f'
 
     In [4]: %precision %e
     Out[4]: '%e'
 
     In [5]: f(3.1415927)
     Out[5]: '3.141593e+00'
     """
 
 
 def test_debug_magic():
     """Test debugging a small code with %debug
 
     In [1]: with PdbTestInput(['c']):
        ...:     %debug print("a b") #doctest: +ELLIPSIS
        ...:
     ...
     ipdb> c
     a b
     In [2]:
     """
 
 
 def test_debug_magic_locals():
     """Test debugging a small code with %debug with locals
 
     In [1]: with PdbTestInput(['c']):
        ...:     def fun():
        ...:         res = 1
        ...:         %debug print(res)
        ...:     fun()
        ...:
     ...
     ipdb> c
     1
     In [2]:
     """
 
-
 def test_psearch():
     with tt.AssertPrints("dict.fromkeys"):
         _ip.run_cell("dict.fr*?")
     with tt.AssertPrints("π.is_integer"):
         _ip.run_cell("π = 3.14;\nπ.is_integ*?")
 
-
 def test_timeit_shlex():
     """test shlex issues with timeit (#1109)"""
     _ip.ex("def f(*a,**kw): pass")
     _ip.run_line_magic("timeit", '-n1 "this is a bug".count(" ")')
     _ip.run_line_magic("timeit", '-r1 -n1 f(" ", 1)')
     _ip.run_line_magic("timeit", '-r1 -n1 f(" ", 1, " ", 2, " ")')
     _ip.run_line_magic("timeit", '-r1 -n1 ("a " + "b")')
     _ip.run_line_magic("timeit", '-r1 -n1 f("a " + "b")')
     _ip.run_line_magic("timeit", '-r1 -n1 f("a " + "b ")')
 
 
 def test_timeit_special_syntax():
     "Test %%timeit with IPython special syntax"
-
     @register_line_magic
     def lmagic(line):
         ip = get_ipython()
-        ip.user_ns["lmagic_out"] = line
+        ip.user_ns['lmagic_out'] = line
 
     # line mode test
     _ip.run_line_magic("timeit", "-n1 -r1 %lmagic my line")
     assert _ip.user_ns["lmagic_out"] == "my line"
     # cell mode test
     _ip.run_cell_magic("timeit", "-n1 -r1", "%lmagic my line2")
     assert _ip.user_ns["lmagic_out"] == "my line2"
 
 
 def test_timeit_return():
     """
     test whether timeit -o return object
     """
 
-    res = _ip.run_line_magic("timeit", "-n10 -r10 -o 1")
-    assert res is not None
-
+    res = _ip.run_line_magic('timeit','-n10 -r10 -o 1')
+    assert(res is not None)
 
 def test_timeit_quiet():
     """
     test quiet option of timeit magic
     """
     with tt.AssertNotPrints("loops"):
         _ip.run_cell("%timeit -n1 -r1 -q 1")
 
-
 def test_timeit_return_quiet():
     with tt.AssertNotPrints("loops"):
-        res = _ip.run_line_magic("timeit", "-n1 -r1 -q -o 1")
-    assert res is not None
-
+        res = _ip.run_line_magic('timeit', '-n1 -r1 -q -o 1')
+    assert (res is not None)
 
 def test_timeit_invalid_return():
     with pytest.raises(SyntaxError):
-        _ip.run_line_magic("timeit", "return")
-
+        _ip.run_line_magic('timeit', 'return')
 
 @dec.skipif(execution.profile is None)
 def test_prun_special_syntax():
     "Test %%prun with IPython special syntax"
-
     @register_line_magic
     def lmagic(line):
         ip = get_ipython()
-        ip.user_ns["lmagic_out"] = line
+        ip.user_ns['lmagic_out'] = line
 
     # line mode test
     _ip.run_line_magic("prun", "-q %lmagic my line")
     assert _ip.user_ns["lmagic_out"] == "my line"
     # cell mode test
     _ip.run_cell_magic("prun", "-q", "%lmagic my line2")
     assert _ip.user_ns["lmagic_out"] == "my line2"
 
 
 @dec.skipif(execution.profile is None)
 def test_prun_quotes():
     "Test that prun does not clobber string escapes (GH #1302)"
     _ip.run_line_magic("prun", r"-q x = '\t'")
     assert _ip.user_ns["x"] == "\t"
 
 
 def test_extension():
     # Debugging information for failures of this test
-    print("sys.path:")
+    print('sys.path:')
     for p in sys.path:
-        print(" ", p)
-    print("CWD", os.getcwd())
+        print(' ', p)
+    print('CWD', os.getcwd())
 
     pytest.raises(ImportError, _ip.run_line_magic, "load_ext", "daft_extension")
     daft_path = os.path.join(os.path.dirname(__file__), "daft_extension")
     sys.path.insert(0, daft_path)
     try:
-        _ip.user_ns.pop("arq", None)
-        invalidate_caches()  # Clear import caches
+        _ip.user_ns.pop('arq', None)
+        invalidate_caches()   # Clear import caches
         _ip.run_line_magic("load_ext", "daft_extension")
         assert _ip.user_ns["arq"] == 185
         _ip.run_line_magic("unload_ext", "daft_extension")
-        assert "arq" not in _ip.user_ns
+        assert 'arq' not in _ip.user_ns
     finally:
         sys.path.remove(daft_path)
 
 
 def test_notebook_export_json():
     pytest.importorskip("nbformat")
     _ip = get_ipython()
-    _ip.history_manager.reset()  # Clear any existing history.
+    _ip.history_manager.reset()   # Clear any existing history.
     cmds = ["a=1", "def b():\n  return a**2", "print('noël, été', b())"]
     for i, cmd in enumerate(cmds, start=1):
         _ip.history_manager.store_inputs(i, cmd)
     with TemporaryDirectory() as td:
         outfile = os.path.join(td, "nb.ipynb")
         _ip.run_line_magic("notebook", "%s" % outfile)
 
 
 class TestEnv(TestCase):
+
     def test_env(self):
         env = _ip.run_line_magic("env", "")
         self.assertTrue(isinstance(env, dict))
 
     def test_env_secret(self):
         env = _ip.run_line_magic("env", "")
         hidden = "<hidden>"
         with mock.patch.dict(
             os.environ,
             {
                 "API_KEY": "abc123",
                 "SECRET_THING": "ssshhh",
                 "JUPYTER_TOKEN": "",
-                "VAR": "abc",
-            },
+                "VAR": "abc"
+            }
         ):
             env = _ip.run_line_magic("env", "")
         assert env["API_KEY"] == hidden
         assert env["SECRET_THING"] == hidden
         assert env["JUPYTER_TOKEN"] == hidden
         assert env["VAR"] == "abc"
 
     def test_env_get_set_simple(self):
         env = _ip.run_line_magic("env", "var val1")
         self.assertEqual(env, None)
         self.assertEqual(os.environ["var"], "val1")
         self.assertEqual(_ip.run_line_magic("env", "var"), "val1")
         env = _ip.run_line_magic("env", "var=val2")
         self.assertEqual(env, None)
-        self.assertEqual(os.environ["var"], "val2")
+        self.assertEqual(os.environ['var'], 'val2')
 
     def test_env_get_set_complex(self):
         env = _ip.run_line_magic("env", "var 'val1 '' 'val2")
         self.assertEqual(env, None)
-        self.assertEqual(os.environ["var"], "'val1 '' 'val2")
+        self.assertEqual(os.environ['var'], "'val1 '' 'val2")
         self.assertEqual(_ip.run_line_magic("env", "var"), "'val1 '' 'val2")
         env = _ip.run_line_magic("env", 'var=val2 val3="val4')
         self.assertEqual(env, None)
-        self.assertEqual(os.environ["var"], 'val2 val3="val4')
+        self.assertEqual(os.environ['var'], 'val2 val3="val4')
 
     def test_env_set_bad_input(self):
         self.assertRaises(UsageError, lambda: _ip.run_line_magic("set_env", "var"))
 
     def test_env_set_whitespace(self):
         self.assertRaises(UsageError, lambda: _ip.run_line_magic("env", "var A=B"))
 
 
 class CellMagicTestCase(TestCase):
+
     def check_ident(self, magic):
         # Manually called, we get the result
         out = _ip.run_cell_magic(magic, "a", "b")
         assert out == ("a", "b")
         # Via run_cell, it goes into the user's namespace via displayhook
         _ip.run_cell("%%" + magic + " c\nd\n")
         assert _ip.user_ns["_"] == ("c", "d\n")
 
     def test_cell_magic_func_deco(self):
         "Cell magic using simple decorator"
-
         @register_cell_magic
         def cellm(line, cell):
             return line, cell
 
-        self.check_ident("cellm")
+        self.check_ident('cellm')
 
     def test_cell_magic_reg(self):
         "Cell magic manually registered"
-
         def cellm(line, cell):
             return line, cell
 
-        _ip.register_magic_function(cellm, "cell", "cellm2")
-        self.check_ident("cellm2")
+        _ip.register_magic_function(cellm, 'cell', 'cellm2')
+        self.check_ident('cellm2')
 
     def test_cell_magic_class(self):
         "Cell magics declared via a class"
-
         @magics_class
         class MyMagics(Magics):
+
             @cell_magic
             def cellm3(self, line, cell):
                 return line, cell
 
         _ip.register_magics(MyMagics)
-        self.check_ident("cellm3")
+        self.check_ident('cellm3')
 
     def test_cell_magic_class2(self):
         "Cell magics declared via a class, #2"
-
         @magics_class
         class MyMagics2(Magics):
-            @cell_magic("cellm4")
+
+            @cell_magic('cellm4')
             def cellm33(self, line, cell):
                 return line, cell
 
         _ip.register_magics(MyMagics2)
-        self.check_ident("cellm4")
+        self.check_ident('cellm4')
         # Check that nothing is registered as 'cellm33'
-        c33 = _ip.find_cell_magic("cellm33")
-        assert c33 is None
-
+        c33 = _ip.find_cell_magic('cellm33')
+        assert c33 == None
 
 def test_file():
     """Basic %%writefile"""
     ip = get_ipython()
     with TemporaryDirectory() as td:
         fname = os.path.join(td, "file1")
         ip.run_cell_magic(
             "writefile",
             fname,
             "\n".join(
                 [
                     "line1",
                     "line2",
                 ]
             ),
         )
         s = Path(fname).read_text(encoding="utf-8")
         assert "line1\n" in s
         assert "line2" in s
 
 
 @dec.skip_win32
 def test_file_single_quote():
     """Basic %%writefile with embedded single quotes"""
     ip = get_ipython()
     with TemporaryDirectory() as td:
         fname = os.path.join(td, "'file1'")
         ip.run_cell_magic(
             "writefile",
             fname,
             "\n".join(
                 [
                     "line1",
                     "line2",
                 ]
             ),
         )
         s = Path(fname).read_text(encoding="utf-8")
         assert "line1\n" in s
         assert "line2" in s
 
 
 @dec.skip_win32
 def test_file_double_quote():
     """Basic %%writefile with embedded double quotes"""
     ip = get_ipython()
     with TemporaryDirectory() as td:
         fname = os.path.join(td, '"file1"')
         ip.run_cell_magic(
             "writefile",
             fname,
             "\n".join(
                 [
                     "line1",
                     "line2",
                 ]
             ),
         )
         s = Path(fname).read_text(encoding="utf-8")
         assert "line1\n" in s
         assert "line2" in s
 
 
 def test_file_var_expand():
     """%%writefile $filename"""
     ip = get_ipython()
     with TemporaryDirectory() as td:
         fname = os.path.join(td, "file1")
         ip.user_ns["filename"] = fname
         ip.run_cell_magic(
             "writefile",
             "$filename",
             "\n".join(
                 [
                     "line1",
                     "line2",
                 ]
             ),
         )
         s = Path(fname).read_text(encoding="utf-8")
         assert "line1\n" in s
         assert "line2" in s
 
 
 def test_file_unicode():
     """%%writefile with unicode cell"""
     ip = get_ipython()
     with TemporaryDirectory() as td:
-        fname = os.path.join(td, "file1")
-        ip.run_cell_magic(
-            "writefile",
-            fname,
-            "\n".join(
-                [
-                    "liné1",
-                    "liné2",
-                ]
-            ),
-        )
-        with io.open(fname, encoding="utf-8") as f:
+        fname = os.path.join(td, 'file1')
+        ip.run_cell_magic("writefile", fname, u'\n'.join([
+            u'liné1',
+            u'liné2',
+        ]))
+        with io.open(fname, encoding='utf-8') as f:
             s = f.read()
         assert "liné1\n" in s
         assert "liné2" in s
 
 
 def test_file_amend():
     """%%writefile -a amends files"""
     ip = get_ipython()
     with TemporaryDirectory() as td:
         fname = os.path.join(td, "file2")
         ip.run_cell_magic(
             "writefile",
             fname,
             "\n".join(
                 [
                     "line1",
                     "line2",
                 ]
             ),
         )
         ip.run_cell_magic(
             "writefile",
             "-a %s" % fname,
             "\n".join(
                 [
                     "line3",
                     "line4",
                 ]
             ),
         )
         s = Path(fname).read_text(encoding="utf-8")
         assert "line1\n" in s
         assert "line3\n" in s
 
 
 def test_file_spaces():
     """%%file with spaces in filename"""
     ip = get_ipython()
-    with TemporaryWorkingDirectory():
+    with TemporaryWorkingDirectory() as td:
         fname = "file name"
         ip.run_cell_magic(
             "file",
             '"%s"' % fname,
             "\n".join(
                 [
                     "line1",
                     "line2",
                 ]
             ),
         )
         s = Path(fname).read_text(encoding="utf-8")
         assert "line1\n" in s
         assert "line2" in s
 
 
 def test_script_config():
     ip = get_ipython()
-    ip.config.ScriptMagics.script_magics = ["whoda"]
+    ip.config.ScriptMagics.script_magics = ['whoda']
     sm = script.ScriptMagics(shell=ip)
     assert "whoda" in sm.magics["cell"]
 
 
 def test_script_out():
     ip = get_ipython()
     ip.run_cell_magic("script", f"--out output {sys.executable}", "print('hi')")
     assert ip.user_ns["output"].strip() == "hi"
 
 
 def test_script_err():
     ip = get_ipython()
     ip.run_cell_magic(
         "script",
         f"--err error {sys.executable}",
         "import sys; print('hello', file=sys.stderr)",
     )
     assert ip.user_ns["error"].strip() == "hello"
 
 
 def test_script_out_err():
     ip = get_ipython()
     ip.run_cell_magic(
         "script",
         f"--out output --err error {sys.executable}",
         "\n".join(
             [
                 "import sys",
                 "print('hi')",
                 "print('hello', file=sys.stderr)",
             ]
         ),
     )
     assert ip.user_ns["output"].strip() == "hi"
     assert ip.user_ns["error"].strip() == "hello"
 
 
 async def test_script_bg_out():
     ip = get_ipython()
     ip.run_cell_magic("script", f"--bg --out output {sys.executable}", "print('hi')")
     assert (await ip.user_ns["output"].read()).strip() == b"hi"
     assert ip.user_ns["output"].at_eof()
 
 
 async def test_script_bg_err():
     ip = get_ipython()
     ip.run_cell_magic(
         "script",
         f"--bg --err error {sys.executable}",
         "import sys; print('hello', file=sys.stderr)",
     )
     assert (await ip.user_ns["error"].read()).strip() == b"hello"
     assert ip.user_ns["error"].at_eof()
 
 
 async def test_script_bg_out_err():
     ip = get_ipython()
     ip.run_cell_magic(
         "script",
         f"--bg --out output --err error {sys.executable}",
         "\n".join(
             [
                 "import sys",
                 "print('hi')",
                 "print('hello', file=sys.stderr)",
             ]
         ),
     )
     assert (await ip.user_ns["output"].read()).strip() == b"hi"
     assert (await ip.user_ns["error"].read()).strip() == b"hello"
     assert ip.user_ns["output"].at_eof()
     assert ip.user_ns["error"].at_eof()
 
 
 async def test_script_bg_proc():
     ip = get_ipython()
     ip.run_cell_magic(
         "script",
         f"--bg --out output --proc p {sys.executable}",
         "\n".join(
             [
                 "import sys",
                 "print('hi')",
                 "print('hello', file=sys.stderr)",
             ]
         ),
     )
     p = ip.user_ns["p"]
     await p.wait()
     assert p.returncode == 0
     assert (await p.stdout.read()).strip() == b"hi"
     # not captured, so empty
     assert (await p.stderr.read()) == b""
     assert p.stdout.at_eof()
     assert p.stderr.at_eof()
 
 
 def test_script_defaults():
     ip = get_ipython()
-    for cmd in ["sh", "bash", "perl", "ruby"]:
+    for cmd in ['sh', 'bash', 'perl', 'ruby']:
         try:
             find_cmd(cmd)
         except Exception:
             pass
         else:
             assert cmd in ip.magics_manager.magics["cell"]
 
 
 @magics_class
 class FooFoo(Magics):
     """class with both %foo and %%foo magics"""
-
-    @line_magic("foo")
+    @line_magic('foo')
     def line_foo(self, line):
         "I am line foo"
         pass
 
     @cell_magic("foo")
     def cell_foo(self, line, cell):
         "I am cell foo, not line foo"
         pass
 
-
 def test_line_cell_info():
     """%%foo and %foo magics are distinguishable to inspect"""
     ip = get_ipython()
     ip.magics_manager.register(FooFoo)
     oinfo = ip.object_inspect("foo")
     assert oinfo["found"] is True
     assert oinfo["ismagic"] is True
 
     oinfo = ip.object_inspect("%%foo")
     assert oinfo["found"] is True
     assert oinfo["ismagic"] is True
     assert oinfo["docstring"] == FooFoo.cell_foo.__doc__
 
     oinfo = ip.object_inspect("%foo")
     assert oinfo["found"] is True
     assert oinfo["ismagic"] is True
     assert oinfo["docstring"] == FooFoo.line_foo.__doc__
 
 
 def test_multiple_magics():
     ip = get_ipython()
     foo1 = FooFoo(ip)
     foo2 = FooFoo(ip)
     mm = ip.magics_manager
     mm.register(foo1)
     assert mm.magics["line"]["foo"].__self__ is foo1
     mm.register(foo2)
     assert mm.magics["line"]["foo"].__self__ is foo2
 
 
 def test_alias_magic():
     """Test %alias_magic."""
     ip = get_ipython()
     mm = ip.magics_manager
 
     # Basic operation: both cell and line magics are created, if possible.
     ip.run_line_magic("alias_magic", "timeit_alias timeit")
     assert "timeit_alias" in mm.magics["line"]
     assert "timeit_alias" in mm.magics["cell"]
 
     # --cell is specified, line magic not created.
     ip.run_line_magic("alias_magic", "--cell timeit_cell_alias timeit")
     assert "timeit_cell_alias" not in mm.magics["line"]
     assert "timeit_cell_alias" in mm.magics["cell"]
 
     # Test that line alias is created successfully.
     ip.run_line_magic("alias_magic", "--line env_alias env")
     assert ip.run_line_magic("env", "") == ip.run_line_magic("env_alias", "")
 
     # Test that line alias with parameters passed in is created successfully.
     ip.run_line_magic(
         "alias_magic", "--line history_alias history --params " + shlex.quote("3")
     )
     assert "history_alias" in mm.magics["line"]
 
 
 def test_save():
     """Test %save."""
     ip = get_ipython()
-    ip.history_manager.reset()  # Clear any existing history.
+    ip.history_manager.reset()   # Clear any existing history.
     cmds = ["a=1", "def b():\n  return a**2", "print(a, b())"]
     for i, cmd in enumerate(cmds, start=1):
         ip.history_manager.store_inputs(i, cmd)
     with TemporaryDirectory() as tmpdir:
         file = os.path.join(tmpdir, "testsave.py")
         ip.run_line_magic("save", "%s 1-10" % file)
         content = Path(file).read_text(encoding="utf-8")
         assert content.count(cmds[0]) == 1
         assert "coding: utf-8" in content
         ip.run_line_magic("save", "-a %s 1-10" % file)
         content = Path(file).read_text(encoding="utf-8")
         assert content.count(cmds[0]) == 2
         assert "coding: utf-8" in content
 
 
 def test_save_with_no_args():
     ip = get_ipython()
     ip.history_manager.reset()  # Clear any existing history.
     cmds = ["a=1", "def b():\n    return a**2", "print(a, b())", "%save"]
     for i, cmd in enumerate(cmds, start=1):
         ip.history_manager.store_inputs(i, cmd)
 
     with TemporaryDirectory() as tmpdir:
         path = os.path.join(tmpdir, "testsave.py")
         ip.run_line_magic("save", path)
         content = Path(path).read_text(encoding="utf-8")
         expected_content = dedent(
             """\
             # coding: utf-8
             a=1
             def b():
                 return a**2
             print(a, b())
             """
         )
         assert content == expected_content
 
 
 def test_store():
     """Test %store."""
     ip = get_ipython()
-    ip.run_line_magic("load_ext", "storemagic")
+    ip.run_line_magic('load_ext', 'storemagic')
 
     # make sure the storage is empty
     ip.run_line_magic("store", "-z")
     ip.user_ns["var"] = 42
     ip.run_line_magic("store", "var")
     ip.user_ns["var"] = 39
     ip.run_line_magic("store", "-r")
     assert ip.user_ns["var"] == 42
 
     ip.run_line_magic("store", "-d var")
     ip.user_ns["var"] = 39
     ip.run_line_magic("store", "-r")
     assert ip.user_ns["var"] == 39
 
 
-def _run_edit_test(
-    arg_s, exp_filename=None, exp_lineno=-1, exp_contents=None, exp_is_temp=None
-):
+def _run_edit_test(arg_s, exp_filename=None,
+                        exp_lineno=-1,
+                        exp_contents=None,
+                        exp_is_temp=None):
     ip = get_ipython()
     M = code.CodeMagics(ip)
-    last_call = ["", ""]
-    opts, args = M.parse_options(arg_s, "prxn:")
+    last_call = ['','']
+    opts,args = M.parse_options(arg_s,'prxn:')
     filename, lineno, is_temp = M._find_edit_target(ip, args, opts, last_call)
 
     if exp_filename is not None:
         assert exp_filename == filename
     if exp_contents is not None:
-        with io.open(filename, "r", encoding="utf-8") as f:
+        with io.open(filename, 'r', encoding='utf-8') as f:
             contents = f.read()
         assert exp_contents == contents
     if exp_lineno != -1:
         assert exp_lineno == lineno
     if exp_is_temp is not None:
         assert exp_is_temp == is_temp
 
 
 def test_edit_interactive():
     """%edit on interactively defined objects"""
     ip = get_ipython()
     n = ip.execution_count
     ip.run_cell("def foo(): return 1", store_history=True)
 
     with pytest.raises(code.InteractivelyDefined) as e:
         _run_edit_test("foo")
     assert e.value.index == n
 
 
 def test_edit_cell():
     """%edit [cell id]"""
     ip = get_ipython()
 
     ip.run_cell("def foo(): return 1", store_history=True)
 
     # test
-    _run_edit_test("1", exp_contents=ip.user_ns["In"][1], exp_is_temp=True)
-
+    _run_edit_test("1", exp_contents=ip.user_ns['In'][1], exp_is_temp=True)
 
 def test_edit_fname():
     """%edit file"""
     # test
     _run_edit_test("test file.py", exp_filename="test file.py")
 
-
 def test_bookmark():
     ip = get_ipython()
-    ip.run_line_magic("bookmark", "bmname")
-    with tt.AssertPrints("bmname"):
-        ip.run_line_magic("bookmark", "-l")
-    ip.run_line_magic("bookmark", "-d bmname")
-
+    ip.run_line_magic('bookmark', 'bmname')
+    with tt.AssertPrints('bmname'):
+        ip.run_line_magic('bookmark', '-l')
+    ip.run_line_magic('bookmark', '-d bmname')
 
 def test_ls_magic():
     ip = get_ipython()
-    json_formatter = ip.display_formatter.formatters["application/json"]
+    json_formatter = ip.display_formatter.formatters['application/json']
     json_formatter.enabled = True
     lsmagic = ip.run_line_magic("lsmagic", "")
     with warnings.catch_warnings(record=True) as w:
         j = json_formatter(lsmagic)
     assert sorted(j) == ["cell", "line"]
     assert w == []  # no warnings
 
 
 def test_strip_initial_indent():
     def sii(s):
         lines = s.splitlines()
-        return "\n".join(code.strip_initial_indent(lines))
+        return '\n'.join(code.strip_initial_indent(lines))
 
     assert sii("  a = 1\nb = 2") == "a = 1\nb = 2"
     assert sii("  a\n    b\nc") == "a\n  b\nc"
     assert sii("a\n  b") == "a\n  b"
 
-
 def test_logging_magic_quiet_from_arg():
     _ip.config.LoggingMagics.quiet = False
     lm = logging.LoggingMagics(shell=_ip)
     with TemporaryDirectory() as td:
         try:
             with tt.AssertNotPrints(re.compile("Activating.*")):
-                lm.logstart("-q {}".format(os.path.join(td, "quiet_from_arg.log")))
+                lm.logstart('-q {}'.format(
+                        os.path.join(td, "quiet_from_arg.log")))
         finally:
             _ip.logger.logstop()
 
-
 def test_logging_magic_quiet_from_config():
     _ip.config.LoggingMagics.quiet = True
     lm = logging.LoggingMagics(shell=_ip)
     with TemporaryDirectory() as td:
         try:
             with tt.AssertNotPrints(re.compile("Activating.*")):
                 lm.logstart(os.path.join(td, "quiet_from_config.log"))
         finally:
             _ip.logger.logstop()
 
 
 def test_logging_magic_not_quiet():
     _ip.config.LoggingMagics.quiet = False
     lm = logging.LoggingMagics(shell=_ip)
     with TemporaryDirectory() as td:
         try:
             with tt.AssertPrints(re.compile("Activating.*")):
                 lm.logstart(os.path.join(td, "not_quiet.log"))
         finally:
             _ip.logger.logstop()
 
 
 def test_time_no_var_expand():
     _ip.user_ns["a"] = 5
     _ip.user_ns["b"] = []
     _ip.run_line_magic("time", 'b.append("{a}")')
     assert _ip.user_ns["b"] == ["{a}"]
 
 
 # this is slow, put at the end for local testing.
 def test_timeit_arguments():
     "Test valid timeit arguments, should not cause SyntaxError (GH #1269)"
     _ip.run_line_magic("timeit", "-n1 -r1 a=('#')")
 
 
 MINIMAL_LAZY_MAGIC = """
 from IPython.core.magic import (
     Magics,
     magics_class,
     line_magic,
     cell_magic,
 )
 
 
 @magics_class
 class LazyMagics(Magics):
     @line_magic
     def lazy_line(self, line):
         print("Lazy Line")
 
     @cell_magic
     def lazy_cell(self, line, cell):
         print("Lazy Cell")
 
 
 def load_ipython_extension(ipython):
     ipython.register_magics(LazyMagics)
 """
 
 
 def test_lazy_magics():
     with pytest.raises(UsageError):
-        _ip.run_line_magic("lazy_line", "")
+        ip.run_line_magic("lazy_line", "")
+
+    startdir = os.getcwd()
 
     with TemporaryDirectory() as tmpdir:
         with prepended_to_syspath(tmpdir):
             ptempdir = Path(tmpdir)
             tf = ptempdir / "lazy_magic_module.py"
             tf.write_text(MINIMAL_LAZY_MAGIC)
-            _ip.magics_manager.register_lazy("lazy_line", Path(tf.name).name[:-3])
+            ip.magics_manager.register_lazy("lazy_line", Path(tf.name).name[:-3])
             with tt.AssertPrints("Lazy Line"):
-                _ip.run_line_magic("lazy_line", "")
+                ip.run_line_magic("lazy_line", "")
 
 
 TEST_MODULE = """
 print('Loaded my_tmp')
 if __name__ == "__main__":
     print('I just ran a script')
 """
 
-
 def test_run_module_from_import_hook():
     "Test that a module can be loaded via an import hook"
     with TemporaryDirectory() as tmpdir:
         fullpath = os.path.join(tmpdir, "my_tmp.py")
         Path(fullpath).write_text(TEST_MODULE, encoding="utf-8")
 
         import importlib.abc
         import importlib.util
 
         class MyTempImporter(importlib.abc.MetaPathFinder, importlib.abc.SourceLoader):
             def find_spec(self, fullname, path, target=None):
                 if fullname == "my_tmp":
                     return importlib.util.spec_from_loader(fullname, self)
 
             def get_filename(self, fullname):
                 assert fullname == "my_tmp"
                 return fullpath
 
             def get_data(self, path):
                 assert Path(path).samefile(fullpath)
                 return Path(fullpath).read_text(encoding="utf-8")
 
         sys.meta_path.insert(0, MyTempImporter())
 
         with capture_output() as captured:
             _ip.run_line_magic("run", "-m my_tmp")
             _ip.run_cell("import my_tmp")
 
         output = "Loaded my_tmp\nI just ran a script\nLoaded my_tmp\n"
         assert output == captured.stdout
 
         sys.meta_path.pop(0)

IPython/core/tests/test_paths.py

@@ -1,202 +1,202
 import errno
 import os
 import shutil
 import tempfile
 import warnings
 from unittest.mock import patch
 
 from tempfile import TemporaryDirectory
 from testpath import assert_isdir, assert_isfile, modified_env
 
 from IPython import paths
 from IPython.testing.decorators import skip_win32
 
 TMP_TEST_DIR = os.path.realpath(tempfile.mkdtemp())
 HOME_TEST_DIR = os.path.join(TMP_TEST_DIR, "home_test_dir")
 XDG_TEST_DIR = os.path.join(HOME_TEST_DIR, "xdg_test_dir")
 XDG_CACHE_DIR = os.path.join(HOME_TEST_DIR, "xdg_cache_dir")
 IP_TEST_DIR = os.path.join(HOME_TEST_DIR,'.ipython')
 
 def setup_module():
     """Setup testenvironment for the module:
 
             - Adds dummy home dir tree
     """
     # Do not mask exceptions here.  In particular, catching WindowsError is a
     # problem because that exception is only defined on Windows...
     os.makedirs(IP_TEST_DIR)
     os.makedirs(os.path.join(XDG_TEST_DIR, 'ipython'))
     os.makedirs(os.path.join(XDG_CACHE_DIR, 'ipython'))
 
 
 def teardown_module():
     """Teardown testenvironment for the module:
 
             - Remove dummy home dir tree
     """
     # Note: we remove the parent test dir, which is the root of all test
     # subdirs we may have created.  Use shutil instead of os.removedirs, so
     # that non-empty directories are all recursively removed.
     shutil.rmtree(TMP_TEST_DIR)
 
 def patch_get_home_dir(dirpath):
     return patch.object(paths, 'get_home_dir', return_value=dirpath)
 
 
 def test_get_ipython_dir_1():
     """test_get_ipython_dir_1, Testcase to see if we can call get_ipython_dir without Exceptions."""
     env_ipdir = os.path.join("someplace", ".ipython")
     with patch.object(paths, '_writable_dir', return_value=True), \
             modified_env({'IPYTHONDIR': env_ipdir}):
         ipdir = paths.get_ipython_dir()
 
     assert ipdir == env_ipdir
 
 def test_get_ipython_dir_2():
     """test_get_ipython_dir_2, Testcase to see if we can call get_ipython_dir without Exceptions."""
     with patch_get_home_dir('someplace'), \
             patch.object(paths, 'get_xdg_dir', return_value=None), \
             patch.object(paths, '_writable_dir', return_value=True), \
             patch('os.name', "posix"), \
             modified_env({'IPYTHON_DIR': None,
                           'IPYTHONDIR': None,
                           'XDG_CONFIG_HOME': None
                          }):
         ipdir = paths.get_ipython_dir()
 
     assert ipdir == os.path.join("someplace", ".ipython")
 
 def test_get_ipython_dir_3():
     """test_get_ipython_dir_3, use XDG if defined and exists, and .ipython doesn't exist."""
     tmphome = TemporaryDirectory()
     try:
         with patch_get_home_dir(tmphome.name), \
                 patch('os.name', 'posix'), \
                 modified_env({
                     'IPYTHON_DIR': None,
                     'IPYTHONDIR': None,
                     'XDG_CONFIG_HOME': XDG_TEST_DIR,
                 }), warnings.catch_warnings(record=True) as w:
             ipdir = paths.get_ipython_dir()
 
         assert ipdir == os.path.join(tmphome.name, XDG_TEST_DIR, "ipython")
         assert len(w) == 0
     finally:
         tmphome.cleanup()
 
 def test_get_ipython_dir_4():
     """test_get_ipython_dir_4, warn if XDG and home both exist."""
     with patch_get_home_dir(HOME_TEST_DIR), \
             patch('os.name', 'posix'):
         try:
             os.mkdir(os.path.join(XDG_TEST_DIR, 'ipython'))
         except OSError as e:
             if e.errno != errno.EEXIST:
                 raise
 
 
         with modified_env({
             'IPYTHON_DIR': None,
             'IPYTHONDIR': None,
             'XDG_CONFIG_HOME': XDG_TEST_DIR,
         }), warnings.catch_warnings(record=True) as w:
-            _ipdir = paths.get_ipython_dir()
+            ipdir = paths.get_ipython_dir()
 
         assert len(w) == 1
         assert "Ignoring" in str(w[0])
 
 
 def test_get_ipython_dir_5():
     """test_get_ipython_dir_5, use .ipython if exists and XDG defined, but doesn't exist."""
     with patch_get_home_dir(HOME_TEST_DIR), \
             patch('os.name', 'posix'):
         try:
             os.rmdir(os.path.join(XDG_TEST_DIR, 'ipython'))
         except OSError as e:
             if e.errno != errno.ENOENT:
                 raise
 
         with modified_env({
             'IPYTHON_DIR': None,
             'IPYTHONDIR': None,
             'XDG_CONFIG_HOME': XDG_TEST_DIR,
         }):
             ipdir = paths.get_ipython_dir()
 
         assert ipdir == IP_TEST_DIR
 
 def test_get_ipython_dir_6():
     """test_get_ipython_dir_6, use home over XDG if defined and neither exist."""
     xdg = os.path.join(HOME_TEST_DIR, 'somexdg')
     os.mkdir(xdg)
     shutil.rmtree(os.path.join(HOME_TEST_DIR, '.ipython'))
     print(paths._writable_dir)
     with patch_get_home_dir(HOME_TEST_DIR), \
             patch.object(paths, 'get_xdg_dir', return_value=xdg), \
             patch('os.name', 'posix'), \
             modified_env({
                 'IPYTHON_DIR': None,
                 'IPYTHONDIR': None,
                 'XDG_CONFIG_HOME': None,
             }), warnings.catch_warnings(record=True) as w:
         ipdir = paths.get_ipython_dir()
 
     assert ipdir == os.path.join(HOME_TEST_DIR, ".ipython")
     assert len(w) == 0
 
 def test_get_ipython_dir_7():
     """test_get_ipython_dir_7, test home directory expansion on IPYTHONDIR"""
     home_dir = os.path.normpath(os.path.expanduser('~'))
     with modified_env({'IPYTHONDIR': os.path.join('~', 'somewhere')}), \
             patch.object(paths, '_writable_dir', return_value=True):
         ipdir = paths.get_ipython_dir()
     assert ipdir == os.path.join(home_dir, "somewhere")
 
 
 @skip_win32
 def test_get_ipython_dir_8():
     """test_get_ipython_dir_8, test / home directory"""
     if not os.access("/", os.W_OK):
         # test only when HOME directory actually writable
         return
 
     with (
         patch.object(paths, "_writable_dir", lambda path: bool(path)),
         patch.object(paths, "get_xdg_dir", return_value=None),
         modified_env(
             {
                 "IPYTHON_DIR": None,
                 "IPYTHONDIR": None,
                 "HOME": "/",
             }
         ),
     ):
         assert paths.get_ipython_dir() == "/.ipython"
 
 
 def test_get_ipython_cache_dir():
     with modified_env({'HOME': HOME_TEST_DIR}):
         if os.name == "posix":
             # test default
             os.makedirs(os.path.join(HOME_TEST_DIR, ".cache"))
             with modified_env({'XDG_CACHE_HOME': None}):
                 ipdir = paths.get_ipython_cache_dir()
             assert os.path.join(HOME_TEST_DIR, ".cache", "ipython") == ipdir
             assert_isdir(ipdir)
 
             # test env override
             with modified_env({"XDG_CACHE_HOME": XDG_CACHE_DIR}):
                 ipdir = paths.get_ipython_cache_dir()
             assert_isdir(ipdir)
             assert ipdir == os.path.join(XDG_CACHE_DIR, "ipython")
         else:
             assert paths.get_ipython_cache_dir() == paths.get_ipython_dir()
 
 def test_get_ipython_package_dir():
     ipdir = paths.get_ipython_package_dir()
     assert_isdir(ipdir)
 
 
 def test_get_ipython_module_path():
     ipapp_path = paths.get_ipython_module_path('IPython.terminal.ipapp')
     assert_isfile(ipapp_path)

IPython/core/tests/test_prefilter.py

@@ -1,148 +1,149
 """Tests for input manipulation machinery."""
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
+import pytest
 
 from IPython.core.prefilter import AutocallChecker
 
 #-----------------------------------------------------------------------------
 # Tests
 #-----------------------------------------------------------------------------
 
 def test_prefilter():
     """Test user input conversions"""
 
     # pairs of (raw, expected correct) input
     pairs = [ ('2+2','2+2'),
              ]
 
     for raw, correct in pairs:
         assert ip.prefilter(raw) == correct
 
 def test_prefilter_shadowed():
     def dummy_magic(line): pass
 
     prev_automagic_state = ip.automagic
     ip.automagic = True
     ip.autocall = 0
 
     try:
         # These should not be transformed - they are shadowed by other names
         for name in ['if', 'zip', 'get_ipython']: # keyword, builtin, global
             ip.register_magic_function(dummy_magic, magic_name=name)
             res = ip.prefilter(name + " foo")
             assert res == name + " foo"
             del ip.magics_manager.magics["line"][name]
 
         # These should be transformed
         for name in ['fi', 'piz', 'nohtypi_teg']:
             ip.register_magic_function(dummy_magic, magic_name=name)
             res = ip.prefilter(name + " foo")
             assert res != name + " foo"
             del ip.magics_manager.magics["line"][name]
 
     finally:
         ip.automagic = prev_automagic_state
 
 def test_autocall_binops():
     """See https://github.com/ipython/ipython/issues/81"""
     ip.run_line_magic("autocall", "2")
     f = lambda x: x
     ip.user_ns['f'] = f
     try:
         assert ip.prefilter("f 1") == "f(1)"
         for t in ["f +1", "f -1"]:
             assert ip.prefilter(t) == t
 
         # Run tests again with a more permissive exclude_regexp, which will
         # allow transformation of binary operations ('f -1' -> 'f(-1)').
         pm = ip.prefilter_manager
         ac = AutocallChecker(shell=pm.shell, prefilter_manager=pm,
                              config=pm.config)
         try:
             ac.priority = 1
             ac.exclude_regexp = r'^[,&^\|\*/]|^is |^not |^in |^and |^or '
             pm.sort_checkers()
 
             assert ip.prefilter("f -1") == "f(-1)"
             assert ip.prefilter("f +1") == "f(+1)"
         finally:
             pm.unregister_checker(ac)
     finally:
         ip.run_line_magic("autocall", "0")
         del ip.user_ns["f"]
 
 
 def test_issue_114():
     """Check that multiline string literals don't expand as magic
     see http://github.com/ipython/ipython/issues/114"""
 
     template = '"""\n%s\n"""'
     # Store the current value of multi_line_specials and turn it off before
     # running test, since it could be true (case in which the test doesn't make
     # sense, as multiline string literals *will* expand as magic in that case).
     msp = ip.prefilter_manager.multi_line_specials
     ip.prefilter_manager.multi_line_specials = False
     try:
         for mgk in ip.magics_manager.lsmagic()['line']:
             raw = template % mgk
             assert ip.prefilter(raw) == raw
     finally:
         ip.prefilter_manager.multi_line_specials = msp
 
 
 def test_prefilter_attribute_errors():
     """Capture exceptions thrown by user objects on attribute access.
 
     See http://github.com/ipython/ipython/issues/988."""
 
     class X(object):
         def __getattr__(self, k):
             raise ValueError('broken object')
         def __call__(self, x):
             return x
 
     # Create a callable broken object
     ip.user_ns["x"] = X()
     ip.run_line_magic("autocall", "2")
     try:
         # Even if x throws an attribute error when looking at its rewrite
         # attribute, we should not crash.  So the test here is simply making
         # the prefilter call and not having an exception.
         ip.prefilter('x 1')
     finally:
         del ip.user_ns["x"]
         ip.run_line_magic("autocall", "0")
 
 
 def test_autocall_type_ann():
     ip.run_cell("import collections.abc")
     ip.run_line_magic("autocall", "1")
     try:
         assert (
             ip.prefilter("collections.abc.Callable[[int], None]")
             == "collections.abc.Callable[[int], None]"
         )
     finally:
         ip.run_line_magic("autocall", "0")
 
 
 def test_autocall_should_support_unicode():
     ip.run_line_magic("autocall", "2")
     ip.user_ns["π"] = lambda x: x
     try:
         assert ip.prefilter("π 3") == "π(3)"
     finally:
         ip.run_line_magic("autocall", "0")
         del ip.user_ns["π"]
 
 
 def test_autocall_regression_gh_14513():
     ip.run_line_magic("autocall", "2")
     ip.user_ns["foo"] = dict()
     try:
         assert ip.prefilter("foo") == "foo"
     finally:
         ip.run_line_magic("autocall", "0")
         del ip.user_ns["foo"]

IPython/core/tests/test_ultratb.py

@@ -1,456 +1,457
 # encoding: utf-8
 """Tests for IPython.core.ultratb
 """
 import io
 import os.path
 import platform
 import re
 import sys
 import traceback
 import unittest
 from textwrap import dedent
 
 from tempfile import TemporaryDirectory
 
 from IPython.core.ultratb import ColorTB, VerboseTB
 from IPython.testing import tools as tt
 from IPython.testing.decorators import onlyif_unicode_paths, skip_without
 from IPython.utils.syspathcontext import prepended_to_syspath
 
 file_1 = """1
 2
 3
 def f():
   1/0
 """
 
 file_2 = """def f():
   1/0
 """
 
 
 def recursionlimit(frames):
     """
     decorator to set the recursion limit temporarily
     """
 
     def inner(test_function):
         def wrapper(*args, **kwargs):
             rl = sys.getrecursionlimit()
             sys.setrecursionlimit(frames)
             try:
                 return test_function(*args, **kwargs)
             finally:
                 sys.setrecursionlimit(rl)
 
         return wrapper
 
     return inner
 
 
 class ChangedPyFileTest(unittest.TestCase):
     def test_changing_py_file(self):
         """Traceback produced if the line where the error occurred is missing?
 
         https://github.com/ipython/ipython/issues/1456
         """
         with TemporaryDirectory() as td:
             fname = os.path.join(td, "foo.py")
             with open(fname, "w", encoding="utf-8") as f:
                 f.write(file_1)
 
             with prepended_to_syspath(td):
                 ip.run_cell("import foo")
 
             with tt.AssertPrints("ZeroDivisionError"):
                 ip.run_cell("foo.f()")
 
             # Make the file shorter, so the line of the error is missing.
             with open(fname, "w", encoding="utf-8") as f:
                 f.write(file_2)
 
             # For some reason, this was failing on the *second* call after
             # changing the file, so we call f() twice.
             with tt.AssertNotPrints("Internal Python error", channel='stderr'):
                 with tt.AssertPrints("ZeroDivisionError"):
                     ip.run_cell("foo.f()")
                 with tt.AssertPrints("ZeroDivisionError"):
                     ip.run_cell("foo.f()")
 
 iso_8859_5_file = u'''# coding: iso-8859-5
 
 def fail():
     """дбИЖ"""
     1/0     # дбИЖ
 '''
 
 class NonAsciiTest(unittest.TestCase):
     @onlyif_unicode_paths
     def test_nonascii_path(self):
         # Non-ascii directory name as well.
         with TemporaryDirectory(suffix=u'é') as td:
             fname = os.path.join(td, u"fooé.py")
             with open(fname, "w", encoding="utf-8") as f:
                 f.write(file_1)
 
             with prepended_to_syspath(td):
                 ip.run_cell("import foo")
 
             with tt.AssertPrints("ZeroDivisionError"):
                 ip.run_cell("foo.f()")
 
     def test_iso8859_5(self):
         with TemporaryDirectory() as td:
             fname = os.path.join(td, 'dfghjkl.py')
 
             with io.open(fname, 'w', encoding='iso-8859-5') as f:
                 f.write(iso_8859_5_file)
 
             with prepended_to_syspath(td):
                 ip.run_cell("from dfghjkl import fail")
 
             with tt.AssertPrints("ZeroDivisionError"):
                 with tt.AssertPrints(u'дбИЖ', suppress=False):
                     ip.run_cell('fail()')
 
     def test_nonascii_msg(self):
         cell = u"raise Exception('é')"
         expected = u"Exception('é')"
         ip.run_cell("%xmode plain")
         with tt.AssertPrints(expected):
             ip.run_cell(cell)
 
         ip.run_cell("%xmode verbose")
         with tt.AssertPrints(expected):
             ip.run_cell(cell)
 
         ip.run_cell("%xmode context")
         with tt.AssertPrints(expected):
             ip.run_cell(cell)
 
         ip.run_cell("%xmode minimal")
         with tt.AssertPrints(u"Exception: é"):
             ip.run_cell(cell)
 
         # Put this back into Context mode for later tests.
         ip.run_cell("%xmode context")
 
 class NestedGenExprTestCase(unittest.TestCase):
     """
     Regression test for the following issues:
     https://github.com/ipython/ipython/issues/8293
     https://github.com/ipython/ipython/issues/8205
     """
     def test_nested_genexpr(self):
         code = dedent(
             """\
             class SpecificException(Exception):
                 pass
 
             def foo(x):
                 raise SpecificException("Success!")
 
             sum(sum(foo(x) for _ in [0]) for x in [0])
             """
         )
         with tt.AssertPrints('SpecificException: Success!', suppress=False):
             ip.run_cell(code)
 
 
 indentationerror_file = """if True:
 zoom()
 """
 
 class IndentationErrorTest(unittest.TestCase):
     def test_indentationerror_shows_line(self):
         # See issue gh-2398
         with tt.AssertPrints("IndentationError"):
             with tt.AssertPrints("zoom()", suppress=False):
                 ip.run_cell(indentationerror_file)
 
         with TemporaryDirectory() as td:
             fname = os.path.join(td, "foo.py")
             with open(fname, "w", encoding="utf-8") as f:
                 f.write(indentationerror_file)
 
             with tt.AssertPrints("IndentationError"):
                 with tt.AssertPrints("zoom()", suppress=False):
                     ip.run_line_magic("run", fname)
 
 
 @skip_without("pandas")
 def test_dynamic_code():
     code = """
     import pandas
     df = pandas.DataFrame([])
 
     # Important: only fails inside of an "exec" call:
     exec("df.foobarbaz()")
     """
 
     with tt.AssertPrints("Could not get source"):
         ip.run_cell(code)
 
 
 se_file_1 = """1
 2
 7/
 """
 
 se_file_2 = """7/
 """
 
 class SyntaxErrorTest(unittest.TestCase):
 
     def test_syntaxerror_no_stacktrace_at_compile_time(self):
         syntax_error_at_compile_time = """
 def foo():
     ..
 """
         with tt.AssertPrints("SyntaxError"):
             ip.run_cell(syntax_error_at_compile_time)
 
         with tt.AssertNotPrints("foo()"):
             ip.run_cell(syntax_error_at_compile_time)
 
     def test_syntaxerror_stacktrace_when_running_compiled_code(self):
         syntax_error_at_runtime = """
 def foo():
     eval("..")
 
 def bar():
     foo()
 
 bar()
 """
         with tt.AssertPrints("SyntaxError"):
             ip.run_cell(syntax_error_at_runtime)
         # Assert syntax error during runtime generate stacktrace
         with tt.AssertPrints(["foo()", "bar()"]):
             ip.run_cell(syntax_error_at_runtime)
         del ip.user_ns['bar']
         del ip.user_ns['foo']
 
     def test_changing_py_file(self):
         with TemporaryDirectory() as td:
             fname = os.path.join(td, "foo.py")
             with open(fname, "w", encoding="utf-8") as f:
                 f.write(se_file_1)
 
             with tt.AssertPrints(["7/", "SyntaxError"]):
                 ip.run_line_magic("run", fname)
 
             # Modify the file
             with open(fname, "w", encoding="utf-8") as f:
                 f.write(se_file_2)
 
             # The SyntaxError should point to the correct line
             with tt.AssertPrints(["7/", "SyntaxError"]):
                 ip.run_line_magic("run", fname)
 
     def test_non_syntaxerror(self):
         # SyntaxTB may be called with an error other than a SyntaxError
         # See e.g. gh-4361
         try:
             raise ValueError('QWERTY')
         except ValueError:
             with tt.AssertPrints('QWERTY'):
                 ip.showsyntaxerror()
 
+import sys
 
 if platform.python_implementation() != "PyPy":
     """
     New 3.9 Pgen Parser does not raise Memory error, except on failed malloc.
     """
     class MemoryErrorTest(unittest.TestCase):
         def test_memoryerror(self):
             memoryerror_code = "(" * 200 + ")" * 200
             ip.run_cell(memoryerror_code)
 
 
 class Python3ChainedExceptionsTest(unittest.TestCase):
     DIRECT_CAUSE_ERROR_CODE = """
 try:
     x = 1 + 2
     print(not_defined_here)
 except Exception as e:
     x += 55
     x - 1
     y = {}
     raise KeyError('uh') from e
     """
 
     EXCEPTION_DURING_HANDLING_CODE = """
 try:
     x = 1 + 2
     print(not_defined_here)
 except Exception as e:
     x += 55
     x - 1
     y = {}
     raise KeyError('uh')
     """
 
     SUPPRESS_CHAINING_CODE = """
 try:
     1/0
 except Exception:
     raise ValueError("Yikes") from None
     """
 
     SYS_EXIT_WITH_CONTEXT_CODE = """
 try:
     1/0
 except Exception as e:
     raise SystemExit(1)
     """
 
     def test_direct_cause_error(self):
         with tt.AssertPrints(["KeyError", "NameError", "direct cause"]):
             ip.run_cell(self.DIRECT_CAUSE_ERROR_CODE)
 
     def test_exception_during_handling_error(self):
         with tt.AssertPrints(["KeyError", "NameError", "During handling"]):
             ip.run_cell(self.EXCEPTION_DURING_HANDLING_CODE)
 
     def test_sysexit_while_handling_error(self):
         with tt.AssertPrints(["SystemExit", "to see the full traceback"]):
             with tt.AssertNotPrints(["another exception"], suppress=False):
                 ip.run_cell(self.SYS_EXIT_WITH_CONTEXT_CODE)
 
     def test_suppress_exception_chaining(self):
         with tt.AssertNotPrints("ZeroDivisionError"), \
              tt.AssertPrints("ValueError", suppress=False):
             ip.run_cell(self.SUPPRESS_CHAINING_CODE)
 
     def test_plain_direct_cause_error(self):
         with tt.AssertPrints(["KeyError", "NameError", "direct cause"]):
             ip.run_cell("%xmode Plain")
             ip.run_cell(self.DIRECT_CAUSE_ERROR_CODE)
             ip.run_cell("%xmode Verbose")
 
     def test_plain_exception_during_handling_error(self):
         with tt.AssertPrints(["KeyError", "NameError", "During handling"]):
             ip.run_cell("%xmode Plain")
             ip.run_cell(self.EXCEPTION_DURING_HANDLING_CODE)
             ip.run_cell("%xmode Verbose")
 
     def test_plain_suppress_exception_chaining(self):
         with tt.AssertNotPrints("ZeroDivisionError"), \
              tt.AssertPrints("ValueError", suppress=False):
             ip.run_cell("%xmode Plain")
             ip.run_cell(self.SUPPRESS_CHAINING_CODE)
             ip.run_cell("%xmode Verbose")
 
 
 class RecursionTest(unittest.TestCase):
     DEFINITIONS = """
 def non_recurs():
     1/0
 
 def r1():
     r1()
 
 def r3a():
     r3b()
 
 def r3b():
     r3c()
 
 def r3c():
     r3a()
 
 def r3o1():
     r3a()
 
 def r3o2():
     r3o1()
 """
     def setUp(self):
         ip.run_cell(self.DEFINITIONS)
 
     def test_no_recursion(self):
         with tt.AssertNotPrints("skipping similar frames"):
             ip.run_cell("non_recurs()")
 
     @recursionlimit(200)
     def test_recursion_one_frame(self):
         with tt.AssertPrints(re.compile(
             r"\[\.\.\. skipping similar frames: r1 at line 5 \(\d{2,3} times\)\]")
         ):
             ip.run_cell("r1()")
 
     @recursionlimit(160)
     def test_recursion_three_frames(self):
         with tt.AssertPrints("[... skipping similar frames: "), \
                 tt.AssertPrints(re.compile(r"r3a at line 8 \(\d{2} times\)"), suppress=False), \
                 tt.AssertPrints(re.compile(r"r3b at line 11 \(\d{2} times\)"), suppress=False), \
                 tt.AssertPrints(re.compile(r"r3c at line 14 \(\d{2} times\)"), suppress=False):
             ip.run_cell("r3o2()")
 
 
 class PEP678NotesReportingTest(unittest.TestCase):
     ERROR_WITH_NOTE = """
 try:
     raise AssertionError("Message")
 except Exception as e:
     try:
         e.add_note("This is a PEP-678 note.")
     except AttributeError:  # Python <= 3.10
         e.__notes__ = ("This is a PEP-678 note.",)
     raise
     """
 
     def test_verbose_reports_notes(self):
         with tt.AssertPrints(["AssertionError", "Message", "This is a PEP-678 note."]):
             ip.run_cell(self.ERROR_WITH_NOTE)
 
     def test_plain_reports_notes(self):
         with tt.AssertPrints(["AssertionError", "Message", "This is a PEP-678 note."]):
             ip.run_cell("%xmode Plain")
             ip.run_cell(self.ERROR_WITH_NOTE)
             ip.run_cell("%xmode Verbose")
 
 
 #----------------------------------------------------------------------------
 
 # module testing (minimal)
 def test_handlers():
     def spam(c, d_e):
         (d, e) = d_e
         x = c + d
         y = c * d
         foo(x, y)
 
     def foo(a, b, bar=1):
         eggs(a, b + bar)
 
     def eggs(f, g, z=globals()):
         h = f + g
         i = f - g
         return h / i
 
     buff = io.StringIO()
 
     buff.write('')
     buff.write('*** Before ***')
     try:
         buff.write(spam(1, (2, 3)))
     except:
         traceback.print_exc(file=buff)
 
     handler = ColorTB(ostream=buff)
     buff.write('*** ColorTB ***')
     try:
         buff.write(spam(1, (2, 3)))
     except:
         handler(*sys.exc_info())
     buff.write('')
 
     handler = VerboseTB(ostream=buff)
     buff.write('*** VerboseTB ***')
     try:
         buff.write(spam(1, (2, 3)))
     except:
         handler(*sys.exc_info())
     buff.write('')

IPython/core/ultratb.py

@@ -1,1545 +1,1547
 # -*- coding: utf-8 -*-
 """
 Verbose and colourful traceback formatting.
 
 **ColorTB**
 
 I've always found it a bit hard to visually parse tracebacks in Python.  The
 ColorTB class is a solution to that problem.  It colors the different parts of a
 traceback in a manner similar to what you would expect from a syntax-highlighting
 text editor.
 
 Installation instructions for ColorTB::
 
     import sys,ultratb
     sys.excepthook = ultratb.ColorTB()
 
 **VerboseTB**
 
 I've also included a port of Ka-Ping Yee's "cgitb.py" that produces all kinds
 of useful info when a traceback occurs.  Ping originally had it spit out HTML
 and intended it for CGI programmers, but why should they have all the fun?  I
 altered it to spit out colored text to the terminal.  It's a bit overwhelming,
 but kind of neat, and maybe useful for long-running programs that you believe
 are bug-free.  If a crash *does* occur in that type of program you want details.
 Give it a shot--you'll love it or you'll hate it.
 
 .. note::
 
   The Verbose mode prints the variables currently visible where the exception
   happened (shortening their strings if too long). This can potentially be
   very slow, if you happen to have a huge data structure whose string
   representation is complex to compute. Your computer may appear to freeze for
   a while with cpu usage at 100%. If this occurs, you can cancel the traceback
   with Ctrl-C (maybe hitting it more than once).
 
   If you encounter this kind of situation often, you may want to use the
   Verbose_novars mode instead of the regular Verbose, which avoids formatting
   variables (but otherwise includes the information and context given by
   Verbose).
 
 .. note::
 
   The verbose mode print all variables in the stack, which means it can
   potentially leak sensitive information like access keys, or unencrypted
   password.
 
 Installation instructions for VerboseTB::
 
     import sys,ultratb
     sys.excepthook = ultratb.VerboseTB()
 
 Note:  Much of the code in this module was lifted verbatim from the standard
 library module 'traceback.py' and Ka-Ping Yee's 'cgitb.py'.
 
 Color schemes
 -------------
 
 The colors are defined in the class TBTools through the use of the
 ColorSchemeTable class. Currently the following exist:
 
   - NoColor: allows all of this module to be used in any terminal (the color
     escapes are just dummy blank strings).
 
   - Linux: is meant to look good in a terminal like the Linux console (black
     or very dark background).
 
   - LightBG: similar to Linux but swaps dark/light colors to be more readable
     in light background terminals.
 
   - Neutral: a neutral color scheme that should be readable on both light and
     dark background
 
 You can implement other color schemes easily, the syntax is fairly
 self-explanatory. Please send back new schemes you develop to the author for
 possible inclusion in future releases.
 
 Inheritance diagram:
 
 .. inheritance-diagram:: IPython.core.ultratb
    :parts: 3
 """
 
 #*****************************************************************************
 # Copyright (C) 2001 Nathaniel Gray <[e-mail]>
 # Copyright (C) 2001-2004 Fernando Perez <[e-mail]>
 #
 # Distributed under the terms of the BSD License.  The full license is in
 # the file COPYING, distributed as part of this software.
 #*****************************************************************************
 
 
 from collections.abc import Sequence
 import functools
 import inspect
 import linecache
 import pydoc
 import sys
 import time
 import traceback
 import types
 from types import TracebackType
 from typing import Any, List, Optional, Tuple
 
 import stack_data
 from pygments.formatters.terminal256 import Terminal256Formatter
 from pygments.styles import get_style_by_name
 
 import IPython.utils.colorable as colorable
 # IPython's own modules
 from IPython import get_ipython
 from IPython.core import debugger
 from IPython.core.display_trap import DisplayTrap
 from IPython.core.excolors import exception_colors
 from IPython.utils import PyColorize
 from IPython.utils import path as util_path
 from IPython.utils import py3compat
 from IPython.utils.terminal import get_terminal_size
 
 # Globals
 # amount of space to put line numbers before verbose tracebacks
 INDENT_SIZE = 8
 
 # Default color scheme.  This is used, for example, by the traceback
 # formatter.  When running in an actual IPython instance, the user's rc.colors
 # value is used, but having a module global makes this functionality available
 # to users of ultratb who are NOT running inside ipython.
 DEFAULT_SCHEME = 'NoColor'
 FAST_THRESHOLD = 10_000
 
 # ---------------------------------------------------------------------------
 # Code begins
 
 # Helper function -- largely belongs to VerboseTB, but we need the same
 # functionality to produce a pseudo verbose TB for SyntaxErrors, so that they
 # can be recognized properly by ipython.el's py-traceback-line-re
 # (SyntaxErrors have to be treated specially because they have no traceback)
 
 
 @functools.lru_cache()
 def count_lines_in_py_file(filename: str) -> int:
     """
     Given a filename, returns the number of lines in the file
     if it ends with the extension ".py". Otherwise, returns 0.
     """
     if not filename.endswith(".py"):
         return 0
     else:
         try:
             with open(filename, "r") as file:
                 s = sum(1 for line in file)
         except UnicodeError:
             return 0
     return s
 
     """
     Given a frame object, returns the total number of lines in the file
     if the filename ends with the extension ".py". Otherwise, returns 0.
     """
 
 
 def get_line_number_of_frame(frame: types.FrameType) -> int:
     """
     Given a frame object, returns the total number of lines in the file
     containing the frame's code object, or the number of lines in the
     frame's source code if the file is not available.
 
     Parameters
     ----------
     frame : FrameType
         The frame object whose line number is to be determined.
 
     Returns
     -------
     int
         The total number of lines in the file containing the frame's
         code object, or the number of lines in the frame's source code
         if the file is not available.
     """
     filename = frame.f_code.co_filename
     if filename is None:
         print("No file....")
         lines, first = inspect.getsourcelines(frame)
         return first + len(lines)
     return count_lines_in_py_file(filename)
 
 
 def _safe_string(value, what, func=str):
     # Copied from cpython/Lib/traceback.py
     try:
         return func(value)
-    except Exception:
+    except:
         return f"<{what} {func.__name__}() failed>"
 
 
 def _format_traceback_lines(lines, Colors, has_colors: bool, lvals):
     """
     Format tracebacks lines with pointing arrow, leading numbers...
 
     Parameters
     ----------
     lines : list[Line]
     Colors
         ColorScheme used.
     lvals : str
         Values of local variables, already colored, to inject just after the error line.
     """
     numbers_width = INDENT_SIZE - 1
     res = []
 
     for stack_line in lines:
         if stack_line is stack_data.LINE_GAP:
             res.append('%s   (...)%s\n' % (Colors.linenoEm, Colors.Normal))
             continue
 
         line = stack_line.render(pygmented=has_colors).rstrip('\n') + '\n'
         lineno = stack_line.lineno
         if stack_line.is_current:
             # This is the line with the error
             pad = numbers_width - len(str(lineno))
             num = '%s%s' % (debugger.make_arrow(pad), str(lineno))
             start_color = Colors.linenoEm
         else:
             num = '%*s' % (numbers_width, lineno)
             start_color = Colors.lineno
 
         line = '%s%s%s %s' % (start_color, num, Colors.Normal, line)
 
         res.append(line)
         if lvals and stack_line.is_current:
             res.append(lvals + '\n')
     return res
 
 def _simple_format_traceback_lines(lnum, index, lines, Colors, lvals, _line_format):
     """
     Format tracebacks lines with pointing arrow, leading numbers...
 
     Parameters
     ==========
 
     lnum: int
         number of the target line of code.
     index: int
         which line in the list should be highlighted.
     lines: list[string]
     Colors:
         ColorScheme used.
     lvals: bytes
         Values of local variables, already colored, to inject just after the error line.
     _line_format: f (str) -> (str, bool)
         return (colorized version of str, failure to do so)
     """
     numbers_width = INDENT_SIZE - 1
     res = []
     for i, line in enumerate(lines, lnum - index):
         # assert isinstance(line, str)
         line = py3compat.cast_unicode(line)
 
         new_line, err = _line_format(line, "str")
         if not err:
             line = new_line
 
         if i == lnum:
             # This is the line with the error
             pad = numbers_width - len(str(i))
             num = "%s%s" % (debugger.make_arrow(pad), str(lnum))
             line = "%s%s%s %s%s" % (
                 Colors.linenoEm,
                 num,
                 Colors.line,
                 line,
                 Colors.Normal,
             )
         else:
             num = "%*s" % (numbers_width, i)
             line = "%s%s%s %s" % (Colors.lineno, num, Colors.Normal, line)
 
         res.append(line)
         if lvals and i == lnum:
             res.append(lvals + "\n")
     return res
 
 
 def _format_filename(file, ColorFilename, ColorNormal, *, lineno=None):
     """
     Format filename lines with custom formatting from caching compiler or `File *.py` by default
 
     Parameters
     ----------
     file : str
     ColorFilename
         ColorScheme's filename coloring to be used.
     ColorNormal
         ColorScheme's normal coloring to be used.
     """
     ipinst = get_ipython()
     if (
         ipinst is not None
         and (data := ipinst.compile.format_code_name(file)) is not None
     ):
         label, name = data
         if lineno is None:
             tpl_link = f"{{label}} {ColorFilename}{{name}}{ColorNormal}"
         else:
             tpl_link = (
                 f"{{label}} {ColorFilename}{{name}}, line {{lineno}}{ColorNormal}"
             )
     else:
         label = "File"
         name = util_path.compress_user(
             py3compat.cast_unicode(file, util_path.fs_encoding)
         )
         if lineno is None:
             tpl_link = f"{{label}} {ColorFilename}{{name}}{ColorNormal}"
         else:
             # can we make this the more friendly ", line {{lineno}}", or do we need to preserve the formatting with the colon?
             tpl_link = f"{{label}} {ColorFilename}{{name}}:{{lineno}}{ColorNormal}"
 
     return tpl_link.format(label=label, name=name, lineno=lineno)
 
 #---------------------------------------------------------------------------
 # Module classes
 class TBTools(colorable.Colorable):
     """Basic tools used by all traceback printer classes."""
 
     # Number of frames to skip when reporting tracebacks
     tb_offset = 0
 
     def __init__(
         self,
         color_scheme="NoColor",
         call_pdb=False,
         ostream=None,
         parent=None,
         config=None,
         *,
         debugger_cls=None,
     ):
         # Whether to call the interactive pdb debugger after printing
         # tracebacks or not
         super(TBTools, self).__init__(parent=parent, config=config)
         self.call_pdb = call_pdb
 
         # Output stream to write to.  Note that we store the original value in
         # a private attribute and then make the public ostream a property, so
         # that we can delay accessing sys.stdout until runtime.  The way
         # things are written now, the sys.stdout object is dynamically managed
         # so a reference to it should NEVER be stored statically.  This
         # property approach confines this detail to a single location, and all
         # subclasses can simply access self.ostream for writing.
         self._ostream = ostream
 
         # Create color table
         self.color_scheme_table = exception_colors()
 
         self.set_colors(color_scheme)
         self.old_scheme = color_scheme  # save initial value for toggles
         self.debugger_cls = debugger_cls or debugger.Pdb
 
         if call_pdb:
             self.pdb = self.debugger_cls()
         else:
             self.pdb = None
 
     def _get_ostream(self):
         """Output stream that exceptions are written to.
 
         Valid values are:
 
         - None: the default, which means that IPython will dynamically resolve
           to sys.stdout.  This ensures compatibility with most tools, including
           Windows (where plain stdout doesn't recognize ANSI escapes).
 
         - Any object with 'write' and 'flush' attributes.
         """
         return sys.stdout if self._ostream is None else self._ostream
 
     def _set_ostream(self, val):
         assert val is None or (hasattr(val, 'write') and hasattr(val, 'flush'))
         self._ostream = val
 
     ostream = property(_get_ostream, _set_ostream)
 
     @staticmethod
     def _get_chained_exception(exception_value):
         cause = getattr(exception_value, "__cause__", None)
         if cause:
             return cause
         if getattr(exception_value, "__suppress_context__", False):
             return None
         return getattr(exception_value, "__context__", None)
 
     def get_parts_of_chained_exception(
         self, evalue
     ) -> Optional[Tuple[type, BaseException, TracebackType]]:
         chained_evalue = self._get_chained_exception(evalue)
 
         if chained_evalue:
             return chained_evalue.__class__, chained_evalue, chained_evalue.__traceback__
         return None
 
     def prepare_chained_exception_message(self, cause) -> List[Any]:
         direct_cause = "\nThe above exception was the direct cause of the following exception:\n"
         exception_during_handling = "\nDuring handling of the above exception, another exception occurred:\n"
 
         if cause:
             message = [[direct_cause]]
         else:
             message = [[exception_during_handling]]
         return message
 
     @property
     def has_colors(self) -> bool:
         return self.color_scheme_table.active_scheme_name.lower() != "nocolor"
 
     def set_colors(self, *args, **kw):
         """Shorthand access to the color table scheme selector method."""
 
         # Set own color table
         self.color_scheme_table.set_active_scheme(*args, **kw)
         # for convenience, set Colors to the active scheme
         self.Colors = self.color_scheme_table.active_colors
         # Also set colors of debugger
         if hasattr(self, 'pdb') and self.pdb is not None:
             self.pdb.set_colors(*args, **kw)
 
     def color_toggle(self):
         """Toggle between the currently active color scheme and NoColor."""
 
         if self.color_scheme_table.active_scheme_name == 'NoColor':
             self.color_scheme_table.set_active_scheme(self.old_scheme)
             self.Colors = self.color_scheme_table.active_colors
         else:
             self.old_scheme = self.color_scheme_table.active_scheme_name
             self.color_scheme_table.set_active_scheme('NoColor')
             self.Colors = self.color_scheme_table.active_colors
 
     def stb2text(self, stb):
         """Convert a structured traceback (a list) to a string."""
         return '\n'.join(stb)
 
     def text(self, etype, value, tb, tb_offset: Optional[int] = None, context=5):
         """Return formatted traceback.
 
         Subclasses may override this if they add extra arguments.
         """
         tb_list = self.structured_traceback(etype, value, tb,
                                             tb_offset, context)
         return self.stb2text(tb_list)
 
     def structured_traceback(
         self,
         etype: type,
         evalue: Optional[BaseException],
         etb: Optional[TracebackType] = None,
         tb_offset: Optional[int] = None,
         number_of_lines_of_context: int = 5,
     ):
         """Return a list of traceback frames.
 
         Must be implemented by each class.
         """
         raise NotImplementedError()
 
 
 #---------------------------------------------------------------------------
 class ListTB(TBTools):
     """Print traceback information from a traceback list, with optional color.
 
     Calling requires 3 arguments: (etype, evalue, elist)
     as would be obtained by::
 
       etype, evalue, tb = sys.exc_info()
       if tb:
         elist = traceback.extract_tb(tb)
       else:
         elist = None
 
     It can thus be used by programs which need to process the traceback before
     printing (such as console replacements based on the code module from the
     standard library).
 
     Because they are meant to be called without a full traceback (only a
     list), instances of this class can't call the interactive pdb debugger."""
 
 
     def __call__(self, etype, value, elist):
         self.ostream.flush()
         self.ostream.write(self.text(etype, value, elist))
         self.ostream.write('\n')
 
     def _extract_tb(self, tb):
         if tb:
             return traceback.extract_tb(tb)
         else:
             return None
 
     def structured_traceback(
         self,
         etype: type,
         evalue: Optional[BaseException],
         etb: Optional[TracebackType] = None,
         tb_offset: Optional[int] = None,
         context=5,
     ):
         """Return a color formatted string with the traceback info.
 
         Parameters
         ----------
         etype : exception type
             Type of the exception raised.
         evalue : object
             Data stored in the exception
         etb : list | TracebackType | None
             If list: List of frames, see class docstring for details.
             If Traceback: Traceback of the exception.
         tb_offset : int, optional
             Number of frames in the traceback to skip.  If not given, the
             instance evalue is used (set in constructor).
         context : int, optional
             Number of lines of context information to print.
 
         Returns
         -------
         String with formatted exception.
         """
         # This is a workaround to get chained_exc_ids in recursive calls
         # etb should not be a tuple if structured_traceback is not recursive
         if isinstance(etb, tuple):
             etb, chained_exc_ids = etb
         else:
             chained_exc_ids = set()
 
         if isinstance(etb, list):
             elist = etb
         elif etb is not None:
             elist = self._extract_tb(etb)
         else:
             elist = []
         tb_offset = self.tb_offset if tb_offset is None else tb_offset
         assert isinstance(tb_offset, int)
         Colors = self.Colors
         out_list = []
         if elist:
 
             if tb_offset and len(elist) > tb_offset:
                 elist = elist[tb_offset:]
 
             out_list.append('Traceback %s(most recent call last)%s:' %
                             (Colors.normalEm, Colors.Normal) + '\n')
             out_list.extend(self._format_list(elist))
         # The exception info should be a single entry in the list.
         lines = ''.join(self._format_exception_only(etype, evalue))
         out_list.append(lines)
 
         # Find chained exceptions if we have a traceback (not for exception-only mode)
         if etb is not None:
             exception = self.get_parts_of_chained_exception(evalue)
 
             if exception and (id(exception[1]) not in chained_exc_ids):
                 chained_exception_message = (
                     self.prepare_chained_exception_message(evalue.__cause__)[0]
                     if evalue is not None
                     else ""
                 )
                 etype, evalue, etb = exception
                 # Trace exception to avoid infinite 'cause' loop
                 chained_exc_ids.add(id(exception[1]))
                 chained_exceptions_tb_offset = 0
                 out_list = (
                     self.structured_traceback(
                         etype,
                         evalue,
                         (etb, chained_exc_ids),  # type: ignore
                         chained_exceptions_tb_offset,
                         context,
                     )
                     + chained_exception_message
                     + out_list
                 )
 
         return out_list
 
     def _format_list(self, extracted_list):
         """Format a list of traceback entry tuples for printing.
 
         Given a list of tuples as returned by extract_tb() or
         extract_stack(), return a list of strings ready for printing.
         Each string in the resulting list corresponds to the item with the
         same index in the argument list.  Each string ends in a newline;
         the strings may contain internal newlines as well, for those items
         whose source text line is not None.
 
         Lifted almost verbatim from traceback.py
         """
 
         Colors = self.Colors
         output_list = []
         for ind, (filename, lineno, name, line) in enumerate(extracted_list):
             normalCol, nameCol, fileCol, lineCol = (
                 # Emphasize the last entry
                 (Colors.normalEm, Colors.nameEm, Colors.filenameEm, Colors.line)
                 if ind == len(extracted_list) - 1
                 else (Colors.Normal, Colors.name, Colors.filename, "")
             )
 
             fns = _format_filename(filename, fileCol, normalCol, lineno=lineno)
             item = f"{normalCol}  {fns}"
 
             if name != "<module>":
                 item += f" in {nameCol}{name}{normalCol}\n"
             else:
                 item += "\n"
             if line:
                 item += f"{lineCol}    {line.strip()}{normalCol}\n"
             output_list.append(item)
 
         return output_list
 
     def _format_exception_only(self, etype, value):
         """Format the exception part of a traceback.
 
         The arguments are the exception type and value such as given by
         sys.exc_info()[:2]. The return value is a list of strings, each ending
         in a newline.  Normally, the list contains a single string; however,
         for SyntaxError exceptions, it contains several lines that (when
         printed) display detailed information about where the syntax error
         occurred.  The message indicating which exception occurred is the
         always last string in the list.
 
         Also lifted nearly verbatim from traceback.py
         """
         have_filedata = False
         Colors = self.Colors
         output_list = []
         stype = py3compat.cast_unicode(Colors.excName + etype.__name__ + Colors.Normal)
         if value is None:
             # Not sure if this can still happen in Python 2.6 and above
             output_list.append(stype + "\n")
         else:
             if issubclass(etype, SyntaxError):
                 have_filedata = True
                 if not value.filename: value.filename = "<string>"
                 if value.lineno:
                     lineno = value.lineno
                     textline = linecache.getline(value.filename, value.lineno)
                 else:
                     lineno = "unknown"
                     textline = ""
                 output_list.append(
                     "%s  %s%s\n"
                     % (
                         Colors.normalEm,
                         _format_filename(
                             value.filename,
                             Colors.filenameEm,
                             Colors.normalEm,
                             lineno=(None if lineno == "unknown" else lineno),
                         ),
                         Colors.Normal,
                     )
                 )
                 if textline == "":
                     textline = py3compat.cast_unicode(value.text, "utf-8")
 
                 if textline is not None:
                     i = 0
                     while i < len(textline) and textline[i].isspace():
                         i += 1
                     output_list.append(
                         "%s    %s%s\n" % (Colors.line, textline.strip(), Colors.Normal)
                     )
                     if value.offset is not None:
                         s = '    '
                         for c in textline[i:value.offset - 1]:
                             if c.isspace():
                                 s += c
                             else:
                                 s += " "
                         output_list.append(
                             "%s%s^%s\n" % (Colors.caret, s, Colors.Normal)
                         )
 
             try:
                 s = value.msg
             except Exception:
                 s = self._some_str(value)
             if s:
                 output_list.append(
                     "%s%s:%s %s\n" % (stype, Colors.excName, Colors.Normal, s)
                 )
             else:
                 output_list.append("%s\n" % stype)
 
             # PEP-678 notes
             output_list.extend(f"{x}\n" for x in getattr(value, "__notes__", []))
 
         # sync with user hooks
         if have_filedata:
             ipinst = get_ipython()
             if ipinst is not None:
                 ipinst.hooks.synchronize_with_editor(value.filename, value.lineno, 0)
 
         return output_list
 
     def get_exception_only(self, etype, value):
         """Only print the exception type and message, without a traceback.
 
         Parameters
         ----------
         etype : exception type
         value : exception value
         """
         return ListTB.structured_traceback(self, etype, value)
 
     def show_exception_only(self, etype, evalue):
         """Only print the exception type and message, without a traceback.
 
         Parameters
         ----------
         etype : exception type
         evalue : exception value
         """
         # This method needs to use __call__ from *this* class, not the one from
         # a subclass whose signature or behavior may be different
         ostream = self.ostream
         ostream.flush()
         ostream.write('\n'.join(self.get_exception_only(etype, evalue)))
         ostream.flush()
 
     def _some_str(self, value):
         # Lifted from traceback.py
         try:
-            return str(value)
+            return py3compat.cast_unicode(str(value))
         except:
-            return "<unprintable %s object>" % type(value).__name__
+            return u'<unprintable %s object>' % type(value).__name__
 
 
 class FrameInfo:
     """
     Mirror of stack data's FrameInfo, but so that we can bypass highlighting on
     really long frames.
     """
 
     description: Optional[str]
     filename: Optional[str]
     lineno: Tuple[int]
     # number of context lines to use
     context: Optional[int]
     raw_lines: List[str]
 
     @classmethod
     def _from_stack_data_FrameInfo(cls, frame_info):
         return cls(
             getattr(frame_info, "description", None),
             getattr(frame_info, "filename", None),  # type: ignore[arg-type]
             getattr(frame_info, "lineno", None),  # type: ignore[arg-type]
             getattr(frame_info, "frame", None),
             getattr(frame_info, "code", None),
             sd=frame_info,
             context=None,
         )
 
     def __init__(
         self,
         description: Optional[str],
         filename: str,
         lineno: Tuple[int],
         frame,
         code,
         *,
         sd=None,
         context=None,
     ):
         self.description = description
         self.filename = filename
         self.lineno = lineno
         self.frame = frame
         self.code = code
         self._sd = sd
         self.context = context
 
         # self.lines = []
         if sd is None:
             try:
                 # return a list of source lines and a starting line number
                 self.raw_lines = inspect.getsourcelines(frame)[0]
             except OSError:
                 self.raw_lines = [
                     "'Could not get source, probably due dynamically evaluated source code.'"
                 ]
 
     @property
     def variables_in_executing_piece(self):
         if self._sd:
             return self._sd.variables_in_executing_piece
         else:
             return []
 
     @property
     def lines(self):
         from executing.executing import NotOneValueFound
 
         try:
             return self._sd.lines
         except NotOneValueFound:
 
             class Dummy:
                 lineno = 0
                 is_current = False
 
                 def render(self, *, pygmented):
                     return "<Error retrieving source code with stack_data see ipython/ipython#13598>"
 
             return [Dummy()]
 
     @property
     def executing(self):
         if self._sd:
             return self._sd.executing
         else:
             return None
 
 
 # ----------------------------------------------------------------------------
 class VerboseTB(TBTools):
     """A port of Ka-Ping Yee's cgitb.py module that outputs color text instead
     of HTML.  Requires inspect and pydoc.  Crazy, man.
 
     Modified version which optionally strips the topmost entries from the
     traceback, to be used with alternate interpreters (because their own code
     would appear in the traceback)."""
 
     tb_highlight = "bg:ansiyellow"
     tb_highlight_style = "default"
 
     def __init__(
         self,
         color_scheme: str = "Linux",
         call_pdb: bool = False,
         ostream=None,
         tb_offset: int = 0,
         long_header: bool = False,
         include_vars: bool = True,
         check_cache=None,
         debugger_cls=None,
         parent=None,
         config=None,
     ):
         """Specify traceback offset, headers and color scheme.
 
         Define how many frames to drop from the tracebacks. Calling it with
         tb_offset=1 allows use of this handler in interpreters which will have
         their own code at the top of the traceback (VerboseTB will first
         remove that frame before printing the traceback info)."""
         TBTools.__init__(
             self,
             color_scheme=color_scheme,
             call_pdb=call_pdb,
             ostream=ostream,
             parent=parent,
             config=config,
             debugger_cls=debugger_cls,
         )
         self.tb_offset = tb_offset
         self.long_header = long_header
         self.include_vars = include_vars
         # By default we use linecache.checkcache, but the user can provide a
         # different check_cache implementation.  This was formerly used by the
         # IPython kernel for interactive code, but is no longer necessary.
         if check_cache is None:
             check_cache = linecache.checkcache
         self.check_cache = check_cache
 
         self.skip_hidden = True
 
     def format_record(self, frame_info: FrameInfo):
         """Format a single stack frame"""
         assert isinstance(frame_info, FrameInfo)
         Colors = self.Colors  # just a shorthand + quicker name lookup
         ColorsNormal = Colors.Normal  # used a lot
 
         if isinstance(frame_info._sd, stack_data.RepeatedFrames):
             return '    %s[... skipping similar frames: %s]%s\n' % (
                 Colors.excName, frame_info.description, ColorsNormal)
 
         indent = " " * INDENT_SIZE
         em_normal = "%s\n%s%s" % (Colors.valEm, indent, ColorsNormal)
         tpl_call = f"in {Colors.vName}{{file}}{Colors.valEm}{{scope}}{ColorsNormal}"
         tpl_call_fail = "in %s%%s%s(***failed resolving arguments***)%s" % (
             Colors.vName,
             Colors.valEm,
             ColorsNormal,
         )
         tpl_name_val = "%%s %s= %%s%s" % (Colors.valEm, ColorsNormal)
 
         link = _format_filename(
             frame_info.filename,
             Colors.filenameEm,
             ColorsNormal,
             lineno=frame_info.lineno,
         )
         args, varargs, varkw, locals_ = inspect.getargvalues(frame_info.frame)
         if frame_info.executing is not None:
             func = frame_info.executing.code_qualname()
         else:
             func = "?"
         if func == "<module>":
             call = ""
         else:
             # Decide whether to include variable details or not
             var_repr = eqrepr if self.include_vars else nullrepr
             try:
                 scope = inspect.formatargvalues(
                     args, varargs, varkw, locals_, formatvalue=var_repr
                 )
                 call = tpl_call.format(file=func, scope=scope)
             except KeyError:
                 # This happens in situations like errors inside generator
                 # expressions, where local variables are listed in the
                 # line, but can't be extracted from the frame.  I'm not
                 # 100% sure this isn't actually a bug in inspect itself,
                 # but since there's no info for us to compute with, the
                 # best we can do is report the failure and move on.  Here
                 # we must *not* call any traceback construction again,
                 # because that would mess up use of %debug later on.  So we
                 # simply report the failure and move on.  The only
                 # limitation will be that this frame won't have locals
                 # listed in the call signature.  Quite subtle problem...
                 # I can't think of a good way to validate this in a unit
                 # test, but running a script consisting of:
                 #  dict( (k,v.strip()) for (k,v) in range(10) )
                 # will illustrate the error, if this exception catch is
                 # disabled.
                 call = tpl_call_fail % func
 
         lvals = ''
         lvals_list = []
         if self.include_vars:
             try:
                 # we likely want to fix stackdata at some point, but
                 # still need a workaround.
                 fibp = frame_info.variables_in_executing_piece
                 for var in fibp:
                     lvals_list.append(tpl_name_val % (var.name, repr(var.value)))
             except Exception:
                 lvals_list.append(
                     "Exception trying to inspect frame. No more locals available."
                 )
         if lvals_list:
             lvals = '%s%s' % (indent, em_normal.join(lvals_list))
 
         result = f'{link}{", " if call else ""}{call}\n'
         if frame_info._sd is None:
             # fast fallback if file is too long
             tpl_link = "%s%%s%s" % (Colors.filenameEm, ColorsNormal)
             link = tpl_link % util_path.compress_user(frame_info.filename)
             level = "%s %s\n" % (link, call)
             _line_format = PyColorize.Parser(
                 style=self.color_scheme_table.active_scheme_name, parent=self
             ).format2
             first_line = frame_info.code.co_firstlineno
             current_line = frame_info.lineno[0]
             raw_lines = frame_info.raw_lines
             index = current_line - first_line
 
             if index >= frame_info.context:
                 start = max(index - frame_info.context, 0)
                 stop = index + frame_info.context
                 index = frame_info.context
             else:
                 start = 0
                 stop = index + frame_info.context
             raw_lines = raw_lines[start:stop]
 
             return "%s%s" % (
                 level,
                 "".join(
                     _simple_format_traceback_lines(
                         current_line,
                         index,
                         raw_lines,
                         Colors,
                         lvals,
                         _line_format,
                     )
                 ),
             )
             # result += "\n".join(frame_info.raw_lines)
         else:
             result += "".join(
                 _format_traceback_lines(
                     frame_info.lines, Colors, self.has_colors, lvals
                 )
             )
         return result
 
     def prepare_header(self, etype: str, long_version: bool = False):
         colors = self.Colors  # just a shorthand + quicker name lookup
         colorsnormal = colors.Normal  # used a lot
         exc = '%s%s%s' % (colors.excName, etype, colorsnormal)
         width = min(75, get_terminal_size()[0])
         if long_version:
             # Header with the exception type, python version, and date
             pyver = 'Python ' + sys.version.split()[0] + ': ' + sys.executable
             date = time.ctime(time.time())
 
             head = "%s%s%s\n%s%s%s\n%s" % (
                 colors.topline,
                 "-" * width,
                 colorsnormal,
                 exc,
                 " " * (width - len(etype) - len(pyver)),
                 pyver,
                 date.rjust(width),
             )
             head += (
                 "\nA problem occurred executing Python code.  Here is the sequence of function"
                 "\ncalls leading up to the error, with the most recent (innermost) call last."
             )
         else:
             # Simplified header
             head = "%s%s" % (
                 exc,
                 "Traceback (most recent call last)".rjust(width - len(etype)),
             )
 
         return head
 
     def format_exception(self, etype, evalue):
         colors = self.Colors  # just a shorthand + quicker name lookup
         colorsnormal = colors.Normal  # used a lot
         # Get (safely) a string form of the exception info
         try:
             etype_str, evalue_str = map(str, (etype, evalue))
         except:
             # User exception is improperly defined.
             etype, evalue = str, sys.exc_info()[:2]
             etype_str, evalue_str = map(str, (etype, evalue))
 
         # PEP-678 notes
         notes = getattr(evalue, "__notes__", [])
         if not isinstance(notes, Sequence) or isinstance(notes, (str, bytes)):
             notes = [_safe_string(notes, "__notes__", func=repr)]
 
         # ... and format it
         return [
             "{}{}{}: {}".format(
                 colors.excName,
                 etype_str,
                 colorsnormal,
-                evalue_str,
+                py3compat.cast_unicode(evalue_str),
             ),
             *(
                 "{}{}".format(
                     colorsnormal, _safe_string(py3compat.cast_unicode(n), "note")
                 )
                 for n in notes
             ),
         ]
 
     def format_exception_as_a_whole(
         self,
         etype: type,
         evalue: Optional[BaseException],
         etb: Optional[TracebackType],
         number_of_lines_of_context,
         tb_offset: Optional[int],
     ):
         """Formats the header, traceback and exception message for a single exception.
 
         This may be called multiple times by Python 3 exception chaining
         (PEP 3134).
         """
+        # some locals
+        orig_etype = etype
         try:
             etype = etype.__name__  # type: ignore
         except AttributeError:
             pass
 
         tb_offset = self.tb_offset if tb_offset is None else tb_offset
         assert isinstance(tb_offset, int)
         head = self.prepare_header(str(etype), self.long_header)
         records = (
             self.get_records(etb, number_of_lines_of_context, tb_offset) if etb else []
         )
 
         frames = []
         skipped = 0
         lastrecord = len(records) - 1
         for i, record in enumerate(records):
             if (
                 not isinstance(record._sd, stack_data.RepeatedFrames)
                 and self.skip_hidden
             ):
                 if (
                     record.frame.f_locals.get("__tracebackhide__", 0)
                     and i != lastrecord
                 ):
                     skipped += 1
                     continue
             if skipped:
                 Colors = self.Colors  # just a shorthand + quicker name lookup
                 ColorsNormal = Colors.Normal  # used a lot
                 frames.append(
                     "    %s[... skipping hidden %s frame]%s\n"
                     % (Colors.excName, skipped, ColorsNormal)
                 )
                 skipped = 0
             frames.append(self.format_record(record))
         if skipped:
             Colors = self.Colors  # just a shorthand + quicker name lookup
             ColorsNormal = Colors.Normal  # used a lot
             frames.append(
                 "    %s[... skipping hidden %s frame]%s\n"
                 % (Colors.excName, skipped, ColorsNormal)
             )
 
         formatted_exception = self.format_exception(etype, evalue)
         if records:
             frame_info = records[-1]
             ipinst = get_ipython()
             if ipinst is not None:
                 ipinst.hooks.synchronize_with_editor(frame_info.filename, frame_info.lineno, 0)
 
         return [[head] + frames + formatted_exception]
 
     def get_records(
         self, etb: TracebackType, number_of_lines_of_context: int, tb_offset: int
     ):
         assert etb is not None
         context = number_of_lines_of_context - 1
         after = context // 2
         before = context - after
         if self.has_colors:
             style = get_style_by_name(self.tb_highlight_style)
             style = stack_data.style_with_executing_node(style, self.tb_highlight)
             formatter = Terminal256Formatter(style=style)
         else:
             formatter = None
         options = stack_data.Options(
             before=before,
             after=after,
             pygments_formatter=formatter,
         )
 
         # Let's estimate the amount of code we will have to parse/highlight.
         cf: Optional[TracebackType] = etb
         max_len = 0
         tbs = []
         while cf is not None:
             try:
                 mod = inspect.getmodule(cf.tb_frame)
                 if mod is not None:
                     mod_name = mod.__name__
                     root_name, *_ = mod_name.split(".")
                     if root_name == "IPython":
                         cf = cf.tb_next
                         continue
                 max_len = get_line_number_of_frame(cf.tb_frame)
 
             except OSError:
                 max_len = 0
             max_len = max(max_len, max_len)
             tbs.append(cf)
             cf = getattr(cf, "tb_next", None)
 
         if max_len > FAST_THRESHOLD:
             FIs = []
             for tb in tbs:
                 frame = tb.tb_frame  # type: ignore
                 lineno = (frame.f_lineno,)
                 code = frame.f_code
                 filename = code.co_filename
                 # TODO: Here we need to use before/after/
                 FIs.append(
                     FrameInfo(
                         "Raw frame", filename, lineno, frame, code, context=context
                     )
                 )
             return FIs
         res = list(stack_data.FrameInfo.stack_data(etb, options=options))[tb_offset:]
         res = [FrameInfo._from_stack_data_FrameInfo(r) for r in res]
         return res
 
     def structured_traceback(
         self,
         etype: type,
         evalue: Optional[BaseException],
         etb: Optional[TracebackType] = None,
         tb_offset: Optional[int] = None,
         number_of_lines_of_context: int = 5,
     ):
         """Return a nice text document describing the traceback."""
         formatted_exception = self.format_exception_as_a_whole(etype, evalue, etb, number_of_lines_of_context,
                                                                tb_offset)
 
         colors = self.Colors  # just a shorthand + quicker name lookup
         colorsnormal = colors.Normal  # used a lot
         head = '%s%s%s' % (colors.topline, '-' * min(75, get_terminal_size()[0]), colorsnormal)
         structured_traceback_parts = [head]
         chained_exceptions_tb_offset = 0
         lines_of_context = 3
         formatted_exceptions = formatted_exception
         exception = self.get_parts_of_chained_exception(evalue)
         if exception:
             assert evalue is not None
             formatted_exceptions += self.prepare_chained_exception_message(evalue.__cause__)
             etype, evalue, etb = exception
         else:
             evalue = None
         chained_exc_ids = set()
         while evalue:
             formatted_exceptions += self.format_exception_as_a_whole(etype, evalue, etb, lines_of_context,
                                                                      chained_exceptions_tb_offset)
             exception = self.get_parts_of_chained_exception(evalue)
 
-            if exception and id(exception[1]) not in chained_exc_ids:
+            if exception and not id(exception[1]) in chained_exc_ids:
                 chained_exc_ids.add(id(exception[1])) # trace exception to avoid infinite 'cause' loop
                 formatted_exceptions += self.prepare_chained_exception_message(evalue.__cause__)
                 etype, evalue, etb = exception
             else:
                 evalue = None
 
         # we want to see exceptions in a reversed order:
         # the first exception should be on top
         for formatted_exception in reversed(formatted_exceptions):
             structured_traceback_parts += formatted_exception
 
         return structured_traceback_parts
 
     def debugger(self, force: bool = False):
         """Call up the pdb debugger if desired, always clean up the tb
         reference.
 
         Keywords:
 
           - force(False): by default, this routine checks the instance call_pdb
             flag and does not actually invoke the debugger if the flag is false.
             The 'force' option forces the debugger to activate even if the flag
             is false.
 
         If the call_pdb flag is set, the pdb interactive debugger is
         invoked. In all cases, the self.tb reference to the current traceback
         is deleted to prevent lingering references which hamper memory
         management.
 
         Note that each call to pdb() does an 'import readline', so if your app
         requires a special setup for the readline completers, you'll have to
         fix that by hand after invoking the exception handler."""
 
         if force or self.call_pdb:
             if self.pdb is None:
                 self.pdb = self.debugger_cls()
             # the system displayhook may have changed, restore the original
             # for pdb
             display_trap = DisplayTrap(hook=sys.__displayhook__)
             with display_trap:
                 self.pdb.reset()
                 # Find the right frame so we don't pop up inside ipython itself
                 if hasattr(self, "tb") and self.tb is not None:  # type: ignore[has-type]
                     etb = self.tb  # type: ignore[has-type]
                 else:
                     etb = self.tb = sys.last_traceback
                 while self.tb is not None and self.tb.tb_next is not None:
                     assert self.tb.tb_next is not None
                     self.tb = self.tb.tb_next
                 if etb and etb.tb_next:
                     etb = etb.tb_next
                 self.pdb.botframe = etb.tb_frame
                 # last_value should be deprecated, but last-exc sometimme not set
                 # please check why later and remove the getattr.
                 exc = sys.last_value if sys.version_info < (3, 12) else getattr(sys, "last_exc", sys.last_value)  # type: ignore[attr-defined]
                 if exc:
                     self.pdb.interaction(None, exc)
                 else:
                     self.pdb.interaction(None, etb)
 
         if hasattr(self, 'tb'):
             del self.tb
 
     def handler(self, info=None):
         (etype, evalue, etb) = info or sys.exc_info()
         self.tb = etb
         ostream = self.ostream
         ostream.flush()
         ostream.write(self.text(etype, evalue, etb))
         ostream.write('\n')
         ostream.flush()
 
     # Changed so an instance can just be called as VerboseTB_inst() and print
     # out the right info on its own.
     def __call__(self, etype=None, evalue=None, etb=None):
         """This hook can replace sys.excepthook (for Python 2.1 or higher)."""
         if etb is None:
             self.handler()
         else:
             self.handler((etype, evalue, etb))
         try:
             self.debugger()
         except KeyboardInterrupt:
             print("\nKeyboardInterrupt")
 
 
 #----------------------------------------------------------------------------
 class FormattedTB(VerboseTB, ListTB):
     """Subclass ListTB but allow calling with a traceback.
 
     It can thus be used as a sys.excepthook for Python > 2.1.
 
     Also adds 'Context' and 'Verbose' modes, not available in ListTB.
 
     Allows a tb_offset to be specified. This is useful for situations where
     one needs to remove a number of topmost frames from the traceback (such as
     occurs with python programs that themselves execute other python code,
     like Python shells).  """
 
     mode: str
 
     def __init__(self, mode='Plain', color_scheme='Linux', call_pdb=False,
                  ostream=None,
                  tb_offset=0, long_header=False, include_vars=False,
                  check_cache=None, debugger_cls=None,
                  parent=None, config=None):
 
         # NEVER change the order of this list. Put new modes at the end:
         self.valid_modes = ['Plain', 'Context', 'Verbose', 'Minimal']
         self.verbose_modes = self.valid_modes[1:3]
 
         VerboseTB.__init__(self, color_scheme=color_scheme, call_pdb=call_pdb,
                            ostream=ostream, tb_offset=tb_offset,
                            long_header=long_header, include_vars=include_vars,
                            check_cache=check_cache, debugger_cls=debugger_cls,
                            parent=parent, config=config)
 
         # Different types of tracebacks are joined with different separators to
         # form a single string.  They are taken from this dict
         self._join_chars = dict(Plain='', Context='\n', Verbose='\n',
                                 Minimal='')
         # set_mode also sets the tb_join_char attribute
         self.set_mode(mode)
 
     def structured_traceback(self, etype, value, tb, tb_offset=None, number_of_lines_of_context=5):
         tb_offset = self.tb_offset if tb_offset is None else tb_offset
         mode = self.mode
         if mode in self.verbose_modes:
             # Verbose modes need a full traceback
             return VerboseTB.structured_traceback(
                 self, etype, value, tb, tb_offset, number_of_lines_of_context
             )
         elif mode == 'Minimal':
             return ListTB.get_exception_only(self, etype, value)
         else:
             # We must check the source cache because otherwise we can print
             # out-of-date source code.
             self.check_cache()
             # Now we can extract and format the exception
             return ListTB.structured_traceback(
                 self, etype, value, tb, tb_offset, number_of_lines_of_context
             )
 
     def stb2text(self, stb):
         """Convert a structured traceback (a list) to a string."""
         return self.tb_join_char.join(stb)
 
     def set_mode(self, mode: Optional[str] = None):
         """Switch to the desired mode.
 
         If mode is not specified, cycles through the available modes."""
 
         if not mode:
             new_idx = (self.valid_modes.index(self.mode) + 1 ) % \
                       len(self.valid_modes)
             self.mode = self.valid_modes[new_idx]
         elif mode not in self.valid_modes:
             raise ValueError(
                 "Unrecognized mode in FormattedTB: <" + mode + ">\n"
                 "Valid modes: " + str(self.valid_modes)
             )
         else:
             assert isinstance(mode, str)
             self.mode = mode
         # include variable details only in 'Verbose' mode
         self.include_vars = (self.mode == self.valid_modes[2])
         # Set the join character for generating text tracebacks
         self.tb_join_char = self._join_chars[self.mode]
 
     # some convenient shortcuts
     def plain(self):
         self.set_mode(self.valid_modes[0])
 
     def context(self):
         self.set_mode(self.valid_modes[1])
 
     def verbose(self):
         self.set_mode(self.valid_modes[2])
 
     def minimal(self):
         self.set_mode(self.valid_modes[3])
 
 
 #----------------------------------------------------------------------------
 class AutoFormattedTB(FormattedTB):
     """A traceback printer which can be called on the fly.
 
     It will find out about exceptions by itself.
 
     A brief example::
 
         AutoTB = AutoFormattedTB(mode = 'Verbose',color_scheme='Linux')
         try:
           ...
-        except Exception:
+        except:
           AutoTB()  # or AutoTB(out=logfile) where logfile is an open file object
     """
 
     def __call__(self, etype=None, evalue=None, etb=None,
                  out=None, tb_offset=None):
         """Print out a formatted exception traceback.
 
         Optional arguments:
           - out: an open file-like object to direct output to.
 
           - tb_offset: the number of frames to skip over in the stack, on a
           per-call basis (this overrides temporarily the instance's tb_offset
           given at initialization time."""
 
         if out is None:
             out = self.ostream
         out.flush()
         out.write(self.text(etype, evalue, etb, tb_offset))
         out.write('\n')
         out.flush()
         # FIXME: we should remove the auto pdb behavior from here and leave
         # that to the clients.
         try:
             self.debugger()
         except KeyboardInterrupt:
             print("\nKeyboardInterrupt")
 
     def structured_traceback(
         self,
         etype: type,
         evalue: Optional[BaseException],
         etb: Optional[TracebackType] = None,
         tb_offset: Optional[int] = None,
         number_of_lines_of_context: int = 5,
     ):
         # tb: TracebackType or tupleof tb types ?
         if etype is None:
             etype, evalue, etb = sys.exc_info()
         if isinstance(etb, tuple):
             # tb is a tuple if this is a chained exception.
             self.tb = etb[0]
         else:
             self.tb = etb
         return FormattedTB.structured_traceback(
             self, etype, evalue, etb, tb_offset, number_of_lines_of_context
         )
 
 
 #---------------------------------------------------------------------------
 
 # A simple class to preserve Nathan's original functionality.
 class ColorTB(FormattedTB):
     """Shorthand to initialize a FormattedTB in Linux colors mode."""
 
     def __init__(self, color_scheme='Linux', call_pdb=0, **kwargs):
         FormattedTB.__init__(self, color_scheme=color_scheme,
                              call_pdb=call_pdb, **kwargs)
 
 
 class SyntaxTB(ListTB):
     """Extension which holds some state: the last exception value"""
 
     def __init__(self, color_scheme='NoColor', parent=None, config=None):
         ListTB.__init__(self, color_scheme, parent=parent, config=config)
         self.last_syntax_error = None
 
     def __call__(self, etype, value, elist):
         self.last_syntax_error = value
 
         ListTB.__call__(self, etype, value, elist)
 
     def structured_traceback(self, etype, value, elist, tb_offset=None,
                              context=5):
         # If the source file has been edited, the line in the syntax error can
         # be wrong (retrieved from an outdated cache). This replaces it with
         # the current value.
         if isinstance(value, SyntaxError) \
                 and isinstance(value.filename, str) \
                 and isinstance(value.lineno, int):
             linecache.checkcache(value.filename)
             newtext = linecache.getline(value.filename, value.lineno)
             if newtext:
                 value.text = newtext
         self.last_syntax_error = value
         return super(SyntaxTB, self).structured_traceback(etype, value, elist,
                                                           tb_offset=tb_offset, context=context)
 
     def clear_err_state(self):
         """Return the current error state and clear it"""
         e = self.last_syntax_error
         self.last_syntax_error = None
         return e
 
     def stb2text(self, stb):
         """Convert a structured traceback (a list) to a string."""
         return ''.join(stb)
 
 
 # some internal-use functions
 def text_repr(value):
     """Hopefully pretty robust repr equivalent."""
     # this is pretty horrible but should always return *something*
     try:
         return pydoc.text.repr(value)  # type: ignore[call-arg]
     except KeyboardInterrupt:
         raise
-    except Exception:
+    except:
         try:
             return repr(value)
         except KeyboardInterrupt:
             raise
-        except Exception:
+        except:
             try:
                 # all still in an except block so we catch
                 # getattr raising
                 name = getattr(value, '__name__', None)
                 if name:
                     # ick, recursion
                     return text_repr(name)
                 klass = getattr(value, '__class__', None)
                 if klass:
                     return '%s instance' % text_repr(klass)
             except KeyboardInterrupt:
                 raise
-            except Exception:
+            except:
                 return 'UNRECOVERABLE REPR FAILURE'
 
 
 def eqrepr(value, repr=text_repr):
     return '=%s' % repr(value)
 
 
 def nullrepr(value, repr=text_repr):
     return ''

IPython/lib/demo.py

@@ -1,672 +1,672
 """Module for interactive demos using IPython.
 
 This module implements a few classes for running Python scripts interactively
 in IPython for demonstrations.  With very simple markup (a few tags in
 comments), you can control points where the script stops executing and returns
 control to IPython.
 
 
 Provided classes
 ----------------
 
 The classes are (see their docstrings for further details):
 
  - Demo: pure python demos
 
  - IPythonDemo: demos with input to be processed by IPython as if it had been
    typed interactively (so magics work, as well as any other special syntax you
    may have added via input prefilters).
 
  - LineDemo: single-line version of the Demo class.  These demos are executed
    one line at a time, and require no markup.
 
  - IPythonLineDemo: IPython version of the LineDemo class (the demo is
    executed a line at a time, but processed via IPython).
 
  - ClearMixin: mixin to make Demo classes with less visual clutter.  It
    declares an empty marquee and a pre_cmd that clears the screen before each
    block (see Subclassing below).
 
  - ClearDemo, ClearIPDemo: mixin-enabled versions of the Demo and IPythonDemo
    classes.
 
 Inheritance diagram:
 
 .. inheritance-diagram:: IPython.lib.demo
    :parts: 3
 
 Subclassing
 -----------
 
 The classes here all include a few methods meant to make customization by
 subclassing more convenient.  Their docstrings below have some more details:
 
   - highlight(): format every block and optionally highlight comments and
     docstring content.
 
   - marquee(): generates a marquee to provide visible on-screen markers at each
     block start and end.
 
   - pre_cmd(): run right before the execution of each block.
 
   - post_cmd(): run right after the execution of each block.  If the block
     raises an exception, this is NOT called.
 
 
 Operation
 ---------
 
 The file is run in its own empty namespace (though you can pass it a string of
 arguments as if in a command line environment, and it will see those as
 sys.argv).  But at each stop, the global IPython namespace is updated with the
 current internal demo namespace, so you can work interactively with the data
 accumulated so far.
 
 By default, each block of code is printed (with syntax highlighting) before
 executing it and you have to confirm execution.  This is intended to show the
 code to an audience first so you can discuss it, and only proceed with
 execution once you agree.  There are a few tags which allow you to modify this
 behavior.
 
 The supported tags are:
 
 # <demo> stop
 
   Defines block boundaries, the points where IPython stops execution of the
   file and returns to the interactive prompt.
 
   You can optionally mark the stop tag with extra dashes before and after the
   word 'stop', to help visually distinguish the blocks in a text editor:
 
   # <demo> --- stop ---
 
 
 # <demo> silent
 
   Make a block execute silently (and hence automatically).  Typically used in
   cases where you have some boilerplate or initialization code which you need
   executed but do not want to be seen in the demo.
 
 # <demo> auto
 
   Make a block execute automatically, but still being printed.  Useful for
   simple code which does not warrant discussion, since it avoids the extra
   manual confirmation.
 
 # <demo> auto_all
 
   This tag can _only_ be in the first block, and if given it overrides the
   individual auto tags to make the whole demo fully automatic (no block asks
   for confirmation).  It can also be given at creation time (or the attribute
   set later) to override what's in the file.
 
 While _any_ python file can be run as a Demo instance, if there are no stop
 tags the whole file will run in a single block (no different that calling
 first %pycat and then %run).  The minimal markup to make this useful is to
 place a set of stop tags; the other tags are only there to let you fine-tune
 the execution.
 
 This is probably best explained with the simple example file below.  You can
 copy this into a file named ex_demo.py, and try running it via::
 
     from IPython.lib.demo import Demo
     d = Demo('ex_demo.py')
     d()
 
 Each time you call the demo object, it runs the next block.  The demo object
 has a few useful methods for navigation, like again(), edit(), jump(), seek()
 and back().  It can be reset for a new run via reset() or reloaded from disk
 (in case you've edited the source) via reload().  See their docstrings below.
 
 Note: To make this simpler to explore, a file called "demo-exercizer.py" has
 been added to the "docs/examples/core" directory.  Just cd to this directory in
 an IPython session, and type::
 
   %run demo-exercizer.py
 
 and then follow the directions.
 
 Example
 -------
 
 The following is a very simple example of a valid demo file.
 
 ::
 
     #################### EXAMPLE DEMO <ex_demo.py> ###############################
     '''A simple interactive demo to illustrate the use of IPython's Demo class.'''
 
     print('Hello, welcome to an interactive IPython demo.')
 
     # The mark below defines a block boundary, which is a point where IPython will
     # stop execution and return to the interactive prompt. The dashes are actually
     # optional and used only as a visual aid to clearly separate blocks while
     # editing the demo code.
     # <demo> stop
 
     x = 1
     y = 2
 
     # <demo> stop
 
     # the mark below makes this block as silent
     # <demo> silent
 
     print('This is a silent block, which gets executed but not printed.')
 
     # <demo> stop
     # <demo> auto
     print('This is an automatic block.')
     print('It is executed without asking for confirmation, but printed.')
     z = x + y
 
     print('z =', x)
 
     # <demo> stop
     # This is just another normal block.
     print('z is now:', z)
 
     print('bye!')
     ################### END EXAMPLE DEMO <ex_demo.py> ############################
 """
 
 
 #*****************************************************************************
 #     Copyright (C) 2005-2006 Fernando Perez. <[e-mail]>
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #
 #*****************************************************************************
 
 import os
 import re
 import shlex
 import sys
 import pygments
 from pathlib import Path
 
 from IPython.utils.text import marquee
 from IPython.utils import openpy
 from IPython.utils import py3compat
 __all__ = ['Demo','IPythonDemo','LineDemo','IPythonLineDemo','DemoError']
 
 class DemoError(Exception): pass
 
 def re_mark(mark):
     return re.compile(r'^\s*#\s+<demo>\s+%s\s*$' % mark,re.MULTILINE)
 
 class Demo(object):
 
     re_stop     = re_mark(r'-*\s?stop\s?-*')
     re_silent   = re_mark('silent')
     re_auto     = re_mark('auto')
     re_auto_all = re_mark('auto_all')
 
     def __init__(self,src,title='',arg_str='',auto_all=None, format_rst=False,
                  formatter='terminal', style='default'):
         """Make a new demo object.  To run the demo, simply call the object.
 
         See the module docstring for full details and an example (you can use
         IPython.Demo? in IPython to see it).
 
         Inputs:
 
           - src is either a file, or file-like object, or a
               string that can be resolved to a filename.
 
         Optional inputs:
 
           - title: a string to use as the demo name.  Of most use when the demo
             you are making comes from an object that has no filename, or if you
             want an alternate denotation distinct from the filename.
 
           - arg_str(''): a string of arguments, internally converted to a list
             just like sys.argv, so the demo script can see a similar
             environment.
 
           - auto_all(None): global flag to run all blocks automatically without
             confirmation.  This attribute overrides the block-level tags and
             applies to the whole demo.  It is an attribute of the object, and
             can be changed at runtime simply by reassigning it to a boolean
             value.
 
           - format_rst(False): a bool to enable comments and doc strings
             formatting with pygments rst lexer
 
           - formatter('terminal'): a string of pygments formatter name to be
             used. Useful values for terminals: terminal, terminal256,
             terminal16m
 
           - style('default'): a string of pygments style name to be used.
         """
         if hasattr(src, "read"):
              # It seems to be a file or a file-like object
             self.fname = "from a file-like object"
             if title == '':
                 self.title = "from a file-like object"
             else:
                 self.title = title
         else:
              # Assume it's a string or something that can be converted to one
             self.fname = src
             if title == '':
                 (filepath, filename) = os.path.split(src)
                 self.title = filename
             else:
                 self.title = title
         self.sys_argv = [src] + shlex.split(arg_str)
         self.auto_all = auto_all
         self.src = src
 
         try:
             ip = get_ipython()  # this is in builtins whenever IPython is running
             self.inside_ipython = True
         except NameError:
             self.inside_ipython = False
 
         if self.inside_ipython:
             # get a few things from ipython.  While it's a bit ugly design-wise,
             # it ensures that things like color scheme and the like are always in
             # sync with the ipython mode being used.  This class is only meant to
             # be used inside ipython anyways,  so it's OK.
             self.ip_ns       = ip.user_ns
             self.ip_colorize = ip.pycolorize
             self.ip_showtb   = ip.showtraceback
             self.ip_run_cell = ip.run_cell
             self.shell       = ip
 
         self.formatter = pygments.formatters.get_formatter_by_name(formatter,
                                                                    style=style)
         self.python_lexer = pygments.lexers.get_lexer_by_name("py3")
         self.format_rst = format_rst
         if format_rst:
             self.rst_lexer = pygments.lexers.get_lexer_by_name("rst")
 
         # load user data and initialize data structures
         self.reload()
 
     def fload(self):
         """Load file object."""
         # read data and parse into blocks
         if hasattr(self, 'fobj') and self.fobj is not None:
            self.fobj.close()
         if hasattr(self.src, "read"):
              # It seems to be a file or a file-like object
             self.fobj = self.src
         else:
              # Assume it's a string or something that can be converted to one
             self.fobj = openpy.open(self.fname)
 
     def reload(self):
         """Reload source from disk and initialize state."""
         self.fload()
 
         self.src     = "".join(openpy.strip_encoding_cookie(self.fobj))
         src_b        = [b.strip() for b in self.re_stop.split(self.src) if b]
         self._silent = [bool(self.re_silent.findall(b)) for b in src_b]
         self._auto   = [bool(self.re_auto.findall(b)) for b in src_b]
 
         # if auto_all is not given (def. None), we read it from the file
         if self.auto_all is None:
             self.auto_all = bool(self.re_auto_all.findall(src_b[0]))
         else:
             self.auto_all = bool(self.auto_all)
 
         # Clean the sources from all markup so it doesn't get displayed when
         # running the demo
         src_blocks = []
         auto_strip = lambda s: self.re_auto.sub('',s)
         for i,b in enumerate(src_b):
             if self._auto[i]:
                 src_blocks.append(auto_strip(b))
             else:
                 src_blocks.append(b)
         # remove the auto_all marker
         src_blocks[0] = self.re_auto_all.sub('',src_blocks[0])
 
         self.nblocks = len(src_blocks)
         self.src_blocks = src_blocks
 
         # also build syntax-highlighted source
         self.src_blocks_colored = list(map(self.highlight,self.src_blocks))
 
         # ensure clean namespace and seek offset
         self.reset()
 
     def reset(self):
         """Reset the namespace and seek pointer to restart the demo"""
         self.user_ns     = {}
         self.finished    = False
         self.block_index = 0
 
     def _validate_index(self,index):
         if index<0 or index>=self.nblocks:
             raise ValueError('invalid block index %s' % index)
 
     def _get_index(self,index):
         """Get the current block index, validating and checking status.
 
         Returns None if the demo is finished"""
 
         if index is None:
             if self.finished:
                 print('Demo finished.  Use <demo_name>.reset() if you want to rerun it.')
                 return None
             index = self.block_index
         else:
             self._validate_index(index)
         return index
 
     def seek(self,index):
         """Move the current seek pointer to the given block.
 
         You can use negative indices to seek from the end, with identical
         semantics to those of Python lists."""
         if index<0:
             index = self.nblocks + index
         self._validate_index(index)
         self.block_index = index
         self.finished = False
 
     def back(self,num=1):
         """Move the seek pointer back num blocks (default is 1)."""
         self.seek(self.block_index-num)
 
     def jump(self,num=1):
         """Jump a given number of blocks relative to the current one.
 
         The offset can be positive or negative, defaults to 1."""
         self.seek(self.block_index+num)
 
     def again(self):
         """Move the seek pointer back one block and re-execute."""
         self.back(1)
         self()
 
     def edit(self,index=None):
         """Edit a block.
 
         If no number is given, use the last block executed.
 
         This edits the in-memory copy of the demo, it does NOT modify the
         original source file.  If you want to do that, simply open the file in
         an editor and use reload() when you make changes to the file.  This
         method is meant to let you change a block during a demonstration for
         explanatory purposes, without damaging your original script."""
 
         index = self._get_index(index)
         if index is None:
             return
         # decrease the index by one (unless we're at the very beginning), so
         # that the default demo.edit() call opens up the sblock we've last run
         if index>0:
             index -= 1
 
         filename = self.shell.mktempfile(self.src_blocks[index])
         self.shell.hooks.editor(filename, 1)
         with open(Path(filename), "r", encoding="utf-8") as f:
             new_block = f.read()
         # update the source and colored block
         self.src_blocks[index] = new_block
         self.src_blocks_colored[index] = self.highlight(new_block)
         self.block_index = index
         # call to run with the newly edited index
         self()
 
     def show(self,index=None):
         """Show a single block on screen"""
 
         index = self._get_index(index)
         if index is None:
             return
 
         print(self.marquee('<%s> block # %s (%s remaining)' %
                            (self.title,index,self.nblocks-index-1)))
         print(self.src_blocks_colored[index])
         sys.stdout.flush()
 
     def show_all(self):
         """Show entire demo on screen, block by block"""
 
-        self.title
+        fname = self.title
         title = self.title
         nblocks = self.nblocks
         silent = self._silent
         marquee = self.marquee
         for index,block in enumerate(self.src_blocks_colored):
             if silent[index]:
                 print(marquee('<%s> SILENT block # %s (%s remaining)' %
                               (title,index,nblocks-index-1)))
             else:
                 print(marquee('<%s> block # %s (%s remaining)' %
                               (title,index,nblocks-index-1)))
             print(block, end=' ')
         sys.stdout.flush()
 
     def run_cell(self,source):
         """Execute a string with one or more lines of code"""
 
         exec(source, self.user_ns)
 
     def __call__(self,index=None):
         """run a block of the demo.
 
         If index is given, it should be an integer >=1 and <= nblocks.  This
         means that the calling convention is one off from typical Python
         lists.  The reason for the inconsistency is that the demo always
         prints 'Block n/N, and N is the total, so it would be very odd to use
         zero-indexing here."""
 
         index = self._get_index(index)
         if index is None:
             return
         try:
             marquee = self.marquee
             next_block = self.src_blocks[index]
             self.block_index += 1
             if self._silent[index]:
                 print(marquee('Executing silent block # %s (%s remaining)' %
                               (index,self.nblocks-index-1)))
             else:
                 self.pre_cmd()
                 self.show(index)
                 if self.auto_all or self._auto[index]:
                     print(marquee('output:'))
                 else:
                     print(marquee('Press <q> to quit, <Enter> to execute...'), end=' ')
                     ans = py3compat.input().strip()
                     if ans:
                         print(marquee('Block NOT executed'))
                         return
             try:
                 save_argv = sys.argv
                 sys.argv = self.sys_argv
                 self.run_cell(next_block)
                 self.post_cmd()
             finally:
                 sys.argv = save_argv
 
         except:
             if self.inside_ipython:
                 self.ip_showtb(filename=self.fname)
         else:
             if self.inside_ipython:
                 self.ip_ns.update(self.user_ns)
 
         if self.block_index == self.nblocks:
             mq1 = self.marquee('END OF DEMO')
             if mq1:
                 # avoid spurious print if empty marquees are used
                 print()
                 print(mq1)
                 print(self.marquee('Use <demo_name>.reset() if you want to rerun it.'))
             self.finished = True
 
     # These methods are meant to be overridden by subclasses who may wish to
     # customize the behavior of of their demos.
     def marquee(self,txt='',width=78,mark='*'):
         """Return the input string centered in a 'marquee'."""
         return marquee(txt,width,mark)
 
     def pre_cmd(self):
         """Method called before executing each block."""
         pass
 
     def post_cmd(self):
         """Method called after executing each block."""
         pass
 
     def highlight(self, block):
         """Method called on each block to highlight it content"""
         tokens = pygments.lex(block, self.python_lexer)
         if self.format_rst:
             from pygments.token import Token
             toks = []
             for token in tokens:
                 if token[0] == Token.String.Doc and len(token[1]) > 6:
                     toks += pygments.lex(token[1][:3], self.python_lexer)
                     # parse doc string content by rst lexer
                     toks += pygments.lex(token[1][3:-3], self.rst_lexer)
                     toks += pygments.lex(token[1][-3:], self.python_lexer)
                 elif token[0] == Token.Comment.Single:
                     toks.append((Token.Comment.Single, token[1][0]))
                     # parse comment content by rst lexer
                     # remove the extra newline added by rst lexer
                     toks += list(pygments.lex(token[1][1:], self.rst_lexer))[:-1]
                 else:
                     toks.append(token)
             tokens = toks
         return pygments.format(tokens, self.formatter)
 
 
 class IPythonDemo(Demo):
     """Class for interactive demos with IPython's input processing applied.
 
     This subclasses Demo, but instead of executing each block by the Python
     interpreter (via exec), it actually calls IPython on it, so that any input
     filters which may be in place are applied to the input block.
 
     If you have an interactive environment which exposes special input
     processing, you can use this class instead to write demo scripts which
     operate exactly as if you had typed them interactively.  The default Demo
     class requires the input to be valid, pure Python code.
     """
 
     def run_cell(self,source):
         """Execute a string with one or more lines of code"""
 
         self.shell.run_cell(source)
 
 class LineDemo(Demo):
     """Demo where each line is executed as a separate block.
 
     The input script should be valid Python code.
 
     This class doesn't require any markup at all, and it's meant for simple
     scripts (with no nesting or any kind of indentation) which consist of
     multiple lines of input to be executed, one at a time, as if they had been
     typed in the interactive prompt.
 
     Note: the input can not have *any* indentation, which means that only
     single-lines of input are accepted, not even function definitions are
     valid."""
 
     def reload(self):
         """Reload source from disk and initialize state."""
         # read data and parse into blocks
         self.fload()
         lines           = self.fobj.readlines()
         src_b           = [l for l in lines if l.strip()]
         nblocks         = len(src_b)
         self.src        = ''.join(lines)
         self._silent    = [False]*nblocks
         self._auto      = [True]*nblocks
         self.auto_all   = True
         self.nblocks    = nblocks
         self.src_blocks = src_b
 
         # also build syntax-highlighted source
         self.src_blocks_colored = list(map(self.highlight,self.src_blocks))
 
         # ensure clean namespace and seek offset
         self.reset()
 
 
 class IPythonLineDemo(IPythonDemo,LineDemo):
     """Variant of the LineDemo class whose input is processed by IPython."""
     pass
 
 
 class ClearMixin(object):
     """Use this mixin to make Demo classes with less visual clutter.
 
     Demos using this mixin will clear the screen before every block and use
     blank marquees.
 
     Note that in order for the methods defined here to actually override those
     of the classes it's mixed with, it must go /first/ in the inheritance
     tree.  For example:
 
         class ClearIPDemo(ClearMixin,IPythonDemo): pass
 
     will provide an IPythonDemo class with the mixin's features.
     """
 
     def marquee(self,txt='',width=78,mark='*'):
         """Blank marquee that returns '' no matter what the input."""
         return ''
 
     def pre_cmd(self):
         """Method called before executing each block.
 
         This one simply clears the screen."""
         from IPython.utils.terminal import _term_clear
         _term_clear()
 
 class ClearDemo(ClearMixin,Demo):
     pass
 
 
 class ClearIPDemo(ClearMixin,IPythonDemo):
     pass
 
 
 def slide(file_path, noclear=False, format_rst=True, formatter="terminal",
           style="native", auto_all=False, delimiter='...'):
     if noclear:
         demo_class = Demo
     else:
         demo_class = ClearDemo
     demo = demo_class(file_path, format_rst=format_rst, formatter=formatter,
                       style=style, auto_all=auto_all)
     while not demo.finished:
         demo()
         try:
             py3compat.input('\n' + delimiter)
         except KeyboardInterrupt:
             exit(1)
 
 if __name__ == '__main__':
     import argparse
     parser = argparse.ArgumentParser(description='Run python demos')
     parser.add_argument('--noclear', '-C', action='store_true',
                         help='Do not clear terminal on each slide')
     parser.add_argument('--rst', '-r', action='store_true',
                         help='Highlight comments and dostrings as rst')
     parser.add_argument('--formatter', '-f', default='terminal',
                         help='pygments formatter name could be: terminal, '
                         'terminal256, terminal16m')
     parser.add_argument('--style', '-s', default='default',
                         help='pygments style name')
     parser.add_argument('--auto', '-a', action='store_true',
                         help='Run all blocks automatically without'
                         'confirmation')
     parser.add_argument('--delimiter', '-d', default='...',
                         help='slides delimiter added after each slide run')
     parser.add_argument('file', nargs=1,
                         help='python demo file')
     args = parser.parse_args()
     slide(args.file[0], noclear=args.noclear, format_rst=args.rst,
           formatter=args.formatter, style=args.style, auto_all=args.auto,
           delimiter=args.delimiter)

IPython/lib/tests/test_display.py

@@ -1,278 +1,272
 """Tests for IPython.lib.display.
 
 """
 #-----------------------------------------------------------------------------
 # Copyright (c) 2012, the IPython Development Team.
 #
 # Distributed under the terms of the Modified BSD License.
 #
 # The full license is in the file COPYING.txt, distributed with this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 from tempfile import NamedTemporaryFile, mkdtemp
 from os.path import split, join as pjoin, dirname
 import pathlib
 from unittest import TestCase, mock
 import struct
 import wave
 from io import BytesIO
 
 # Third-party imports
 import pytest
 
 try:
     import numpy
 except ImportError:
     pass
 
 # Our own imports
 from IPython.lib import display
 
 from IPython.testing.decorators import skipif_not_numpy
 
 #-----------------------------------------------------------------------------
 # Classes and functions
 #-----------------------------------------------------------------------------
 
 #--------------------------
 # FileLink tests
 #--------------------------
 
 def test_instantiation_FileLink():
     """FileLink: Test class can be instantiated"""
     fl = display.FileLink('example.txt')
     # TODO: remove if when only Python >= 3.6 is supported
     fl = display.FileLink(pathlib.PurePath('example.txt'))
 
 def test_warning_on_non_existent_path_FileLink():
     """FileLink: Calling _repr_html_ on non-existent files returns a warning"""
     fl = display.FileLink("example.txt")
     assert fl._repr_html_().startswith("Path (<tt>example.txt</tt>)")
 
 
 def test_existing_path_FileLink():
     """FileLink: Calling _repr_html_ functions as expected on existing filepath
     """
-    with NamedTemporaryFile() as tf:
-        fl = display.FileLink(tf.name)
-        actual = fl._repr_html_()
-        expected = "<a href='%s' target='_blank'>%s</a><br>" % (tf.name, tf.name)
-        assert actual == expected
+    tf = NamedTemporaryFile()
+    fl = display.FileLink(tf.name)
+    actual = fl._repr_html_()
+    expected = "<a href='%s' target='_blank'>%s</a><br>" % (tf.name, tf.name)
+    assert actual == expected
 
 
 def test_existing_path_FileLink_repr():
     """FileLink: Calling repr() functions as expected on existing filepath
     """
-    with NamedTemporaryFile() as tf:
-        fl = display.FileLink(tf.name)
-        actual = repr(fl)
-        expected = tf.name
-        assert actual == expected
+    tf = NamedTemporaryFile()
+    fl = display.FileLink(tf.name)
+    actual = repr(fl)
+    expected = tf.name
+    assert actual == expected
 
 
 def test_error_on_directory_to_FileLink():
     """FileLink: Raises error when passed directory
     """
     td = mkdtemp()
     pytest.raises(ValueError, display.FileLink, td)
 
 #--------------------------
 # FileLinks tests
 #--------------------------
 
 def test_instantiation_FileLinks():
     """FileLinks: Test class can be instantiated
     """
     fls = display.FileLinks('example')
 
 def test_warning_on_non_existent_path_FileLinks():
     """FileLinks: Calling _repr_html_ on non-existent files returns a warning"""
     fls = display.FileLinks("example")
     assert fls._repr_html_().startswith("Path (<tt>example</tt>)")
 
 
 def test_existing_path_FileLinks():
     """FileLinks: Calling _repr_html_ functions as expected on existing dir
     """
     td = mkdtemp()
-    with NamedTemporaryFile(dir=td) as tf1, NamedTemporaryFile(dir=td) as tf2:
-        fl = display.FileLinks(td)
-        actual = fl._repr_html_()
-        actual = actual.split("\n")
-        actual.sort()
-        # the links should always have forward slashes, even on windows, so replace
-        # backslashes with forward slashes here
-        expected = [
-            "%s/<br>" % td,
-            "&nbsp;&nbsp;<a href='%s' target='_blank'>%s</a><br>"
-            % (tf2.name.replace("\\", "/"), split(tf2.name)[1]),
-            "&nbsp;&nbsp;<a href='%s' target='_blank'>%s</a><br>"
-            % (tf1.name.replace("\\", "/"), split(tf1.name)[1]),
-        ]
-        expected.sort()
-        # We compare the sorted list of links here as that's more reliable
-        assert actual == expected
+    tf1 = NamedTemporaryFile(dir=td)
+    tf2 = NamedTemporaryFile(dir=td)
+    fl = display.FileLinks(td)
+    actual = fl._repr_html_()
+    actual = actual.split('\n')
+    actual.sort()
+    # the links should always have forward slashes, even on windows, so replace
+    # backslashes with forward slashes here
+    expected = ["%s/<br>" % td,
+                "&nbsp;&nbsp;<a href='%s' target='_blank'>%s</a><br>" %\
+                 (tf2.name.replace("\\","/"),split(tf2.name)[1]),
+                "&nbsp;&nbsp;<a href='%s' target='_blank'>%s</a><br>" %\
+                 (tf1.name.replace("\\","/"),split(tf1.name)[1])]
+    expected.sort()
+    # We compare the sorted list of links here as that's more reliable
+    assert actual == expected
 
 
 def test_existing_path_FileLinks_alt_formatter():
     """FileLinks: Calling _repr_html_ functions as expected w/ an alt formatter
     """
     td = mkdtemp()
-    with NamedTemporaryFile(dir=td), NamedTemporaryFile(dir=td):
-
-        def fake_formatter(dirname, fnames, included_suffixes):
-            return ["hello", "world"]
-
-        fl = display.FileLinks(td, notebook_display_formatter=fake_formatter)
-        actual = fl._repr_html_()
-        actual = actual.split("\n")
-        actual.sort()
-        expected = ["hello", "world"]
-        expected.sort()
-        # We compare the sorted list of links here as that's more reliable
-        assert actual == expected
+    tf1 = NamedTemporaryFile(dir=td)
+    tf2 = NamedTemporaryFile(dir=td)
+    def fake_formatter(dirname,fnames,included_suffixes):
+        return ["hello","world"]
+    fl = display.FileLinks(td,notebook_display_formatter=fake_formatter)
+    actual = fl._repr_html_()
+    actual = actual.split('\n')
+    actual.sort()
+    expected = ["hello","world"]
+    expected.sort()
+    # We compare the sorted list of links here as that's more reliable
+    assert actual == expected
 
 
 def test_existing_path_FileLinks_repr():
     """FileLinks: Calling repr() functions as expected on existing directory """
     td = mkdtemp()
-    with NamedTemporaryFile(dir=td) as tf1, NamedTemporaryFile(dir=td) as tf2:
-        fl = display.FileLinks(td)
-        actual = repr(fl)
-        actual = actual.split("\n")
-        actual.sort()
-        expected = [
-            "%s/" % td,
-            "  %s" % split(tf1.name)[1],
-            "  %s" % split(tf2.name)[1],
-        ]
-        expected.sort()
-        # We compare the sorted list of links here as that's more reliable
-        assert actual == expected
+    tf1 = NamedTemporaryFile(dir=td)
+    tf2 = NamedTemporaryFile(dir=td)
+    fl = display.FileLinks(td)
+    actual = repr(fl)
+    actual = actual.split('\n')
+    actual.sort()
+    expected = ['%s/' % td, '  %s' % split(tf1.name)[1],'  %s' % split(tf2.name)[1]]
+    expected.sort()
+    # We compare the sorted list of links here as that's more reliable
+    assert actual == expected
 
 
 def test_existing_path_FileLinks_repr_alt_formatter():
     """FileLinks: Calling repr() functions as expected w/ alt formatter
     """
     td = mkdtemp()
-    with NamedTemporaryFile(dir=td), NamedTemporaryFile(dir=td):
-
-        def fake_formatter(dirname, fnames, included_suffixes):
-            return ["hello", "world"]
-
-        fl = display.FileLinks(td, terminal_display_formatter=fake_formatter)
-        actual = repr(fl)
-        actual = actual.split("\n")
-        actual.sort()
-        expected = ["hello", "world"]
-        expected.sort()
-        # We compare the sorted list of links here as that's more reliable
-        assert actual == expected
+    tf1 = NamedTemporaryFile(dir=td)
+    tf2 = NamedTemporaryFile(dir=td)
+    def fake_formatter(dirname,fnames,included_suffixes):
+        return ["hello","world"]
+    fl = display.FileLinks(td,terminal_display_formatter=fake_formatter)
+    actual = repr(fl)
+    actual = actual.split('\n')
+    actual.sort()
+    expected = ["hello","world"]
+    expected.sort()
+    # We compare the sorted list of links here as that's more reliable
+    assert actual == expected
 
 
 def test_error_on_file_to_FileLinks():
     """FileLinks: Raises error when passed file
     """
     td = mkdtemp()
-    with NamedTemporaryFile(dir=td) as tf:
-        pytest.raises(ValueError, display.FileLinks, tf.name)
+    tf1 = NamedTemporaryFile(dir=td)
+    pytest.raises(ValueError, display.FileLinks, tf1.name)
 
 
 def test_recursive_FileLinks():
     """FileLinks: Does not recurse when recursive=False
     """
     td = mkdtemp()
-    with NamedTemporaryFile(dir=td):
-        subtd = mkdtemp(dir=td)
-        with NamedTemporaryFile(dir=subtd):
-            fl = display.FileLinks(td)
-            actual = str(fl)
-            actual = actual.split("\n")
-            assert len(actual) == 4, actual
-            fl = display.FileLinks(td, recursive=False)
-            actual = str(fl)
-            actual = actual.split("\n")
-            assert len(actual) == 2, actual
+    tf = NamedTemporaryFile(dir=td)
+    subtd = mkdtemp(dir=td)
+    subtf = NamedTemporaryFile(dir=subtd)
+    fl = display.FileLinks(td)
+    actual = str(fl)
+    actual = actual.split('\n')
+    assert len(actual) == 4, actual
+    fl = display.FileLinks(td, recursive=False)
+    actual = str(fl)
+    actual = actual.split('\n')
+    assert len(actual) == 2, actual
 
 def test_audio_from_file():
     path = pjoin(dirname(__file__), 'test.wav')
     display.Audio(filename=path)
 
 class TestAudioDataWithNumpy(TestCase):
 
     @skipif_not_numpy
     def test_audio_from_numpy_array(self):
         test_tone = get_test_tone()
         audio = display.Audio(test_tone, rate=44100)
         assert len(read_wav(audio.data)) == len(test_tone)
 
     @skipif_not_numpy
     def test_audio_from_list(self):
         test_tone = get_test_tone()
         audio = display.Audio(list(test_tone), rate=44100)
         assert len(read_wav(audio.data)) == len(test_tone)
 
     @skipif_not_numpy
     def test_audio_from_numpy_array_without_rate_raises(self):
         self.assertRaises(ValueError, display.Audio, get_test_tone())
 
     @skipif_not_numpy
     def test_audio_data_normalization(self):
         expected_max_value = numpy.iinfo(numpy.int16).max
         for scale in [1, 0.5, 2]:
             audio = display.Audio(get_test_tone(scale), rate=44100)
             actual_max_value = numpy.max(numpy.abs(read_wav(audio.data)))
             assert actual_max_value == expected_max_value
 
     @skipif_not_numpy
     def test_audio_data_without_normalization(self):
         max_int16 = numpy.iinfo(numpy.int16).max
         for scale in [1, 0.5, 0.2]:
             test_tone = get_test_tone(scale)
             test_tone_max_abs = numpy.max(numpy.abs(test_tone))
             expected_max_value = int(max_int16 * test_tone_max_abs)
             audio = display.Audio(test_tone, rate=44100, normalize=False)
             actual_max_value = numpy.max(numpy.abs(read_wav(audio.data)))
             assert actual_max_value == expected_max_value
 
     def test_audio_data_without_normalization_raises_for_invalid_data(self):
         self.assertRaises(
             ValueError,
             lambda: display.Audio([1.001], rate=44100, normalize=False))
         self.assertRaises(
             ValueError,
             lambda: display.Audio([-1.001], rate=44100, normalize=False))
 
 def simulate_numpy_not_installed():
     try:
         import numpy
         return mock.patch('numpy.array', mock.MagicMock(side_effect=ImportError))
     except ModuleNotFoundError:
         return lambda x:x
 
 @simulate_numpy_not_installed()
 class TestAudioDataWithoutNumpy(TestAudioDataWithNumpy):
     # All tests from `TestAudioDataWithNumpy` are inherited.
 
     @skipif_not_numpy
     def test_audio_raises_for_nested_list(self):
         stereo_signal = [list(get_test_tone())] * 2
         self.assertRaises(TypeError, lambda: display.Audio(stereo_signal, rate=44100))
 
 
 @skipif_not_numpy
 def get_test_tone(scale=1):
     return numpy.sin(2 * numpy.pi * 440 * numpy.linspace(0, 1, 44100)) * scale
 
 def read_wav(data):
     with wave.open(BytesIO(data)) as wave_file:
         wave_data = wave_file.readframes(wave_file.getnframes())
         num_samples = wave_file.getnframes() * wave_file.getnchannels()
         return struct.unpack('<%sh' % num_samples, wave_data)
 
 def test_code_from_file():
     c = display.Code(filename=__file__)
     assert c._repr_html_().startswith('<style>')

IPython/terminal/ptutils.py

@@ -1,203 +1,204
 """prompt-toolkit utilities
 
 Everything in this module is a private API,
 not to be used outside IPython.
 """
 
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 import unicodedata
 from wcwidth import wcwidth
 
 from IPython.core.completer import (
     provisionalcompleter, cursor_to_position,
     _deduplicate_completions)
 from prompt_toolkit.completion import Completer, Completion
 from prompt_toolkit.lexers import Lexer
 from prompt_toolkit.lexers import PygmentsLexer
 from prompt_toolkit.patch_stdout import patch_stdout
 
 import pygments.lexers as pygments_lexers
 import os
 import sys
 import traceback
 
 _completion_sentinel = object()
 
 def _elide_point(string:str, *, min_elide=30)->str:
     """
     If a string is long enough, and has at least 3 dots,
     replace the middle part with ellipses.
 
     If a string naming a file is long enough, and has at least 3 slashes,
     replace the middle part with ellipses.
 
     If three consecutive dots, or two consecutive dots are encountered these are
     replaced by the equivalents HORIZONTAL ELLIPSIS or TWO DOT LEADER unicode
     equivalents
     """
     string = string.replace('...','\N{HORIZONTAL ELLIPSIS}')
     string = string.replace('..','\N{TWO DOT LEADER}')
     if len(string) < min_elide:
         return string
 
     object_parts = string.split('.')
     file_parts = string.split(os.sep)
     if file_parts[-1] == '':
         file_parts.pop()
 
     if len(object_parts) > 3:
         return "{}.{}\N{HORIZONTAL ELLIPSIS}{}.{}".format(
             object_parts[0],
             object_parts[1][:1],
             object_parts[-2][-1:],
             object_parts[-1],
         )
 
     elif len(file_parts) > 3:
         return ("{}" + os.sep + "{}\N{HORIZONTAL ELLIPSIS}{}" + os.sep + "{}").format(
             file_parts[0], file_parts[1][:1], file_parts[-2][-1:], file_parts[-1]
         )
 
     return string
 
 def _elide_typed(string:str, typed:str, *, min_elide:int=30)->str:
     """
     Elide the middle of a long string if the beginning has already been typed.
     """
 
     if len(string) < min_elide:
         return string
     cut_how_much = len(typed)-3
     if cut_how_much < 7:
         return string
     if string.startswith(typed) and len(string)> len(typed):
         return f"{string[:3]}\N{HORIZONTAL ELLIPSIS}{string[cut_how_much:]}"
     return string
 
 def _elide(string:str, typed:str, min_elide=30)->str:
     return _elide_typed(
         _elide_point(string, min_elide=min_elide),
         typed, min_elide=min_elide)
 
 
 
 def _adjust_completion_text_based_on_context(text, body, offset):
     if text.endswith('=') and len(body) > offset and body[offset] == '=':
         return text[:-1]
     else:
         return text
 
 
 class IPythonPTCompleter(Completer):
     """Adaptor to provide IPython completions to prompt_toolkit"""
     def __init__(self, ipy_completer=None, shell=None):
         if shell is None and ipy_completer is None:
             raise TypeError("Please pass shell=an InteractiveShell instance.")
         self._ipy_completer = ipy_completer
         self.shell = shell
 
     @property
     def ipy_completer(self):
         if self._ipy_completer:
             return self._ipy_completer
         else:
             return self.shell.Completer
 
     def get_completions(self, document, complete_event):
         if not document.current_line.strip():
             return
         # Some bits of our completion system may print stuff (e.g. if a module
         # is imported). This context manager ensures that doesn't interfere with
         # the prompt.
 
         with patch_stdout(), provisionalcompleter():
             body = document.text
             cursor_row = document.cursor_position_row
             cursor_col = document.cursor_position_col
             cursor_position = document.cursor_position
             offset = cursor_to_position(body, cursor_row, cursor_col)
             try:
                 yield from self._get_completions(body, offset, cursor_position, self.ipy_completer)
-            except Exception:
+            except Exception as e:
                 try:
                     exc_type, exc_value, exc_tb = sys.exc_info()
                     traceback.print_exception(exc_type, exc_value, exc_tb)
                 except AttributeError:
                     print('Unrecoverable Error in completions')
 
     @staticmethod
     def _get_completions(body, offset, cursor_position, ipyc):
         """
         Private equivalent of get_completions() use only for unit_testing.
         """
+        debug = getattr(ipyc, 'debug', False)
         completions = _deduplicate_completions(
             body, ipyc.completions(body, offset))
         for c in completions:
             if not c.text:
                 # Guard against completion machinery giving us an empty string.
                 continue
             text = unicodedata.normalize('NFC', c.text)
             # When the first character of the completion has a zero length,
             # then it's probably a decomposed unicode character. E.g. caused by
             # the "\dot" completion. Try to compose again with the previous
             # character.
             if wcwidth(text[0]) == 0:
                 if cursor_position + c.start > 0:
                     char_before = body[c.start - 1]
                     fixed_text = unicodedata.normalize(
                         'NFC', char_before + text)
 
                     # Yield the modified completion instead, if this worked.
                     if wcwidth(text[0:1]) == 1:
                         yield Completion(fixed_text, start_position=c.start - offset - 1)
                         continue
 
             # TODO: Use Jedi to determine meta_text
             # (Jedi currently has a bug that results in incorrect information.)
             # meta_text = ''
             # yield Completion(m, start_position=start_pos,
             #                  display_meta=meta_text)
             display_text = c.text
 
             adjusted_text = _adjust_completion_text_based_on_context(c.text, body, offset)
             if c.type == 'function':
                 yield Completion(adjusted_text, start_position=c.start - offset, display=_elide(display_text+'()', body[c.start:c.end]), display_meta=c.type+c.signature)
             else:
                 yield Completion(adjusted_text, start_position=c.start - offset, display=_elide(display_text,  body[c.start:c.end]), display_meta=c.type)
 
 class IPythonPTLexer(Lexer):
     """
     Wrapper around PythonLexer and BashLexer.
     """
     def __init__(self):
         l = pygments_lexers
         self.python_lexer = PygmentsLexer(l.Python3Lexer)
         self.shell_lexer = PygmentsLexer(l.BashLexer)
 
         self.magic_lexers = {
             'HTML': PygmentsLexer(l.HtmlLexer),
             'html': PygmentsLexer(l.HtmlLexer),
             'javascript': PygmentsLexer(l.JavascriptLexer),
             'js': PygmentsLexer(l.JavascriptLexer),
             'perl': PygmentsLexer(l.PerlLexer),
             'ruby': PygmentsLexer(l.RubyLexer),
             'latex': PygmentsLexer(l.TexLexer),
         }
 
     def lex_document(self, document):
         text = document.text.lstrip()
 
         lexer = self.python_lexer
 
         if text.startswith('!') or text.startswith('%%bash'):
             lexer = self.shell_lexer
 
         elif text.startswith('%%'):
             for magic, l in self.magic_lexers.items():
                 if text.startswith('%%' + magic):
                     lexer = l
                     break
 
         return lexer.lex_document(document)

IPython/terminal/shortcuts/__init__.py

@@ -1,638 +1,638
 """
 Module to define and register Terminal IPython shortcuts with
 :mod:`prompt_toolkit`
 """
 
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 import os
 import signal
 import sys
 import warnings
 from dataclasses import dataclass
 from typing import Callable, Any, Optional, List
 
 from prompt_toolkit.application.current import get_app
 from prompt_toolkit.key_binding import KeyBindings
 from prompt_toolkit.key_binding.key_processor import KeyPressEvent
 from prompt_toolkit.key_binding.bindings import named_commands as nc
 from prompt_toolkit.key_binding.bindings.completion import (
     display_completions_like_readline,
 )
 from prompt_toolkit.key_binding.vi_state import InputMode, ViState
 from prompt_toolkit.filters import Condition
 
 from IPython.core.getipython import get_ipython
 from IPython.terminal.shortcuts import auto_match as match
 from IPython.terminal.shortcuts import auto_suggest
 from IPython.terminal.shortcuts.filters import filter_from_string
 from IPython.utils.decorators import undoc
 
 from prompt_toolkit.enums import DEFAULT_BUFFER
 
 __all__ = ["create_ipython_shortcuts"]
 
 
 @dataclass
 class BaseBinding:
     command: Callable[[KeyPressEvent], Any]
     keys: List[str]
 
 
 @dataclass
 class RuntimeBinding(BaseBinding):
     filter: Condition
 
 
 @dataclass
 class Binding(BaseBinding):
     # while filter could be created by referencing variables directly (rather
     # than created from strings), by using strings we ensure that users will
     # be able to create filters in configuration (e.g. JSON) files too, which
     # also benefits the documentation by enforcing human-readable filter names.
     condition: Optional[str] = None
 
     def __post_init__(self):
         if self.condition:
             self.filter = filter_from_string(self.condition)
         else:
             self.filter = None
 
 
 def create_identifier(handler: Callable):
     parts = handler.__module__.split(".")
     name = handler.__name__
     package = parts[0]
     if len(parts) > 1:
         final_module = parts[-1]
         return f"{package}:{final_module}.{name}"
     else:
         return f"{package}:{name}"
 
 
 AUTO_MATCH_BINDINGS = [
     *[
         Binding(
             cmd, [key], "focused_insert & auto_match & followed_by_closing_paren_or_end"
         )
         for key, cmd in match.auto_match_parens.items()
     ],
     *[
         # raw string
         Binding(cmd, [key], "focused_insert & auto_match & preceded_by_raw_str_prefix")
         for key, cmd in match.auto_match_parens_raw_string.items()
     ],
     Binding(
         match.double_quote,
         ['"'],
         "focused_insert"
         " & auto_match"
         " & not_inside_unclosed_string"
         " & preceded_by_paired_double_quotes"
         " & followed_by_closing_paren_or_end",
     ),
     Binding(
         match.single_quote,
         ["'"],
         "focused_insert"
         " & auto_match"
         " & not_inside_unclosed_string"
         " & preceded_by_paired_single_quotes"
         " & followed_by_closing_paren_or_end",
     ),
     Binding(
         match.docstring_double_quotes,
         ['"'],
         "focused_insert"
         " & auto_match"
         " & not_inside_unclosed_string"
         " & preceded_by_two_double_quotes",
     ),
     Binding(
         match.docstring_single_quotes,
         ["'"],
         "focused_insert"
         " & auto_match"
         " & not_inside_unclosed_string"
         " & preceded_by_two_single_quotes",
     ),
     Binding(
         match.skip_over,
         [")"],
         "focused_insert & auto_match & followed_by_closing_round_paren",
     ),
     Binding(
         match.skip_over,
         ["]"],
         "focused_insert & auto_match & followed_by_closing_bracket",
     ),
     Binding(
         match.skip_over,
         ["}"],
         "focused_insert & auto_match & followed_by_closing_brace",
     ),
     Binding(
         match.skip_over, ['"'], "focused_insert & auto_match & followed_by_double_quote"
     ),
     Binding(
         match.skip_over, ["'"], "focused_insert & auto_match & followed_by_single_quote"
     ),
     Binding(
         match.delete_pair,
         ["backspace"],
         "focused_insert"
         " & preceded_by_opening_round_paren"
         " & auto_match"
         " & followed_by_closing_round_paren",
     ),
     Binding(
         match.delete_pair,
         ["backspace"],
         "focused_insert"
         " & preceded_by_opening_bracket"
         " & auto_match"
         " & followed_by_closing_bracket",
     ),
     Binding(
         match.delete_pair,
         ["backspace"],
         "focused_insert"
         " & preceded_by_opening_brace"
         " & auto_match"
         " & followed_by_closing_brace",
     ),
     Binding(
         match.delete_pair,
         ["backspace"],
         "focused_insert"
         " & preceded_by_double_quote"
         " & auto_match"
         " & followed_by_double_quote",
     ),
     Binding(
         match.delete_pair,
         ["backspace"],
         "focused_insert"
         " & preceded_by_single_quote"
         " & auto_match"
         " & followed_by_single_quote",
     ),
 ]
 
 AUTO_SUGGEST_BINDINGS = [
     # there are two reasons for re-defining bindings defined upstream:
     # 1) prompt-toolkit does not execute autosuggestion bindings in vi mode,
     # 2) prompt-toolkit checks if we are at the end of text, not end of line
     #    hence it does not work in multi-line mode of navigable provider
     Binding(
         auto_suggest.accept_or_jump_to_end,
         ["end"],
         "has_suggestion & default_buffer_focused & emacs_like_insert_mode",
     ),
     Binding(
         auto_suggest.accept_or_jump_to_end,
         ["c-e"],
         "has_suggestion & default_buffer_focused & emacs_like_insert_mode",
     ),
     Binding(
         auto_suggest.accept,
         ["c-f"],
         "has_suggestion & default_buffer_focused & emacs_like_insert_mode",
     ),
     Binding(
         auto_suggest.accept,
         ["right"],
         "has_suggestion & default_buffer_focused & emacs_like_insert_mode",
     ),
     Binding(
         auto_suggest.accept_word,
         ["escape", "f"],
         "has_suggestion & default_buffer_focused & emacs_like_insert_mode",
     ),
     Binding(
         auto_suggest.accept_token,
         ["c-right"],
         "has_suggestion & default_buffer_focused & emacs_like_insert_mode",
     ),
     Binding(
         auto_suggest.discard,
         ["escape"],
         # note this one is using `emacs_insert_mode`, not `emacs_like_insert_mode`
         # as in `vi_insert_mode` we do not want `escape` to be shadowed (ever).
         "has_suggestion & default_buffer_focused & emacs_insert_mode",
     ),
     Binding(
         auto_suggest.discard,
         ["delete"],
         "has_suggestion & default_buffer_focused & emacs_insert_mode",
     ),
     Binding(
         auto_suggest.swap_autosuggestion_up,
         ["c-up"],
         "navigable_suggestions"
         " & ~has_line_above"
         " & has_suggestion"
         " & default_buffer_focused",
     ),
     Binding(
         auto_suggest.swap_autosuggestion_down,
         ["c-down"],
         "navigable_suggestions"
         " & ~has_line_below"
         " & has_suggestion"
         " & default_buffer_focused",
     ),
     Binding(
         auto_suggest.up_and_update_hint,
         ["c-up"],
         "has_line_above & navigable_suggestions & default_buffer_focused",
     ),
     Binding(
         auto_suggest.down_and_update_hint,
         ["c-down"],
         "has_line_below & navigable_suggestions & default_buffer_focused",
     ),
     Binding(
         auto_suggest.accept_character,
         ["escape", "right"],
         "has_suggestion & default_buffer_focused & emacs_like_insert_mode",
     ),
     Binding(
         auto_suggest.accept_and_move_cursor_left,
         ["c-left"],
         "has_suggestion & default_buffer_focused & emacs_like_insert_mode",
     ),
     Binding(
         auto_suggest.accept_and_keep_cursor,
         ["escape", "down"],
         "has_suggestion & default_buffer_focused & emacs_insert_mode",
     ),
     Binding(
         auto_suggest.backspace_and_resume_hint,
         ["backspace"],
         # no `has_suggestion` here to allow resuming if no suggestion
         "default_buffer_focused & emacs_like_insert_mode",
     ),
     Binding(
         auto_suggest.resume_hinting,
         ["right"],
         "is_cursor_at_the_end_of_line"
         " & default_buffer_focused"
         " & emacs_like_insert_mode"
         " & pass_through",
     ),
 ]
 
 
 SIMPLE_CONTROL_BINDINGS = [
     Binding(cmd, [key], "vi_insert_mode & default_buffer_focused & ebivim")
     for key, cmd in {
         "c-a": nc.beginning_of_line,
         "c-b": nc.backward_char,
         "c-k": nc.kill_line,
         "c-w": nc.backward_kill_word,
         "c-y": nc.yank,
         "c-_": nc.undo,
     }.items()
 ]
 
 
 ALT_AND_COMOBO_CONTROL_BINDINGS = [
     Binding(cmd, list(keys), "vi_insert_mode & default_buffer_focused & ebivim")
     for keys, cmd in {
         # Control Combos
         ("c-x", "c-e"): nc.edit_and_execute,
         ("c-x", "e"): nc.edit_and_execute,
         # Alt
         ("escape", "b"): nc.backward_word,
         ("escape", "c"): nc.capitalize_word,
         ("escape", "d"): nc.kill_word,
         ("escape", "h"): nc.backward_kill_word,
         ("escape", "l"): nc.downcase_word,
         ("escape", "u"): nc.uppercase_word,
         ("escape", "y"): nc.yank_pop,
         ("escape", "."): nc.yank_last_arg,
     }.items()
 ]
 
 
 def add_binding(bindings: KeyBindings, binding: Binding):
     bindings.add(
         *binding.keys,
         **({"filter": binding.filter} if binding.filter is not None else {}),
     )(binding.command)
 
 
 def create_ipython_shortcuts(shell, skip=None) -> KeyBindings:
     """Set up the prompt_toolkit keyboard shortcuts for IPython.
 
     Parameters
     ----------
     shell: InteractiveShell
         The current IPython shell Instance
     skip: List[Binding]
         Bindings to skip.
 
     Returns
     -------
     KeyBindings
         the keybinding instance for prompt toolkit.
 
     """
     kb = KeyBindings()
     skip = skip or []
     for binding in KEY_BINDINGS:
         skip_this_one = False
         for to_skip in skip:
             if (
                 to_skip.command == binding.command
                 and to_skip.filter == binding.filter
                 and to_skip.keys == binding.keys
             ):
                 skip_this_one = True
                 break
         if skip_this_one:
             continue
         add_binding(kb, binding)
 
     def get_input_mode(self):
         app = get_app()
         app.ttimeoutlen = shell.ttimeoutlen
         app.timeoutlen = shell.timeoutlen
 
         return self._input_mode
 
     def set_input_mode(self, mode):
         shape = {InputMode.NAVIGATION: 2, InputMode.REPLACE: 4}.get(mode, 6)
         cursor = "\x1b[{} q".format(shape)
 
         sys.stdout.write(cursor)
         sys.stdout.flush()
 
         self._input_mode = mode
 
     if shell.editing_mode == "vi" and shell.modal_cursor:
         ViState._input_mode = InputMode.INSERT  # type: ignore
         ViState.input_mode = property(get_input_mode, set_input_mode)  # type: ignore
 
     return kb
 
 
 def reformat_and_execute(event):
     """Reformat code and execute it"""
     shell = get_ipython()
     reformat_text_before_cursor(
         event.current_buffer, event.current_buffer.document, shell
     )
     event.current_buffer.validate_and_handle()
 
 
 def reformat_text_before_cursor(buffer, document, shell):
     text = buffer.delete_before_cursor(len(document.text[: document.cursor_position]))
     try:
         formatted_text = shell.reformat_handler(text)
         buffer.insert_text(formatted_text)
-    except Exception:
+    except Exception as e:
         buffer.insert_text(text)
 
 
 def handle_return_or_newline_or_execute(event):
     shell = get_ipython()
     if getattr(shell, "handle_return", None):
         return shell.handle_return(shell)(event)
     else:
         return newline_or_execute_outer(shell)(event)
 
 
 def newline_or_execute_outer(shell):
     def newline_or_execute(event):
         """When the user presses return, insert a newline or execute the code."""
         b = event.current_buffer
         d = b.document
 
         if b.complete_state:
             cc = b.complete_state.current_completion
             if cc:
                 b.apply_completion(cc)
             else:
                 b.cancel_completion()
             return
 
         # If there's only one line, treat it as if the cursor is at the end.
         # See https://github.com/ipython/ipython/issues/10425
         if d.line_count == 1:
             check_text = d.text
         else:
             check_text = d.text[: d.cursor_position]
         status, indent = shell.check_complete(check_text)
 
         # if all we have after the cursor is whitespace: reformat current text
         # before cursor
         after_cursor = d.text[d.cursor_position :]
         reformatted = False
         if not after_cursor.strip():
             reformat_text_before_cursor(b, d, shell)
             reformatted = True
         if not (
             d.on_last_line
             or d.cursor_position_row >= d.line_count - d.empty_line_count_at_the_end()
         ):
             if shell.autoindent:
                 b.insert_text("\n" + indent)
             else:
                 b.insert_text("\n")
             return
 
         if (status != "incomplete") and b.accept_handler:
             if not reformatted:
                 reformat_text_before_cursor(b, d, shell)
             b.validate_and_handle()
         else:
             if shell.autoindent:
                 b.insert_text("\n" + indent)
             else:
                 b.insert_text("\n")
 
     return newline_or_execute
 
 
 def previous_history_or_previous_completion(event):
     """
     Control-P in vi edit mode on readline is history next, unlike default prompt toolkit.
 
     If completer is open this still select previous completion.
     """
     event.current_buffer.auto_up()
 
 
 def next_history_or_next_completion(event):
     """
     Control-N in vi edit mode on readline is history previous, unlike default prompt toolkit.
 
     If completer is open this still select next completion.
     """
     event.current_buffer.auto_down()
 
 
 def dismiss_completion(event):
     """Dismiss completion"""
     b = event.current_buffer
     if b.complete_state:
         b.cancel_completion()
 
 
 def reset_buffer(event):
     """Reset buffer"""
     b = event.current_buffer
     if b.complete_state:
         b.cancel_completion()
     else:
         b.reset()
 
 
 def reset_search_buffer(event):
     """Reset search buffer"""
     if event.current_buffer.document.text:
         event.current_buffer.reset()
     else:
         event.app.layout.focus(DEFAULT_BUFFER)
 
 
 def suspend_to_bg(event):
     """Suspend to background"""
     event.app.suspend_to_background()
 
 
 def quit(event):
     """
     Quit application with ``SIGQUIT`` if supported or ``sys.exit`` otherwise.
 
     On platforms that support SIGQUIT, send SIGQUIT to the current process.
     On other platforms, just exit the process with a message.
     """
     sigquit = getattr(signal, "SIGQUIT", None)
     if sigquit is not None:
         os.kill(0, signal.SIGQUIT)
     else:
         sys.exit("Quit")
 
 
 def indent_buffer(event):
     """Indent buffer"""
     event.current_buffer.insert_text(" " * 4)
 
 
 def newline_autoindent(event):
     """Insert a newline after the cursor indented appropriately.
 
     Fancier version of former ``newline_with_copy_margin`` which should
     compute the correct indentation of the inserted line. That is to say, indent
     by 4 extra space after a function definition, class definition, context
     manager... And dedent by 4 space after ``pass``, ``return``, ``raise ...``.
     """
     shell = get_ipython()
     inputsplitter = shell.input_transformer_manager
     b = event.current_buffer
     d = b.document
 
     if b.complete_state:
         b.cancel_completion()
     text = d.text[: d.cursor_position] + "\n"
     _, indent = inputsplitter.check_complete(text)
     b.insert_text("\n" + (" " * (indent or 0)), move_cursor=False)
 
 
 def open_input_in_editor(event):
     """Open code from input in external editor"""
     event.app.current_buffer.open_in_editor()
 
 
 if sys.platform == "win32":
     from IPython.core.error import TryNext
     from IPython.lib.clipboard import (
         ClipboardEmpty,
         tkinter_clipboard_get,
         win32_clipboard_get,
     )
 
     @undoc
     def win_paste(event):
         try:
             text = win32_clipboard_get()
         except TryNext:
             try:
                 text = tkinter_clipboard_get()
             except (TryNext, ClipboardEmpty):
                 return
         except ClipboardEmpty:
             return
         event.current_buffer.insert_text(text.replace("\t", " " * 4))
 
 else:
 
     @undoc
     def win_paste(event):
         """Stub used on other platforms"""
         pass
 
 
 KEY_BINDINGS = [
     Binding(
         handle_return_or_newline_or_execute,
         ["enter"],
         "default_buffer_focused & ~has_selection & insert_mode",
     ),
     Binding(
         reformat_and_execute,
         ["escape", "enter"],
         "default_buffer_focused & ~has_selection & insert_mode & ebivim",
     ),
     Binding(quit, ["c-\\"]),
     Binding(
         previous_history_or_previous_completion,
         ["c-p"],
         "vi_insert_mode & default_buffer_focused",
     ),
     Binding(
         next_history_or_next_completion,
         ["c-n"],
         "vi_insert_mode & default_buffer_focused",
     ),
     Binding(dismiss_completion, ["c-g"], "default_buffer_focused & has_completions"),
     Binding(reset_buffer, ["c-c"], "default_buffer_focused"),
     Binding(reset_search_buffer, ["c-c"], "search_buffer_focused"),
     Binding(suspend_to_bg, ["c-z"], "supports_suspend"),
     Binding(
         indent_buffer,
         ["tab"],  # Ctrl+I == Tab
         "default_buffer_focused"
         " & ~has_selection"
         " & insert_mode"
         " & cursor_in_leading_ws",
     ),
     Binding(newline_autoindent, ["c-o"], "default_buffer_focused & emacs_insert_mode"),
     Binding(open_input_in_editor, ["f2"], "default_buffer_focused"),
     *AUTO_MATCH_BINDINGS,
     *AUTO_SUGGEST_BINDINGS,
     Binding(
         display_completions_like_readline,
         ["c-i"],
         "readline_like_completions"
         " & default_buffer_focused"
         " & ~has_selection"
         " & insert_mode"
         " & ~cursor_in_leading_ws",
     ),
     Binding(win_paste, ["c-v"], "default_buffer_focused & ~vi_mode & is_windows_os"),
     *SIMPLE_CONTROL_BINDINGS,
     *ALT_AND_COMOBO_CONTROL_BINDINGS,
 ]
 
 UNASSIGNED_ALLOWED_COMMANDS = [
     nc.beginning_of_buffer,
     nc.end_of_buffer,
     nc.end_of_line,
     nc.forward_word,
     nc.unix_line_discard,
 ]

IPython/testing/decorators.py

@@ -1,204 +1,204
 # -*- coding: utf-8 -*-
 """Decorators for labeling test objects.
 
 Decorators that merely return a modified version of the original function
 object are straightforward.  Decorators that return a new function object need
 to use nose.tools.make_decorator(original_function)(decorator) in returning the
 decorator, in order to preserve metadata such as function name, setup and
 teardown functions and so on - see nose.tools for more information.
 
 This module provides a set of useful decorators meant to be ready to use in
 your own tests.  See the bottom of the file for the ready-made ones, and if you
 find yourself writing a new one that may be of generic use, add it here.
 
 Included decorators:
 
 
 Lightweight testing that remains unittest-compatible.
 
 - An @as_unittest decorator can be used to tag any normal parameter-less
   function as a unittest TestCase.  Then, both nose and normal unittest will
   recognize it as such.  This will make it easier to migrate away from Nose if
   we ever need/want to while maintaining very lightweight tests.
 
 NOTE: This file contains IPython-specific decorators. Using the machinery in
 IPython.external.decorators, we import either numpy.testing.decorators if numpy is
 available, OR use equivalent code in IPython.external._decorators, which
 we've copied verbatim from numpy.
 
 """
 
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 import os
 import shutil
 import sys
 import tempfile
 import unittest
 from importlib import import_module
 
 from decorator import decorator
 
 # Expose the unittest-driven decorators
 from .ipunittest import ipdoctest, ipdocstring
 
 #-----------------------------------------------------------------------------
 # Classes and functions
 #-----------------------------------------------------------------------------
 
 # Simple example of the basic idea
 def as_unittest(func):
     """Decorator to make a simple function into a normal test via unittest."""
     class Tester(unittest.TestCase):
         def test(self):
             func()
 
     Tester.__name__ = func.__name__
 
     return Tester
 
 # Utility functions
 
 
 def skipif(skip_condition, msg=None):
     """Make function raise SkipTest exception if skip_condition is true
 
     Parameters
     ----------
 
     skip_condition : bool or callable
       Flag to determine whether to skip test. If the condition is a
       callable, it is used at runtime to dynamically make the decision. This
       is useful for tests that may require costly imports, to delay the cost
       until the test suite is actually executed.
     msg : string
       Message to give on raising a SkipTest exception.
 
     Returns
     -------
     decorator : function
       Decorator, which, when applied to a function, causes SkipTest
       to be raised when the skip_condition was True, and the function
       to be called normally otherwise.
     """
     if msg is None:
         msg = "Test skipped due to test condition."
 
     import pytest
 
     assert isinstance(skip_condition, bool)
     return pytest.mark.skipif(skip_condition, reason=msg)
 
 
 # A version with the condition set to true, common case just to attach a message
 # to a skip decorator
 def skip(msg=None):
     """Decorator factory - mark a test function for skipping from test suite.
 
     Parameters
     ----------
       msg : string
         Optional message to be added.
 
     Returns
     -------
        decorator : function
          Decorator, which, when applied to a function, causes SkipTest
          to be raised, with the optional message added.
       """
     if msg and not isinstance(msg, str):
         raise ValueError('invalid object passed to `@skip` decorator, did you '
                          'meant `@skip()` with brackets ?')
     return skipif(True, msg)
 
 
 def onlyif(condition, msg):
     """The reverse from skipif, see skipif for details."""
 
     return skipif(not condition, msg)
 
 #-----------------------------------------------------------------------------
 # Utility functions for decorators
-def module_available(module):
+def module_not_available(module):
     """Can module be imported?  Returns true if module does NOT import.
 
     This is used to make a decorator to skip tests that require module to be
     available, but delay the 'import numpy' to test execution time.
     """
     try:
-        import_module(module)
-        return True
+        mod = import_module(module)
+        mod_not_avail = False
     except ImportError:
-        return False
+        mod_not_avail = True
+
+    return mod_not_avail
 
 
 #-----------------------------------------------------------------------------
 # Decorators for public use
 
 # Decorators to skip certain tests on specific platforms.
 skip_win32 = skipif(sys.platform == 'win32',
                     "This test does not run under Windows")
 skip_linux = skipif(sys.platform.startswith('linux'),
                     "This test does not run under Linux")
 skip_osx = skipif(sys.platform == 'darwin',"This test does not run under OS X")
 
 
 # Decorators to skip tests if not on specific platforms.
 skip_if_not_win32 = skipif(sys.platform != "win32", "This test only runs under Windows")
 skip_if_not_linux = skipif(
     not sys.platform.startswith("linux"), "This test only runs under Linux"
 )
 skip_if_not_osx = skipif(
     not sys.platform.startswith("darwin"), "This test only runs under macOS"
 )
 
 _x11_skip_cond = (sys.platform not in ('darwin', 'win32') and
                   os.environ.get('DISPLAY', '') == '')
 _x11_skip_msg = "Skipped under *nix when X11/XOrg not available"
 
 skip_if_no_x11 = skipif(_x11_skip_cond, _x11_skip_msg)
 
 # Other skip decorators
 
 # generic skip without module
-skip_without = lambda mod: skipif(
-    not module_available(mod), "This test requires %s" % mod
-)
+skip_without = lambda mod: skipif(module_not_available(mod), "This test requires %s" % mod)
 
 skipif_not_numpy = skip_without('numpy')
 
 skipif_not_matplotlib = skip_without('matplotlib')
 
 # A null 'decorator', useful to make more readable code that needs to pick
 # between different decorators based on OS or other conditions
 null_deco = lambda f: f
 
 # Some tests only run where we can use unicode paths. Note that we can't just
 # check os.path.supports_unicode_filenames, which is always False on Linux.
 try:
     f = tempfile.NamedTemporaryFile(prefix=u"tmp€")
 except UnicodeEncodeError:
     unicode_paths = False
 else:
     unicode_paths = True
     f.close()
 
 onlyif_unicode_paths = onlyif(unicode_paths, ("This test is only applicable "
                                     "where we can use unicode in filenames."))
 
 
 def onlyif_cmds_exist(*commands):
     """
     Decorator to skip test when at least one of `commands` is not found.
     """
     assert (
         os.environ.get("IPTEST_WORKING_DIR", None) is None
     ), "iptest deprecated since IPython 8.0"
     for cmd in commands:
         reason = f"This test runs only if command '{cmd}' is installed"
         if not shutil.which(cmd):
             import pytest
 
             return pytest.mark.skip(reason=reason)
     return null_deco

IPython/tests/cve.py

@@ -1,66 +1,66
 """
 Test that CVEs stay fixed. 
 """
 
 from IPython.utils.tempdir import TemporaryDirectory, TemporaryWorkingDirectory
 from pathlib import Path
 import random
 import sys
 import os
 import string
 import subprocess
 
 
 def test_cve_2022_21699():
     """
     Here we test CVE-2022-21699.
 
     We create a temporary directory, cd into it.
     Make a profile file that should not be executed and start IPython in a subprocess,
     checking for the value.
 
 
 
     """
 
     dangerous_profile_dir = Path("profile_default")
 
     dangerous_startup_dir = dangerous_profile_dir / "startup"
     dangerous_expected = "CVE-2022-21699-" + "".join(
         [random.choice(string.ascii_letters) for i in range(10)]
     )
 
-    with TemporaryWorkingDirectory():
+    with TemporaryWorkingDirectory() as t:
         dangerous_startup_dir.mkdir(parents=True)
         (dangerous_startup_dir / "foo.py").write_text(
             f'print("{dangerous_expected}")', encoding="utf-8"
         )
         # 1 sec to make sure FS is flushed.
         # time.sleep(1)
         cmd = [sys.executable, "-m", "IPython"]
         env = os.environ.copy()
         env["IPY_TEST_SIMPLE_PROMPT"] = "1"
 
         # First we fake old behavior, making sure the profile is/was actually dangerous
         p_dangerous = subprocess.Popen(
             cmd + [f"--profile-dir={dangerous_profile_dir}"],
             env=env,
             stdin=subprocess.PIPE,
             stdout=subprocess.PIPE,
             stderr=subprocess.PIPE,
         )
         out_dangerous, err_dangerouns = p_dangerous.communicate(b"exit\r")
         assert dangerous_expected in out_dangerous.decode()
 
         # Now that we know it _would_ have been dangerous, we test it's not loaded
         p = subprocess.Popen(
             cmd,
             env=env,
             stdin=subprocess.PIPE,
             stdout=subprocess.PIPE,
             stderr=subprocess.PIPE,
         )
         out, err = p.communicate(b"exit\r")
         assert b"IPython" in out
         assert dangerous_expected not in out.decode()
         assert err == b""

IPython/utils/_process_cli.py

@@ -1,71 +1,71
 """cli-specific implementation of process utilities.
 
 cli - Common Language Infrastructure for IronPython. Code
       can run on any operating system. Check os.name for os-
       specific settings.
 
 This file is only meant to be imported by process.py, not by end-users.
 
 This file is largely untested. To become a full drop-in process
 interface for IronPython will probably require you to help fill
 in the details. 
 """
 
 # Import cli libraries:
 import clr
 import System
 
 # Import Python libraries:
 import os
 
 # Import IPython libraries:
 from ._process_common import arg_split
 
 
-def system(cmd: str):
+def system(cmd):
     """
     system(cmd) should work in a cli environment on Mac OSX, Linux,
     and Windows
     """
     psi = System.Diagnostics.ProcessStartInfo(cmd)
     psi.RedirectStandardOutput = True
     psi.RedirectStandardError = True
     psi.WindowStyle = System.Diagnostics.ProcessWindowStyle.Normal
     psi.UseShellExecute = False
     # Start up process:
-    _reg = System.Diagnostics.Process.Start(psi)
+    reg = System.Diagnostics.Process.Start(psi)
 
 
-def getoutput(cmd: str):
+def getoutput(cmd):
     """
     getoutput(cmd) should work in a cli environment on Mac OSX, Linux,
     and Windows
     """
     psi = System.Diagnostics.ProcessStartInfo(cmd)
     psi.RedirectStandardOutput = True
     psi.RedirectStandardError = True
     psi.WindowStyle = System.Diagnostics.ProcessWindowStyle.Normal
     psi.UseShellExecute = False
     # Start up process:
     reg = System.Diagnostics.Process.Start(psi)
     myOutput = reg.StandardOutput
     output = myOutput.ReadToEnd()
     myError = reg.StandardError
-    _error = myError.ReadToEnd()
+    error = myError.ReadToEnd()
     return output
 
 
-def check_pid(pid: int):
+def check_pid(pid):
     """
     Check if a process with the given PID (pid) exists
     """
     try:
         System.Diagnostics.Process.GetProcessById(pid)
         # process with given pid is running
         return True
     except System.InvalidOperationException:
         # process wasn't started by this object (but is running)
         return True
     except System.ArgumentException:
         # process with given pid isn't running
         return False

IPython/utils/io.py

@@ -1,130 +1,134
 # encoding: utf-8
 """
 IO related utilities.
 """
 
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 
 
+import atexit
+import os
 import sys
 import tempfile
 from pathlib import Path
+from warnings import warn
 
-from .capture import capture_output as capture_output
+from IPython.utils.decorators import undoc
+from .capture import CapturedIO, capture_output
 
 class Tee(object):
     """A class to duplicate an output stream to stdout/err.
 
     This works in a manner very similar to the Unix 'tee' command.
 
     When the object is closed or deleted, it closes the original file given to
     it for duplication.
     """
     # Inspired by:
     # http://mail.python.org/pipermail/python-list/2007-May/442737.html
 
     def __init__(self, file_or_name, mode="w", channel='stdout'):
         """Construct a new Tee object.
 
         Parameters
         ----------
         file_or_name : filename or open filehandle (writable)
             File that will be duplicated
         mode : optional, valid mode for open().
             If a filename was give, open with this mode.
         channel : str, one of ['stdout', 'stderr']
         """
         if channel not in ['stdout', 'stderr']:
             raise ValueError('Invalid channel spec %s' % channel)
 
         if hasattr(file_or_name, 'write') and hasattr(file_or_name, 'seek'):
             self.file = file_or_name
         else:
             encoding = None if "b" in mode else "utf-8"
             self.file = open(file_or_name, mode, encoding=encoding)
         self.channel = channel
         self.ostream = getattr(sys, channel)
         setattr(sys, channel, self)
         self._closed = False
 
     def close(self):
         """Close the file and restore the channel."""
         self.flush()
         setattr(sys, self.channel, self.ostream)
         self.file.close()
         self._closed = True
 
     def write(self, data):
         """Write data to both channels."""
         self.file.write(data)
         self.ostream.write(data)
         self.ostream.flush()
 
     def flush(self):
         """Flush both channels."""
         self.file.flush()
         self.ostream.flush()
 
     def __del__(self):
         if not self._closed:
             self.close()
 
     def isatty(self):
         return False
 
 def ask_yes_no(prompt, default=None, interrupt=None):
     """Asks a question and returns a boolean (y/n) answer.
 
     If default is given (one of 'y','n'), it is used if the user input is
     empty. If interrupt is given (one of 'y','n'), it is used if the user
     presses Ctrl-C. Otherwise the question is repeated until an answer is
     given.
 
     An EOF is treated as the default answer.  If there is no default, an
     exception is raised to prevent infinite loops.
 
     Valid answers are: y/yes/n/no (match is not case sensitive)."""
 
     answers = {'y':True,'n':False,'yes':True,'no':False}
     ans = None
     while ans not in answers.keys():
         try:
             ans = input(prompt+' ').lower()
             if not ans:  # response was an empty string
                 ans = default
         except KeyboardInterrupt:
             if interrupt:
                 ans = interrupt
             print("\r")
         except EOFError:
             if default in answers.keys():
                 ans = default
                 print()
             else:
                 raise
 
     return answers[ans]
 
 
 def temp_pyfile(src, ext='.py'):
     """Make a temporary python file, return filename and filehandle.
 
     Parameters
     ----------
     src : string or list of strings (no need for ending newlines if list)
         Source code to be written to the file.
     ext : optional, string
         Extension for the generated file.
 
     Returns
     -------
     (filename, open filehandle)
         It is the caller's responsibility to close the open file and unlink it.
     """
     fname = tempfile.mkstemp(ext)[1]
     with open(Path(fname), "w", encoding="utf-8") as f:
         f.write(src)
         f.flush()
     return fname

IPython/utils/tests/test_io.py

@@ -1,61 +1,61
 # encoding: utf-8
 """Tests for io.py"""
 
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 
 import sys
 from io import StringIO
 
 import unittest
 
 from IPython.utils.io import Tee, capture_output
 
 
 def test_tee_simple():
     "Very simple check with stdout only"
     chan = StringIO()
     text = 'Hello'
     tee = Tee(chan, channel='stdout')
     print(text, file=chan)
     assert chan.getvalue() == text + "\n"
     tee.close()
 
 
 class TeeTestCase(unittest.TestCase):
 
     def tchan(self, channel):
         trap = StringIO()
         chan = StringIO()
         text = 'Hello'
         
         std_ori = getattr(sys, channel)
         setattr(sys, channel, trap)
 
         tee = Tee(chan, channel=channel)
 
         print(text, end='', file=chan)
-        _trap_val = trap.getvalue()
+        trap_val = trap.getvalue()
         self.assertEqual(chan.getvalue(), text)
 
         tee.close()
 
         setattr(sys, channel, std_ori)
         assert getattr(sys, channel) == std_ori
 
     def test(self):
         for chan in ['stdout', 'stderr']:
             self.tchan(chan)
 
 class TestIOStream(unittest.TestCase):
 
     def test_capture_output(self):
         """capture_output() context works"""
         
         with capture_output() as io:
             print("hi, stdout")
             print("hi, stderr", file=sys.stderr)
 
         self.assertEqual(io.stdout, "hi, stdout\n")
         self.assertEqual(io.stderr, "hi, stderr\n")

IPython/utils/tests/test_path.py

@@ -1,506 +1,506
 # encoding: utf-8
 """Tests for IPython.utils.path.py"""
 
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 import os
 import shutil
 import sys
 import tempfile
 import unittest
 from contextlib import contextmanager
 from importlib import reload
 from os.path import abspath, join
 from unittest.mock import patch
 
 import pytest
 from tempfile import TemporaryDirectory
 
 import IPython
 from IPython import paths
 from IPython.testing import decorators as dec
 from IPython.testing.decorators import (
     onlyif_unicode_paths,
     skip_if_not_win32,
     skip_win32,
 )
 from IPython.testing.tools import make_tempfile
 from IPython.utils import path
 
 # Platform-dependent imports
 try:
     import winreg as wreg
 except ImportError:
     #Fake _winreg module on non-windows platforms
     import types
     wr_name = "winreg"
     sys.modules[wr_name] = types.ModuleType(wr_name)
     try:
         import winreg as wreg
     except ImportError:
         import _winreg as wreg
 
         #Add entries that needs to be stubbed by the testing code
         (wreg.OpenKey, wreg.QueryValueEx,) = (None, None)
 
 #-----------------------------------------------------------------------------
 # Globals
 #-----------------------------------------------------------------------------
 env = os.environ
 TMP_TEST_DIR = tempfile.mkdtemp()
 HOME_TEST_DIR = join(TMP_TEST_DIR, "home_test_dir")
 #
 # Setup/teardown functions/decorators
 #
 
 def setup_module():
     """Setup testenvironment for the module:
 
     - Adds dummy home dir tree
     """
     # Do not mask exceptions here.  In particular, catching WindowsError is a
     # problem because that exception is only defined on Windows...
     os.makedirs(os.path.join(HOME_TEST_DIR, 'ipython'))
 
 
 def teardown_module():
     """Teardown testenvironment for the module:
 
     - Remove dummy home dir tree
     """
     # Note: we remove the parent test dir, which is the root of all test
     # subdirs we may have created.  Use shutil instead of os.removedirs, so
     # that non-empty directories are all recursively removed.
     shutil.rmtree(TMP_TEST_DIR)
 
 
 # Build decorator that uses the setup_environment/setup_environment
 @pytest.fixture
 def environment():
     global oldstuff, platformstuff
     oldstuff = (
         env.copy(),
         os.name,
         sys.platform,
         path.get_home_dir,
         IPython.__file__,
         os.getcwd(),
     )
 
     yield
 
     (
         oldenv,
         os.name,
         sys.platform,
         path.get_home_dir,
         IPython.__file__,
         old_wd,
     ) = oldstuff
     os.chdir(old_wd)
     reload(path)
 
     for key in list(env):
         if key not in oldenv:
             del env[key]
     env.update(oldenv)
     assert not hasattr(sys, "frozen")
 
 
 with_environment = pytest.mark.usefixtures("environment")
 
 
 @skip_if_not_win32
 @with_environment
 def test_get_home_dir_1(monkeypatch):
     """Testcase for py2exe logic, un-compressed lib
     """
     unfrozen = path.get_home_dir()
     monkeypatch.setattr(sys, "frozen", True, raising=False)
 
     #fake filename for IPython.__init__
     IPython.__file__ = abspath(join(HOME_TEST_DIR, "Lib/IPython/__init__.py"))
 
     home_dir = path.get_home_dir()
     assert home_dir == unfrozen
 
 
 @skip_if_not_win32
 @with_environment
 def test_get_home_dir_2(monkeypatch):
     """Testcase for py2exe logic, compressed lib
     """
     unfrozen = path.get_home_dir()
     monkeypatch.setattr(sys, "frozen", True, raising=False)
     # fake filename for IPython.__init__
     IPython.__file__ = abspath(
         join(HOME_TEST_DIR, "Library.zip/IPython/__init__.py")
     ).lower()
 
     home_dir = path.get_home_dir(True)
     assert home_dir == unfrozen
 
 
 @skip_win32
 @with_environment
 def test_get_home_dir_3():
     """get_home_dir() uses $HOME if set"""
     env["HOME"] = HOME_TEST_DIR
     home_dir = path.get_home_dir(True)
     # get_home_dir expands symlinks
     assert home_dir == os.path.realpath(env["HOME"])
 
 
 @with_environment
 def test_get_home_dir_4():
     """get_home_dir() still works if $HOME is not set"""
 
     if 'HOME' in env: del env['HOME']
     # this should still succeed, but we don't care what the answer is
-    _home = path.get_home_dir(False)
+    home = path.get_home_dir(False)
 
 @skip_win32
 @with_environment
 def test_get_home_dir_5(monkeypatch):
     """raise HomeDirError if $HOME is specified, but not a writable dir"""
     env['HOME'] = abspath(HOME_TEST_DIR+'garbage')
     # set os.name = posix, to prevent My Documents fallback on Windows
     monkeypatch.setattr(os, "name", "posix")
     pytest.raises(path.HomeDirError, path.get_home_dir, True)
 
 # Should we stub wreg fully so we can run the test on all platforms?
 @skip_if_not_win32
 @with_environment
 def test_get_home_dir_8(monkeypatch):
     """Using registry hack for 'My Documents', os=='nt'
 
     HOMESHARE, HOMEDRIVE, HOMEPATH, USERPROFILE and others are missing.
     """
     monkeypatch.setattr(os, "name", "nt")
     # Remove from stub environment all keys that may be set
     for key in ['HOME', 'HOMESHARE', 'HOMEDRIVE', 'HOMEPATH', 'USERPROFILE']:
         env.pop(key, None)
 
     class key:
         def __enter__(self):
             pass
         def Close(self):
             pass
         def __exit__(*args, **kwargs):
             pass
 
     with patch.object(wreg, 'OpenKey', return_value=key()), \
          patch.object(wreg, 'QueryValueEx', return_value=[abspath(HOME_TEST_DIR)]):
         home_dir = path.get_home_dir()
     assert home_dir == abspath(HOME_TEST_DIR)
 
 @with_environment
 def test_get_xdg_dir_0(monkeypatch):
     """test_get_xdg_dir_0, check xdg_dir"""
     monkeypatch.setattr(path, "_writable_dir", lambda path: True)
     monkeypatch.setattr(path, "get_home_dir", lambda: "somewhere")
     monkeypatch.setattr(os, "name", "posix")
     monkeypatch.setattr(sys, "platform", "linux2")
     env.pop('IPYTHON_DIR', None)
     env.pop('IPYTHONDIR', None)
     env.pop('XDG_CONFIG_HOME', None)
 
     assert path.get_xdg_dir() == os.path.join("somewhere", ".config")
 
 
 @with_environment
 def test_get_xdg_dir_1(monkeypatch):
     """test_get_xdg_dir_1, check nonexistent xdg_dir"""
     monkeypatch.setattr(path, "get_home_dir", lambda: HOME_TEST_DIR)
     monkeypatch.setattr(os, "name", "posix")
     monkeypatch.setattr(sys, "platform", "linux2")
     env.pop("IPYTHON_DIR", None)
     env.pop("IPYTHONDIR", None)
     env.pop("XDG_CONFIG_HOME", None)
     assert path.get_xdg_dir() is None
 
 @with_environment
 def test_get_xdg_dir_2(monkeypatch):
     """test_get_xdg_dir_2, check xdg_dir default to ~/.config"""
     monkeypatch.setattr(path, "get_home_dir", lambda: HOME_TEST_DIR)
     monkeypatch.setattr(os, "name", "posix")
     monkeypatch.setattr(sys, "platform", "linux2")
     env.pop("IPYTHON_DIR", None)
     env.pop("IPYTHONDIR", None)
     env.pop("XDG_CONFIG_HOME", None)
     cfgdir = os.path.join(path.get_home_dir(), ".config")
     if not os.path.exists(cfgdir):
         os.makedirs(cfgdir)
 
     assert path.get_xdg_dir() == cfgdir
 
 @with_environment
 def test_get_xdg_dir_3(monkeypatch):
     """test_get_xdg_dir_3, check xdg_dir not used on non-posix systems"""
     monkeypatch.setattr(path, "get_home_dir", lambda: HOME_TEST_DIR)
     monkeypatch.setattr(os, "name", "nt")
     monkeypatch.setattr(sys, "platform", "win32")
     env.pop("IPYTHON_DIR", None)
     env.pop("IPYTHONDIR", None)
     env.pop("XDG_CONFIG_HOME", None)
     cfgdir = os.path.join(path.get_home_dir(), ".config")
     os.makedirs(cfgdir, exist_ok=True)
 
     assert path.get_xdg_dir() is None
 
 def test_filefind():
     """Various tests for filefind"""
     f = tempfile.NamedTemporaryFile()
     # print('fname:',f.name)
     alt_dirs = paths.get_ipython_dir()
-    _t = path.filefind(f.name, alt_dirs)
+    t = path.filefind(f.name, alt_dirs)
     # print('found:',t)
 
 
 @dec.skip_if_not_win32
 def test_get_long_path_name_win32():
     with TemporaryDirectory() as tmpdir:
 
         # Make a long path. Expands the path of tmpdir prematurely as it may already have a long
         # path component, so ensure we include the long form of it
         long_path = os.path.join(path.get_long_path_name(tmpdir), 'this is my long path name')
         os.makedirs(long_path)
 
         # Test to see if the short path evaluates correctly.
         short_path = os.path.join(tmpdir, 'THISIS~1')
         evaluated_path = path.get_long_path_name(short_path)
         assert evaluated_path.lower() == long_path.lower()
 
 
 @dec.skip_win32
 def test_get_long_path_name():
     p = path.get_long_path_name("/usr/local")
     assert p == "/usr/local"
 
 
 @dec.skip_win32  # can't create not-user-writable dir on win
 @with_environment
 def test_not_writable_ipdir():
     tmpdir = tempfile.mkdtemp()
     os.name = "posix"
     env.pop("IPYTHON_DIR", None)
     env.pop("IPYTHONDIR", None)
     env.pop("XDG_CONFIG_HOME", None)
     env["HOME"] = tmpdir
     ipdir = os.path.join(tmpdir, ".ipython")
     os.mkdir(ipdir, 0o555)
     try:
         open(os.path.join(ipdir, "_foo_"), "w", encoding="utf-8").close()
     except IOError:
         pass
     else:
         # I can still write to an unwritable dir,
         # assume I'm root and skip the test
         pytest.skip("I can't create directories that I can't write to")
 
     with pytest.warns(UserWarning, match="is not a writable location"):
         ipdir = paths.get_ipython_dir()
     env.pop("IPYTHON_DIR", None)
 
 
 @with_environment
 def test_get_py_filename():
     os.chdir(TMP_TEST_DIR)
     with make_tempfile("foo.py"):
         assert path.get_py_filename("foo.py") == "foo.py"
         assert path.get_py_filename("foo") == "foo.py"
     with make_tempfile("foo"):
         assert path.get_py_filename("foo") == "foo"
         pytest.raises(IOError, path.get_py_filename, "foo.py")
     pytest.raises(IOError, path.get_py_filename, "foo")
     pytest.raises(IOError, path.get_py_filename, "foo.py")
     true_fn = "foo with spaces.py"
     with make_tempfile(true_fn):
         assert path.get_py_filename("foo with spaces") == true_fn
         assert path.get_py_filename("foo with spaces.py") == true_fn
         pytest.raises(IOError, path.get_py_filename, '"foo with spaces.py"')
         pytest.raises(IOError, path.get_py_filename, "'foo with spaces.py'")
 
 @onlyif_unicode_paths
 def test_unicode_in_filename():
     """When a file doesn't exist, the exception raised should be safe to call
     str() on - i.e. in Python 2 it must only have ASCII characters.
 
     https://github.com/ipython/ipython/issues/875
     """
     try:
         # these calls should not throw unicode encode exceptions
         path.get_py_filename('fooéè.py')
     except IOError as ex:
         str(ex)
 
 
 class TestShellGlob(unittest.TestCase):
 
     @classmethod
     def setUpClass(cls):
         cls.filenames_start_with_a = ['a0', 'a1', 'a2']
         cls.filenames_end_with_b = ['0b', '1b', '2b']
         cls.filenames = cls.filenames_start_with_a + cls.filenames_end_with_b
         cls.tempdir = TemporaryDirectory()
         td = cls.tempdir.name
 
         with cls.in_tempdir():
             # Create empty files
             for fname in cls.filenames:
                 open(os.path.join(td, fname), "w", encoding="utf-8").close()
 
     @classmethod
     def tearDownClass(cls):
         cls.tempdir.cleanup()
 
     @classmethod
     @contextmanager
     def in_tempdir(cls):
         save = os.getcwd()
         try:
             os.chdir(cls.tempdir.name)
             yield
         finally:
             os.chdir(save)
 
     def check_match(self, patterns, matches):
         with self.in_tempdir():
             # glob returns unordered list. that's why sorted is required.
             assert sorted(path.shellglob(patterns)) == sorted(matches)
 
     def common_cases(self):
         return [
             (['*'], self.filenames),
             (['a*'], self.filenames_start_with_a),
             (['*c'], ['*c']),
             (['*', 'a*', '*b', '*c'], self.filenames
                                       + self.filenames_start_with_a
                                       + self.filenames_end_with_b
                                       + ['*c']),
             (['a[012]'], self.filenames_start_with_a),
         ]
 
     @skip_win32
     def test_match_posix(self):
         for (patterns, matches) in self.common_cases() + [
                 ([r'\*'], ['*']),
                 ([r'a\*', 'a*'], ['a*'] + self.filenames_start_with_a),
                 ([r'a\[012]'], ['a[012]']),
                 ]:
             self.check_match(patterns, matches)
 
     @skip_if_not_win32
     def test_match_windows(self):
         for (patterns, matches) in self.common_cases() + [
                 # In windows, backslash is interpreted as path
                 # separator.  Therefore, you can't escape glob
                 # using it.
                 ([r'a\*', 'a*'], [r'a\*'] + self.filenames_start_with_a),
                 ([r'a\[012]'], [r'a\[012]']),
                 ]:
             self.check_match(patterns, matches)
 
 
 @pytest.mark.parametrize(
     "globstr, unescaped_globstr",
     [
         (r"\*\[\!\]\?", "*[!]?"),
         (r"\\*", r"\*"),
         (r"\\\*", r"\*"),
         (r"\\a", r"\a"),
         (r"\a", r"\a"),
     ],
 )
 def test_unescape_glob(globstr, unescaped_globstr):
     assert path.unescape_glob(globstr) == unescaped_globstr
 
 
 @onlyif_unicode_paths
 def test_ensure_dir_exists():
     with TemporaryDirectory() as td:
         d = os.path.join(td, '∂ir')
         path.ensure_dir_exists(d) # create it
         assert os.path.isdir(d)
         path.ensure_dir_exists(d)  # no-op
         f = os.path.join(td, "ƒile")
         open(f, "w", encoding="utf-8").close()  # touch
         with pytest.raises(IOError):
             path.ensure_dir_exists(f)
 
 class TestLinkOrCopy(unittest.TestCase):
     def setUp(self):
         self.tempdir = TemporaryDirectory()
         self.src = self.dst("src")
         with open(self.src, "w", encoding="utf-8") as f:
             f.write("Hello, world!")
 
     def tearDown(self):
         self.tempdir.cleanup()
 
     def dst(self, *args):
         return os.path.join(self.tempdir.name, *args)
 
     def assert_inode_not_equal(self, a, b):
         assert (
             os.stat(a).st_ino != os.stat(b).st_ino
         ), "%r and %r do reference the same indoes" % (a, b)
 
     def assert_inode_equal(self, a, b):
         assert (
             os.stat(a).st_ino == os.stat(b).st_ino
         ), "%r and %r do not reference the same indoes" % (a, b)
 
     def assert_content_equal(self, a, b):
         with open(a, "rb") as a_f:
             with open(b, "rb") as b_f:
                 assert a_f.read() == b_f.read()
 
     @skip_win32
     def test_link_successful(self):
         dst = self.dst("target")
         path.link_or_copy(self.src, dst)
         self.assert_inode_equal(self.src, dst)
 
     @skip_win32
     def test_link_into_dir(self):
         dst = self.dst("some_dir")
         os.mkdir(dst)
         path.link_or_copy(self.src, dst)
         expected_dst = self.dst("some_dir", os.path.basename(self.src))
         self.assert_inode_equal(self.src, expected_dst)
 
     @skip_win32
     def test_target_exists(self):
         dst = self.dst("target")
         open(dst, "w", encoding="utf-8").close()
         path.link_or_copy(self.src, dst)
         self.assert_inode_equal(self.src, dst)
 
     @skip_win32
     def test_no_link(self):
         real_link = os.link
         try:
             del os.link
             dst = self.dst("target")
             path.link_or_copy(self.src, dst)
             self.assert_content_equal(self.src, dst)
             self.assert_inode_not_equal(self.src, dst)
         finally:
             os.link = real_link
 
     @skip_if_not_win32
     def test_windows(self):
         dst = self.dst("target")
         path.link_or_copy(self.src, dst)
         self.assert_content_equal(self.src, dst)
 
     def test_link_twice(self):
         # Linking the same file twice shouldn't leave duplicates around.
         # See https://github.com/ipython/ipython/issues/6450
         dst = self.dst('target')
         path.link_or_copy(self.src, dst)
         path.link_or_copy(self.src, dst)
         self.assert_inode_equal(self.src, dst)
         assert sorted(os.listdir(self.tempdir.name)) == ["src", "target"]

IPython/utils/tests/test_tokenutil.py

@@ -1,140 +1,141
 """Tests for tokenutil"""
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 import pytest
 
 from IPython.utils.tokenutil import token_at_cursor, line_at_cursor
 
 def expect_token(expected, cell, cursor_pos):
     token = token_at_cursor(cell, cursor_pos)
     offset = 0
     for line in cell.splitlines():
         if offset + len(line) >= cursor_pos:
             break
         else:
             offset += len(line)+1
     column = cursor_pos - offset
     line_with_cursor = "%s|%s" % (line[:column], line[column:])
     assert token == expected, "Expected %r, got %r in: %r (pos %i)" % (
         expected,
         token,
         line_with_cursor,
         cursor_pos,
     )
 
 
 def test_simple():
     cell = "foo"
     for i in range(len(cell)):
         expect_token("foo", cell, i)
 
 def test_function():
     cell = "foo(a=5, b='10')"
+    expected = 'foo'
     # up to `foo(|a=`
     for i in range(cell.find('a=') + 1):
         expect_token("foo", cell, i)
     # find foo after `=`
     for i in [cell.find('=') + 1, cell.rfind('=') + 1]:
         expect_token("foo", cell, i)
     # in between `5,|` and `|b=`
     for i in range(cell.find(','), cell.find('b=')):
         expect_token("foo", cell, i)
 
 def test_multiline():
     cell = '\n'.join([
         'a = 5',
         'b = hello("string", there)'
     ])
     expected = 'hello'
     start = cell.index(expected) + 1
     for i in range(start, start + len(expected)):
         expect_token(expected, cell, i)
     expected = 'hello'
     start = cell.index(expected) + 1
     for i in range(start, start + len(expected)):
         expect_token(expected, cell, i)
 
 def test_multiline_token():
     cell = '\n'.join([
         '"""\n\nxxxxxxxxxx\n\n"""',
         '5, """',
         'docstring',
         'multiline token',
         '""", [',
         '2, 3, "complicated"]',
         'b = hello("string", there)'
     ])
     expected = 'hello'
     start = cell.index(expected) + 1
     for i in range(start, start + len(expected)):
         expect_token(expected, cell, i)
     expected = 'hello'
     start = cell.index(expected) + 1
     for i in range(start, start + len(expected)):
         expect_token(expected, cell, i)
 
 def test_nested_call():
     cell = "foo(bar(a=5), b=10)"
     expected = 'foo'
     start = cell.index('bar') + 1
     for i in range(start, start + 3):
         expect_token(expected, cell, i)
     expected = 'bar'
     start = cell.index('a=')
     for i in range(start, start + 3):
         expect_token(expected, cell, i)
     expected = 'foo'
     start = cell.index(')') + 1
     for i in range(start, len(cell)-1):
         expect_token(expected, cell, i)
 
 def test_attrs():
     cell = "a = obj.attr.subattr"
     expected = 'obj'
     idx = cell.find('obj') + 1
     for i in range(idx, idx + 3):
         expect_token(expected, cell, i)
     idx = cell.find('.attr') + 2
     expected = 'obj.attr'
     for i in range(idx, idx + 4):
         expect_token(expected, cell, i)
     idx = cell.find('.subattr') + 2
     expected = 'obj.attr.subattr'
     for i in range(idx, len(cell)):
         expect_token(expected, cell, i)
 
 def test_line_at_cursor():
     cell = ""
     (line, offset) = line_at_cursor(cell, cursor_pos=11)
     assert line == ""
     assert offset == 0
 
     # The position after a newline should be the start of the following line.
     cell = "One\nTwo\n"
     (line, offset) = line_at_cursor(cell, cursor_pos=4)
     assert line == "Two\n"
     assert offset == 4
 
     # The end of a cell should be on the last line
     cell = "pri\npri"
     (line, offset) = line_at_cursor(cell, cursor_pos=7)
     assert line == "pri"
     assert offset == 4
 
 
 @pytest.mark.parametrize(
     "c, token",
     zip(
         list(range(16, 22)) + list(range(22, 28)),
         ["int"] * (22 - 16) + ["map"] * (28 - 22),
     ),
 )
 def test_multiline_statement(c, token):
     cell = """a = (1,
     3)
 
 int()
 map()
 """
     expect_token(token, cell, c)

docs/sphinxext/github.py

@@ -1,166 +1,168
 """Define text roles for GitHub
 
 * ghissue - Issue
 * ghpull - Pull Request
 * ghuser - User
 
 Adapted from bitbucket example here:
 https://bitbucket.org/birkenfeld/sphinx-contrib/src/tip/bitbucket/sphinxcontrib/bitbucket.py
 
 Authors
 -------
 
 * Doug Hellmann
 * Min RK
 """
 #
 # Original Copyright (c) 2010 Doug Hellmann.  All rights reserved.
 #
 
 from docutils import nodes, utils
 from docutils.parsers.rst.roles import set_classes
 from sphinx.util.logging import getLogger
 
 info = getLogger(__name__).info
 
 def make_link_node(rawtext, app, type, slug, options):
     """Create a link to a github resource.
 
     :param rawtext: Text being replaced with link node.
     :param app: Sphinx application context
     :param type: Link type (issues, changeset, etc.)
     :param slug: ID of the thing to link to
     :param options: Options dictionary passed to role func.
     """
 
     try:
         base = app.config.github_project_url
         if not base:
             raise AttributeError
         if not base.endswith('/'):
             base += '/'
     except AttributeError as err:
         raise ValueError('github_project_url configuration value is not set (%s)' % str(err)) from err
 
     ref = base + type + '/' + slug + '/'
     set_classes(options)
     prefix = "#"
     if type == 'pull':
         prefix = "PR " + prefix
     node = nodes.reference(rawtext, prefix + utils.unescape(slug), refuri=ref,
                            **options)
     return node
 
 def ghissue_role(name, rawtext, text, lineno, inliner, options=None, content=None):
     """Link to a GitHub issue.
 
     Returns 2 part tuple containing list of nodes to insert into the
     document and a list of system messages.  Both are allowed to be
     empty.
 
     :param name: The role name used in the document.
     :param rawtext: The entire markup snippet, with role.
     :param text: The text marked with the role.
     :param lineno: The line number where rawtext appears in the input.
     :param inliner: The inliner instance that called us.
     :param options: Directive options for customization.
     :param content: The directive content for customization.
     """
     if options is None:
         options = {}
 
     try:
         issue_num = int(text)
         if issue_num <= 0:
             raise ValueError
     except ValueError:
         msg = inliner.reporter.error(
             'GitHub issue number must be a number greater than or equal to 1; '
             '"%s" is invalid.' % text, line=lineno)
         prb = inliner.problematic(rawtext, rawtext, msg)
         return [prb], [msg]
+    app = inliner.document.settings.env.app
     #info('issue %r' % text)
     if 'pull' in name.lower():
         category = 'pull'
     elif 'issue' in name.lower():
         category = 'issues'
     else:
         msg = inliner.reporter.error(
             'GitHub roles include "ghpull" and "ghissue", '
             '"%s" is invalid.' % name, line=lineno)
         prb = inliner.problematic(rawtext, rawtext, msg)
         return [prb], [msg]
-    app = inliner.document.settings.env.app
     node = make_link_node(rawtext, app, category, str(issue_num), options)
     return [node], []
 
 def ghuser_role(name, rawtext, text, lineno, inliner, options=None, content=None):
     """Link to a GitHub user.
 
     Returns 2 part tuple containing list of nodes to insert into the
     document and a list of system messages.  Both are allowed to be
     empty.
 
     :param name: The role name used in the document.
     :param rawtext: The entire markup snippet, with role.
     :param text: The text marked with the role.
     :param lineno: The line number where rawtext appears in the input.
     :param inliner: The inliner instance that called us.
     :param options: Directive options for customization.
     :param content: The directive content for customization.
     """
     if options is None:
         options = {}
 
+    app = inliner.document.settings.env.app
     #info('user link %r' % text)
     ref = 'https://www.github.com/' + text
     node = nodes.reference(rawtext, text, refuri=ref, **options)
     return [node], []
 
 def ghcommit_role(name, rawtext, text, lineno, inliner, options=None, content=None):
     """Link to a GitHub commit.
 
     Returns 2 part tuple containing list of nodes to insert into the
     document and a list of system messages.  Both are allowed to be
     empty.
 
     :param name: The role name used in the document.
     :param rawtext: The entire markup snippet, with role.
     :param text: The text marked with the role.
     :param lineno: The line number where rawtext appears in the input.
     :param inliner: The inliner instance that called us.
     :param options: Directive options for customization.
     :param content: The directive content for customization.
     """
     if options is None:
         options = {}
 
+    app = inliner.document.settings.env.app
     #info('user link %r' % text)
     try:
         base = app.config.github_project_url
         if not base:
             raise AttributeError
         if not base.endswith('/'):
             base += '/'
     except AttributeError as err:
         raise ValueError('github_project_url configuration value is not set (%s)' % str(err)) from err
 
     ref = base + text
     node = nodes.reference(rawtext, text[:6], refuri=ref, **options)
     return [node], []
 
 
 def setup(app):
     """Install the plugin.
 
     :param app: Sphinx application context.
     """
     info('Initializing GitHub plugin')
     app.add_role('ghissue', ghissue_role)
     app.add_role('ghpull', ghissue_role)
     app.add_role('ghuser', ghuser_role)
     app.add_role('ghcommit', ghcommit_role)
     app.add_config_value('github_project_url', None, 'env')
 
     metadata = {'parallel_read_safe': True, 'parallel_write_safe': True}
     return metadata

pyproject.toml

@@ -1,392 +1,385
 [build-system]
 requires = ["setuptools>=61.2"]
 # We need access to the 'setupbase' module at build time.
 # Hence we declare a custom build backend.
-build-backend = "_build_meta" # just re-exports setuptools.build_meta definitions
+build-backend = "_build_meta"  # just re-exports setuptools.build_meta definitions
 backend-path = ["."]
 
 [project]
 name = "ipython"
 description = "IPython: Productive Interactive Computing"
 keywords = ["Interactive", "Interpreter", "Shell", "Embedding"]
 classifiers = [
     "Framework :: IPython",
     "Framework :: Jupyter",
     "Intended Audience :: Developers",
     "Intended Audience :: Science/Research",
     "License :: OSI Approved :: BSD License",
     "Programming Language :: Python",
     "Programming Language :: Python :: 3",
     "Programming Language :: Python :: 3 :: Only",
     "Topic :: System :: Shells",
 ]
 requires-python = ">=3.11"
 dependencies = [
     'colorama; sys_platform == "win32"',
     "decorator",
     "jedi>=0.16",
     "matplotlib-inline",
     'pexpect>4.3; sys_platform != "win32" and sys_platform != "emscripten"',
     "prompt_toolkit>=3.0.41,<3.1.0",
     "pygments>=2.4.0",
     "stack_data",
     "traitlets>=5.13.0",
     "typing_extensions>=4.6; python_version<'3.12'",
 ]
 dynamic = ["authors", "license", "version"]
 
 [project.entry-points."pygments.lexers"]
 ipythonconsole = "IPython.lib.lexers:IPythonConsoleLexer"
 ipython = "IPython.lib.lexers:IPythonLexer"
 ipython3 = "IPython.lib.lexers:IPython3Lexer"
 
 [project.scripts]
 ipython = "IPython:start_ipython"
 ipython3 = "IPython:start_ipython"
 
 [project.readme]
 file = "long_description.rst"
 content-type = "text/x-rst"
 
 [project.urls]
 Homepage = "https://ipython.org"
 Documentation = "https://ipython.readthedocs.io/"
 Funding = "https://numfocus.org/"
 Source = "https://github.com/ipython/ipython"
 Tracker = "https://github.com/ipython/ipython/issues"
 
 [project.optional-dependencies]
-black = ["black"]
+black = [
+    "black",
+]
 doc = [
     "docrepr",
     "exceptiongroup",
     "intersphinx_registry",
     "ipykernel",
     "ipython[test]",
     "matplotlib",
     "setuptools>=18.5",
     "sphinx-rtd-theme",
     "sphinx>=1.3",
     "sphinxcontrib-jquery",
 ]
-kernel = ["ipykernel"]
-nbconvert = ["nbconvert"]
-nbformat = ["nbformat"]
-notebook = ["ipywidgets", "notebook"]
-parallel = ["ipyparallel"]
-qtconsole = ["qtconsole"]
+kernel = [
+    "ipykernel",
+]
+nbconvert = [
+    "nbconvert",
+]
+nbformat = [
+    "nbformat",
+]
+notebook = [
+    "ipywidgets",
+    "notebook",
+]
+parallel = [
+    "ipyparallel",
+]
+qtconsole = [
+    "qtconsole",
+]
 terminal = []
-test = ["pytest", "pytest-asyncio<0.22", "testpath", "pickleshare", "packaging"]
+test = [
+    "pytest",
+    "pytest-asyncio<0.22",
+    "testpath",
+    "pickleshare",
+    "packaging",
+]
 test_extra = [
     "ipython[test]",
     "curio",
     "matplotlib!=3.2.0",
     "nbformat",
     "numpy>=1.23",
     "pandas",
     "trio",
 ]
-matplotlib = ["matplotlib"]
+matplotlib = [
+   "matplotlib"
+]
 all = [
     "ipython[black,doc,kernel,nbconvert,nbformat,notebook,parallel,qtconsole,matplotlib]",
     "ipython[test,test_extra]",
 ]
 
 [tool.mypy]
 python_version = "3.10"
 ignore_missing_imports = true
 follow_imports = 'silent'
 exclude = [
-    'test_\.+\.py',
-    'IPython.utils.tests.test_wildcard',
-    'testing',
-    'tests',
-    'PyColorize.py',
-    '_process_win32_controller.py',
-    'IPython/core/application.py',
-    'IPython/core/profileapp.py',
-    'IPython/lib/deepreload.py',
-    'IPython/sphinxext/ipython_directive.py',
-    'IPython/terminal/ipapp.py',
-    'IPython/utils/path.py',
+   'test_\.+\.py',
+   'IPython.utils.tests.test_wildcard',
+   'testing',
+   'tests',
+   'PyColorize.py',
+   '_process_win32_controller.py',
+   'IPython/core/application.py',
+   'IPython/core/profileapp.py',
+   'IPython/lib/deepreload.py',
+   'IPython/sphinxext/ipython_directive.py',
+   'IPython/terminal/ipapp.py',
+   'IPython/utils/path.py',
 ]
 # check_untyped_defs = true
 # disallow_untyped_calls = true
 # disallow_untyped_decorators = true
 # ignore_errors = false
 # ignore_missing_imports = false
 disallow_incomplete_defs = true
 disallow_untyped_defs = true
 warn_redundant_casts = true
 
 [[tool.mypy.overrides]]
-module = ["IPython.core.crashhandler"]
+module = [
+    "IPython.core.crashhandler",
+]
 check_untyped_defs = true
 disallow_incomplete_defs = true
 disallow_untyped_calls = true
 disallow_untyped_decorators = true
 disallow_untyped_defs = true
 ignore_errors = false
 ignore_missing_imports = false
 
 [[tool.mypy.overrides]]
-module = ["IPython.utils.text"]
+module = [
+    "IPython.utils.text",
+]
 disallow_untyped_defs = true
 check_untyped_defs = false
 disallow_untyped_decorators = true
 
 [[tool.mypy.overrides]]
-module = []
+module = [
+]
 disallow_untyped_defs = false
 ignore_errors = true
 ignore_missing_imports = true
 disallow_untyped_calls = false
 disallow_incomplete_defs = false
 check_untyped_defs = false
 disallow_untyped_decorators = false
 
 
 # gloabl ignore error
 [[tool.mypy.overrides]]
 module = [
     "IPython",
     "IPython.conftest",
     "IPython.core.alias",
     "IPython.core.async_helpers",
     "IPython.core.autocall",
     "IPython.core.builtin_trap",
     "IPython.core.compilerop",
     "IPython.core.completer",
     "IPython.core.completerlib",
     "IPython.core.debugger",
     "IPython.core.display",
     "IPython.core.display_functions",
     "IPython.core.display_trap",
     "IPython.core.displayhook",
     "IPython.core.displaypub",
     "IPython.core.events",
     "IPython.core.excolors",
     "IPython.core.extensions",
     "IPython.core.formatters",
     "IPython.core.getipython",
     "IPython.core.guarded_eval",
     "IPython.core.historyapp",
     "IPython.core.hooks",
     "IPython.core.inputtransformer",
     "IPython.core.inputtransformer2",
     "IPython.core.interactiveshell",
     "IPython.core.logger",
     "IPython.core.macro",
     "IPython.core.magic",
     "IPython.core.magic_arguments",
     "IPython.core.magics.ast_mod",
     "IPython.core.magics.auto",
     "IPython.core.magics.basic",
     "IPython.core.magics.code",
     "IPython.core.magics.config",
     "IPython.core.magics.display",
     "IPython.core.magics.execution",
     "IPython.core.magics.extension",
     "IPython.core.magics.history",
     "IPython.core.magics.logging",
     "IPython.core.magics.namespace",
     "IPython.core.magics.osm",
     "IPython.core.magics.packaging",
     "IPython.core.magics.pylab",
     "IPython.core.magics.script",
     "IPython.core.oinspect",
     "IPython.core.page",
     "IPython.core.payload",
     "IPython.core.payloadpage",
     "IPython.core.prefilter",
     "IPython.core.profiledir",
     "IPython.core.prompts",
     "IPython.core.pylabtools",
     "IPython.core.shellapp",
     "IPython.core.splitinput",
     "IPython.core.ultratb",
     "IPython.extensions.autoreload",
     "IPython.extensions.storemagic",
     "IPython.external.qt_for_kernel",
     "IPython.external.qt_loaders",
     "IPython.lib.backgroundjobs",
     "IPython.lib.clipboard",
     "IPython.lib.demo",
     "IPython.lib.display",
     "IPython.lib.editorhooks",
     "IPython.lib.guisupport",
     "IPython.lib.latextools",
     "IPython.lib.lexers",
     "IPython.lib.pretty",
     "IPython.paths",
     "IPython.sphinxext.ipython_console_highlighting",
     "IPython.terminal.debugger",
     "IPython.terminal.embed",
     "IPython.terminal.interactiveshell",
     "IPython.terminal.magics",
     "IPython.terminal.prompts",
     "IPython.terminal.pt_inputhooks",
     "IPython.terminal.pt_inputhooks.asyncio",
     "IPython.terminal.pt_inputhooks.glut",
     "IPython.terminal.pt_inputhooks.gtk",
     "IPython.terminal.pt_inputhooks.gtk3",
     "IPython.terminal.pt_inputhooks.gtk4",
     "IPython.terminal.pt_inputhooks.osx",
     "IPython.terminal.pt_inputhooks.pyglet",
     "IPython.terminal.pt_inputhooks.qt",
     "IPython.terminal.pt_inputhooks.tk",
     "IPython.terminal.pt_inputhooks.wx",
     "IPython.terminal.ptutils",
     "IPython.terminal.shortcuts",
     "IPython.terminal.shortcuts.auto_match",
     "IPython.terminal.shortcuts.auto_suggest",
     "IPython.terminal.shortcuts.filters",
     "IPython.utils._process_cli",
     "IPython.utils._process_common",
     "IPython.utils._process_emscripten",
     "IPython.utils._process_posix",
     "IPython.utils.capture",
     "IPython.utils.coloransi",
     "IPython.utils.contexts",
     "IPython.utils.data",
     "IPython.utils.decorators",
     "IPython.utils.dir2",
     "IPython.utils.encoding",
     "IPython.utils.frame",
     "IPython.utils.generics",
     "IPython.utils.importstring",
     "IPython.utils.io",
     "IPython.utils.ipstruct",
     "IPython.utils.module_paths",
     "IPython.utils.openpy",
     "IPython.utils.process",
     "IPython.utils.py3compat",
     "IPython.utils.sentinel",
     "IPython.utils.strdispatch",
     "IPython.utils.sysinfo",
     "IPython.utils.syspathcontext",
     "IPython.utils.tempdir",
     "IPython.utils.terminal",
     "IPython.utils.timing",
     "IPython.utils.tokenutil",
     "IPython.utils.version",
     "IPython.utils.wildcard",
 
 ]
 disallow_untyped_defs = false
 ignore_errors = true
 ignore_missing_imports = true
 disallow_untyped_calls = false
 disallow_incomplete_defs = false
 check_untyped_defs = false
 disallow_untyped_decorators = false
 
 [tool.pytest.ini_options]
 addopts = [
-    "--durations=10",
-    "-pIPython.testing.plugin.pytest_ipdoctest",
-    "--ipdoctest-modules",
-    "--ignore=docs",
-    "--ignore=examples",
-    "--ignore=htmlcov",
-    "--ignore=ipython_kernel",
-    "--ignore=ipython_parallel",
-    "--ignore=results",
-    "--ignore=tmp",
-    "--ignore=tools",
-    "--ignore=traitlets",
-    "--ignore=IPython/core/tests/daft_extension",
-    "--ignore=IPython/sphinxext",
-    "--ignore=IPython/terminal/pt_inputhooks",
-    "--ignore=IPython/__main__.py",
-    "--ignore=IPython/external/qt_for_kernel.py",
-    "--ignore=IPython/html/widgets/widget_link.py",
-    "--ignore=IPython/html/widgets/widget_output.py",
-    "--ignore=IPython/terminal/console.py",
-    "--ignore=IPython/utils/_process_cli.py",
-    "--ignore=IPython/utils/_process_posix.py",
-    "--ignore=IPython/utils/_process_win32_controller.py",
-    "--ignore=IPython/utils/daemonize.py",
-    "--ignore=IPython/utils/eventful.py",
-    "--ignore=IPython/kernel",
-    "--ignore=IPython/consoleapp.py",
-    "--ignore=IPython/lib/kernel.py",
-    "--ignore=IPython/utils/jsonutil.py",
-    "--ignore=IPython/utils/localinterfaces.py",
-    "--ignore=IPython/utils/log.py",
-    "--ignore=IPython/utils/signatures.py",
-    "--ignore=IPython/utils/version.py",
+   "--durations=10",
+   "-pIPython.testing.plugin.pytest_ipdoctest",
+   "--ipdoctest-modules",
+   "--ignore=docs",
+   "--ignore=examples",
+   "--ignore=htmlcov",
+   "--ignore=ipython_kernel",
+   "--ignore=ipython_parallel",
+   "--ignore=results",
+   "--ignore=tmp",
+   "--ignore=tools",
+   "--ignore=traitlets",
+   "--ignore=IPython/core/tests/daft_extension",
+   "--ignore=IPython/sphinxext",
+   "--ignore=IPython/terminal/pt_inputhooks",
+   "--ignore=IPython/__main__.py",
+   "--ignore=IPython/external/qt_for_kernel.py",
+   "--ignore=IPython/html/widgets/widget_link.py",
+   "--ignore=IPython/html/widgets/widget_output.py",
+   "--ignore=IPython/terminal/console.py",
+   "--ignore=IPython/utils/_process_cli.py",
+   "--ignore=IPython/utils/_process_posix.py",
+   "--ignore=IPython/utils/_process_win32_controller.py",
+   "--ignore=IPython/utils/daemonize.py",
+   "--ignore=IPython/utils/eventful.py",
+   "--ignore=IPython/kernel",
+   "--ignore=IPython/consoleapp.py",
+   "--ignore=IPython/lib/kernel.py",
+   "--ignore=IPython/utils/jsonutil.py",
+   "--ignore=IPython/utils/localinterfaces.py",
+   "--ignore=IPython/utils/log.py",
+   "--ignore=IPython/utils/signatures.py",
+   "--ignore=IPython/utils/version.py"
+]
+doctest_optionflags = [
+   "NORMALIZE_WHITESPACE",
+   "ELLIPSIS"
+]
+ipdoctest_optionflags = [
+   "NORMALIZE_WHITESPACE",
+   "ELLIPSIS"
 ]
-doctest_optionflags = ["NORMALIZE_WHITESPACE", "ELLIPSIS"]
-ipdoctest_optionflags = ["NORMALIZE_WHITESPACE", "ELLIPSIS"]
 asyncio_mode = "strict"
 
 [tool.pyright]
-pythonPlatform = "All"
+pythonPlatform="All"
 
 [tool.setuptools]
 zip-safe = false
 platforms = ["Linux", "Mac OSX", "Windows"]
 license-files = ["LICENSE"]
 include-package-data = false
 
 [tool.setuptools.packages.find]
 exclude = ["setupext"]
 namespaces = false
 
 [tool.setuptools.package-data]
 "IPython" = ["py.typed"]
 "IPython.core" = ["profile/README*"]
 "IPython.core.tests" = ["*.png", "*.jpg", "daft_extension/*.py"]
 "IPython.lib.tests" = ["*.wav"]
 "IPython.testing.plugin" = ["*.txt"]
 
 [tool.setuptools.dynamic]
-version = { attr = "IPython.core.release.__version__" }
+version = {attr = "IPython.core.release.__version__"}
 
 [tool.coverage.run]
 omit = [
     # omit everything in /tmp as we run tempfile
     "/tmp/*",
-]
-
-[tool.ruff.lint]
-extend-select = [
-    #  "B",           # flake8-bugbear
-    #  "I",           # isort
-    # that will be a problem for pytest fixture unless you swap with the usefixture decorator https://docs.pytest.org/en/7.1.x/how-to/fixtures.html#use-fixtures-in-classes-and-modules-with-usefixtures
-    #  "ARG",         # flake8-unused-arguments
-    #  "C4",          # flake8-comprehensions
-    #  "EM",          # flake8-errmsg
-    #  "ICN",         # flake8-import-conventions
-    #  "G",           # flake8-logging-format
-    #  "PGH",         # pygrep-hooks
-    #  "PIE",         # flake8-pie
-    #  "PL",          # pylint
-    #  "PTH",         # flake8-use-pathlib
-    #  "PT",          # flake8-pytest-style
-    #  "RET",         # flake8-return
-    #  "RUF",         # Ruff-specific
-    #  "SIM",         # flake8-simplify
-    #  "T20",         # flake8-print
-    #  "UP",          # pyupgrade
-    #  "YTT",         # flake8-2020
-    #  "EXE",         # flake8-executable
-    #  "PYI",         # flake8-pyi
-    #  "S",           # flake8-bandit
-]
-ignore = [
-    #  "E501",     # E501 Line too long (158 > 100 characters)
-    #  "SIM105",   # SIM105 Use `contextlib.suppress(...)`
-    #  "PLR",      # Design related pylint codes
-    #  "S101",     # Use of `assert` detected
-]
-unfixable = [
-    # Don't touch print statements
-    "T201",
-    # Don't touch noqa lines
-    "RUF100",
-]
-
-[tool.ruff]
-extend-exclude = ["tests"]
+    ]