Improve API documentation around configuration of embedded IPython (#13989)

This PR tries to improve the API documentation of `embed()`,
`embed_kernel()`, `start_ipython()` and `start_kernel()` with respect to
the configuration of IPython by adding links to other sections of the
documentation. It also reorganizes the API docs so that `embed()` is
listed under `IPython`, just as it's presented in the narrative
documentation.

.gitignore

@@ -1,39 +1,40 b''
 MANIFEST
 build
 dist
 _build
 docs/man/*.gz
 docs/source/api/generated
 docs/source/config/options
 docs/source/config/shortcuts/*.csv
+docs/source/config/shortcuts/table.tsv
 docs/source/savefig
 docs/source/interactive/magics-generated.txt
 docs/gh-pages
 jupyter_notebook/notebook/static/mathjax
 jupyter_notebook/static/style/*.map
 *.py[co]
 __pycache__
 *.egg-info
 *~
 *.bak
 .ipynb_checkpoints
 .tox
 .DS_Store
 \#*#
 .#*
 .cache
 .coverage
 *.swp
 .pytest_cache
 .python-version
 venv*/
 .mypy_cache/
 
 # jetbrains ide stuff
 *.iml
 .idea/
 
 # vscode ide stuff
 *.code-workspace
 .history
 .vscode

IPython/__init__.py

@@ -1,156 +1,161 b''
 # PYTHON_ARGCOMPLETE_OK
 """
 IPython: tools for interactive and parallel computing in Python.
 
 https://ipython.org
 """
 #-----------------------------------------------------------------------------
 #  Copyright (c) 2008-2011, IPython Development Team.
 #  Copyright (c) 2001-2007, Fernando Perez <[e-mail]>
 #  Copyright (c) 2001, Janko Hauser <[e-mail]>
 #  Copyright (c) 2001, Nathaniel Gray <[e-mail]>
 #
 #  Distributed under the terms of the Modified BSD License.
 #
 #  The full license is in the file COPYING.txt, distributed with this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import sys
 
 #-----------------------------------------------------------------------------
 # Setup everything
 #-----------------------------------------------------------------------------
 
 # Don't forget to also update setup.py when this changes!
 if sys.version_info < (3, 8):
     raise ImportError(
         """
 IPython 8+ supports Python 3.8 and above, following NEP 29.
 When using Python 2.7, please install IPython 5.x LTS Long Term Support version.
 Python 3.3 and 3.4 were supported up to IPython 6.x.
 Python 3.5 was supported with IPython 7.0 to 7.9.
 Python 3.6 was supported with IPython up to 7.16.
 Python 3.7 was still supported with the 7.x branch.
 
 See IPython `README.rst` file for more information:
 
     https://github.com/ipython/ipython/blob/main/README.rst
 
 """
     )
 
 #-----------------------------------------------------------------------------
 # Setup the top level names
 #-----------------------------------------------------------------------------
 
 from .core.getipython import get_ipython
 from .core import release
 from .core.application import Application
 from .terminal.embed import embed
 
 from .core.interactiveshell import InteractiveShell
 from .utils.sysinfo import sys_info
 from .utils.frame import extract_module_locals
 
+__all__ = ["start_ipython", "embed", "start_kernel", "embed_kernel"]
+
 # Release data
 __author__ = '%s <%s>' % (release.author, release.author_email)
 __license__  = release.license
 __version__  = release.version
 version_info = release.version_info
 # list of CVEs that should have been patched in this release.
 # this is informational and should not be relied upon.
 __patched_cves__ = {"CVE-2022-21699", "CVE-2023-24816"}
 
 
 def embed_kernel(module=None, local_ns=None, **kwargs):
     """Embed and start an IPython kernel in a given scope.
 
     If you don't want the kernel to initialize the namespace
     from the scope of the surrounding function,
     and/or you want to load full IPython configuration,
     you probably want `IPython.start_kernel()` instead.
 
     Parameters
     ----------
     module : types.ModuleType, optional
         The module to load into IPython globals (default: caller)
     local_ns : dict, optional
         The namespace to load into IPython user namespace (default: caller)
     **kwargs : various, optional
         Further keyword args are relayed to the IPKernelApp constructor,
-        allowing configuration of the Kernel.  Will only have an effect
+        such as `config`, a traitlets :class:`Config` object (see :ref:`configure_start_ipython`),
+        allowing configuration of the kernel (see :ref:`kernel_options`).  Will only have an effect
         on the first embed_kernel call for a given process.
     """
     
     (caller_module, caller_locals) = extract_module_locals(1)
     if module is None:
         module = caller_module
     if local_ns is None:
         local_ns = caller_locals
     
     # Only import .zmq when we really need it
     from ipykernel.embed import embed_kernel as real_embed_kernel
     real_embed_kernel(module=module, local_ns=local_ns, **kwargs)
 
 def start_ipython(argv=None, **kwargs):
     """Launch a normal IPython instance (as opposed to embedded)
 
     `IPython.embed()` puts a shell in a particular calling scope,
     such as a function or method for debugging purposes,
     which is often not desirable.
 
     `start_ipython()` does full, regular IPython initialization,
     including loading startup files, configuration, etc.
     much of which is skipped by `embed()`.
 
     This is a public API method, and will survive implementation changes.
 
     Parameters
     ----------
     argv : list or None, optional
         If unspecified or None, IPython will parse command-line options from sys.argv.
         To prevent any command-line parsing, pass an empty list: `argv=[]`.
     user_ns : dict, optional
         specify this dictionary to initialize the IPython user namespace with particular values.
     **kwargs : various, optional
         Any other kwargs will be passed to the Application constructor,
-        such as `config`.
+        such as `config`, a traitlets :class:`Config` object (see :ref:`configure_start_ipython`),
+        allowing configuration of the instance (see :ref:`terminal_options`).
     """
     from IPython.terminal.ipapp import launch_new_instance
     return launch_new_instance(argv=argv, **kwargs)
 
 def start_kernel(argv=None, **kwargs):
     """Launch a normal IPython kernel instance (as opposed to embedded)
 
     `IPython.embed_kernel()` puts a shell in a particular calling scope,
     such as a function or method for debugging purposes,
     which is often not desirable.
 
     `start_kernel()` does full, regular IPython initialization,
     including loading startup files, configuration, etc.
-    much of which is skipped by `embed()`.
+    much of which is skipped by `embed_kernel()`.
 
     Parameters
     ----------
     argv : list or None, optional
         If unspecified or None, IPython will parse command-line options from sys.argv.
         To prevent any command-line parsing, pass an empty list: `argv=[]`.
     user_ns : dict, optional
         specify this dictionary to initialize the IPython user namespace with particular values.
     **kwargs : various, optional
         Any other kwargs will be passed to the Application constructor,
-        such as `config`.
+        such as `config`, a traitlets :class:`Config` object (see :ref:`configure_start_ipython`),
+        allowing configuration of the kernel (see :ref:`kernel_options`).
     """
     import warnings
 
     warnings.warn(
         "start_kernel is deprecated since IPython 8.0, use from `ipykernel.kernelapp.launch_new_instance`",
         DeprecationWarning,
         stacklevel=2,
     )
     from ipykernel.kernelapp import launch_new_instance
     return launch_new_instance(argv=argv, **kwargs)

IPython/terminal/embed.py

@@ -1,410 +1,420 b''
 # encoding: utf-8
 """
 An embedded IPython shell.
 """
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 
 import sys
 import warnings
 
 from IPython.core import ultratb, compilerop
 from IPython.core import magic_arguments
 from IPython.core.magic import Magics, magics_class, line_magic
 from IPython.core.interactiveshell import DummyMod, InteractiveShell
 from IPython.terminal.interactiveshell import TerminalInteractiveShell
 from IPython.terminal.ipapp import load_default_config
 
 from traitlets import Bool, CBool, Unicode
 from IPython.utils.io import ask_yes_no
 
 from typing import Set
 
 class KillEmbedded(Exception):pass
 
 # kept for backward compatibility as IPython 6 was released with
 # the typo. See https://github.com/ipython/ipython/pull/10706
 KillEmbeded = KillEmbedded
 
 # This is an additional magic that is exposed in embedded shells.
 @magics_class
 class EmbeddedMagics(Magics):
 
     @line_magic
     @magic_arguments.magic_arguments()
     @magic_arguments.argument('-i', '--instance', action='store_true',
                               help='Kill instance instead of call location')
     @magic_arguments.argument('-x', '--exit', action='store_true',
                               help='Also exit the current session')
     @magic_arguments.argument('-y', '--yes', action='store_true',
                               help='Do not ask confirmation')
     def kill_embedded(self, parameter_s=''):
         """%kill_embedded : deactivate for good the current embedded IPython
 
         This function (after asking for confirmation) sets an internal flag so
         that an embedded IPython will never activate again for the given call
         location. This is useful to permanently disable a shell that is being
         called inside a loop: once you've figured out what you needed from it,
         you may then kill it and the program will then continue to run without
         the interactive shell interfering again.
 
         Kill Instance Option:
 
             If for some reasons you need to kill the location where the instance
             is created and not called, for example if you create a single
             instance in one place and debug in many locations, you can use the
             ``--instance`` option to kill this specific instance. Like for the
             ``call location`` killing an "instance" should work even if it is
             recreated within a loop.
 
         .. note::
 
             This was the default behavior before IPython 5.2
 
         """
 
         args = magic_arguments.parse_argstring(self.kill_embedded, parameter_s)
         print(args)
         if args.instance:
             # let no ask
             if not args.yes:
                 kill = ask_yes_no(
                     "Are you sure you want to kill this embedded instance? [y/N] ", 'n')
             else:
                 kill = True
             if kill:
                 self.shell._disable_init_location()
                 print("This embedded IPython instance will not reactivate anymore "
                       "once you exit.")
         else:
             if not args.yes:
                 kill = ask_yes_no(
                     "Are you sure you want to kill this embedded call_location? [y/N] ", 'n')
             else:
                 kill = True
             if kill:
                 self.shell.embedded_active = False
                 print("This embedded IPython  call location will not reactivate anymore "
                       "once you exit.")
 
         if args.exit:
             # Ask-exit does not really ask, it just set internals flags to exit
             # on next loop.
             self.shell.ask_exit()
 
 
     @line_magic
     def exit_raise(self, parameter_s=''):
         """%exit_raise Make the current embedded kernel exit and raise and exception.
 
         This function sets an internal flag so that an embedded IPython will
         raise a `IPython.terminal.embed.KillEmbedded` Exception on exit, and then exit the current I. This is
         useful to permanently exit a loop that create IPython embed instance.
         """
 
         self.shell.should_raise = True
         self.shell.ask_exit()
 
 
 class _Sentinel:
     def __init__(self, repr):
         assert isinstance(repr, str)
         self.repr = repr
 
     def __repr__(self):
         return repr
 
 
 class InteractiveShellEmbed(TerminalInteractiveShell):
 
     dummy_mode = Bool(False)
     exit_msg = Unicode('')
     embedded = CBool(True)
     should_raise = CBool(False)
     # Like the base class display_banner is not configurable, but here it
     # is True by default.
     display_banner = CBool(True)
     exit_msg = Unicode()
 
     # When embedding, by default we don't change the terminal title
     term_title = Bool(False,
         help="Automatically set the terminal title"
     ).tag(config=True)
 
     _inactive_locations: Set[str] = set()
 
     def _disable_init_location(self):
         """Disable the current Instance creation location"""
         InteractiveShellEmbed._inactive_locations.add(self._init_location_id)
 
     @property
     def embedded_active(self):
         return (self._call_location_id not in InteractiveShellEmbed._inactive_locations)\
             and (self._init_location_id not in InteractiveShellEmbed._inactive_locations)
 
     @embedded_active.setter
     def embedded_active(self, value):
         if value:
             InteractiveShellEmbed._inactive_locations.discard(
                 self._call_location_id)
             InteractiveShellEmbed._inactive_locations.discard(
                 self._init_location_id)
         else:
             InteractiveShellEmbed._inactive_locations.add(
                 self._call_location_id)
 
     def __init__(self, **kw):
         assert (
             "user_global_ns" not in kw
         ), "Key word argument `user_global_ns` has been replaced by `user_module` since IPython 4.0."
 
         clid = kw.pop('_init_location_id', None)
         if not clid:
             frame = sys._getframe(1)
             clid = '%s:%s' % (frame.f_code.co_filename, frame.f_lineno)
         self._init_location_id = clid
 
         super(InteractiveShellEmbed,self).__init__(**kw)
 
         # don't use the ipython crash handler so that user exceptions aren't
         # trapped
         sys.excepthook = ultratb.FormattedTB(color_scheme=self.colors,
                                              mode=self.xmode,
                                              call_pdb=self.pdb)
 
     def init_sys_modules(self):
         """
         Explicitly overwrite :mod:`IPython.core.interactiveshell` to do nothing.
         """
         pass
 
     def init_magics(self):
         super(InteractiveShellEmbed, self).init_magics()
         self.register_magics(EmbeddedMagics)
 
     def __call__(
         self,
         header="",
         local_ns=None,
         module=None,
         dummy=None,
         stack_depth=1,
         compile_flags=None,
         **kw
     ):
         """Activate the interactive interpreter.
 
         __call__(self,header='',local_ns=None,module=None,dummy=None) -> Start
         the interpreter shell with the given local and global namespaces, and
         optionally print a header string at startup.
 
         The shell can be globally activated/deactivated using the
         dummy_mode attribute. This allows you to turn off a shell used
         for debugging globally.
 
         However, *each* time you call the shell you can override the current
         state of dummy_mode with the optional keyword parameter 'dummy'. For
         example, if you set dummy mode on with IPShell.dummy_mode = True, you
         can still have a specific call work by making it as IPShell(dummy=False).
         """
 
         # we are called, set the underlying interactiveshell not to exit.
         self.keep_running = True
 
         # If the user has turned it off, go away
         clid = kw.pop('_call_location_id', None)
         if not clid:
             frame = sys._getframe(1)
             clid = '%s:%s' % (frame.f_code.co_filename, frame.f_lineno)
         self._call_location_id = clid
 
         if not self.embedded_active:
             return
 
         # Normal exits from interactive mode set this flag, so the shell can't
         # re-enter (it checks this variable at the start of interactive mode).
         self.exit_now = False
 
         # Allow the dummy parameter to override the global __dummy_mode
         if dummy or (dummy != 0 and self.dummy_mode):
             return
 
         # self.banner is auto computed
         if header:
             self.old_banner2 = self.banner2
             self.banner2 = self.banner2 + '\n' + header + '\n'
         else:
             self.old_banner2 = ''
 
         if self.display_banner:
             self.show_banner()
 
         # Call the embedding code with a stack depth of 1 so it can skip over
         # our call and get the original caller's namespaces.
         self.mainloop(
             local_ns, module, stack_depth=stack_depth, compile_flags=compile_flags
         )
 
         self.banner2 = self.old_banner2
 
         if self.exit_msg is not None:
             print(self.exit_msg)
 
         if self.should_raise:
             raise KillEmbedded('Embedded IPython raising error, as user requested.')
 
     def mainloop(
         self,
         local_ns=None,
         module=None,
         stack_depth=0,
         compile_flags=None,
     ):
         """Embeds IPython into a running python program.
 
         Parameters
         ----------
         local_ns, module
             Working local namespace (a dict) and module (a module or similar
             object). If given as None, they are automatically taken from the scope
             where the shell was called, so that program variables become visible.
         stack_depth : int
             How many levels in the stack to go to looking for namespaces (when
             local_ns or module is None). This allows an intermediate caller to
             make sure that this function gets the namespace from the intended
             level in the stack. By default (0) it will get its locals and globals
             from the immediate caller.
         compile_flags
             A bit field identifying the __future__ features
             that are enabled, as passed to the builtin :func:`compile` function.
             If given as None, they are automatically taken from the scope where
             the shell was called.
 
         """
         
         # Get locals and globals from caller
         if ((local_ns is None or module is None or compile_flags is None)
             and self.default_user_namespaces):
             call_frame = sys._getframe(stack_depth).f_back
 
             if local_ns is None:
                 local_ns = call_frame.f_locals
             if module is None:
                 global_ns = call_frame.f_globals
                 try:
                     module = sys.modules[global_ns['__name__']]
                 except KeyError:
                     warnings.warn("Failed to get module %s" % \
                         global_ns.get('__name__', 'unknown module')
                     )
                     module = DummyMod()
                     module.__dict__ = global_ns
             if compile_flags is None:
                 compile_flags = (call_frame.f_code.co_flags &
                                  compilerop.PyCF_MASK)
         
         # Save original namespace and module so we can restore them after 
         # embedding; otherwise the shell doesn't shut down correctly.
         orig_user_module = self.user_module
         orig_user_ns = self.user_ns
         orig_compile_flags = self.compile.flags
         
         # Update namespaces and fire up interpreter
         
         # The global one is easy, we can just throw it in
         if module is not None:
             self.user_module = module
 
         # But the user/local one is tricky: ipython needs it to store internal
         # data, but we also need the locals. We'll throw our hidden variables
         # like _ih and get_ipython() into the local namespace, but delete them
         # later.
         if local_ns is not None:
             reentrant_local_ns = {k: v for (k, v) in local_ns.items() if k not in self.user_ns_hidden.keys()}
             self.user_ns = reentrant_local_ns
             self.init_user_ns()
 
         # Compiler flags
         if compile_flags is not None:
             self.compile.flags = compile_flags
 
         # make sure the tab-completer has the correct frame information, so it
         # actually completes using the frame's locals/globals
         self.set_completer_frame()
 
         with self.builtin_trap, self.display_trap:
             self.interact()
         
         # now, purge out the local namespace of IPython's hidden variables.
         if local_ns is not None:
             local_ns.update({k: v for (k, v) in self.user_ns.items() if k not in self.user_ns_hidden.keys()})
 
         
         # Restore original namespace so shell can shut down when we exit.
         self.user_module = orig_user_module
         self.user_ns = orig_user_ns
         self.compile.flags = orig_compile_flags
 
 
 def embed(*, header="", compile_flags=None, **kwargs):
     """Call this to embed IPython at the current point in your program.
 
-    The first invocation of this will create an :class:`InteractiveShellEmbed`
+    The first invocation of this will create a :class:`terminal.embed.InteractiveShellEmbed`
     instance and then call it.  Consecutive calls just call the already
     created instance.
 
     If you don't want the kernel to initialize the namespace
     from the scope of the surrounding function,
     and/or you want to load full IPython configuration,
     you probably want `IPython.start_ipython()` instead.
 
     Here is a simple example::
 
         from IPython import embed
         a = 10
         b = 20
         embed(header='First time')
         c = 30
         d = 40
         embed()
 
-    Full customization can be done by passing a :class:`Config` in as the
-    config argument.
+    Parameters
+    ----------
+
+    header : str
+        Optional header string to print at startup.
+    compile_flags
+        Passed to the `compile_flags` parameter of :py:meth:`terminal.embed.InteractiveShellEmbed.mainloop()`,
+        which is called when the :class:`terminal.embed.InteractiveShellEmbed` instance is called.
+    **kwargs : various, optional
+        Any other kwargs will be passed to the :class:`terminal.embed.InteractiveShellEmbed` constructor.
+        Full customization can be done by passing a traitlets :class:`Config` in as the
+        `config` argument (see :ref:`configure_start_ipython` and :ref:`terminal_options`).
     """
     config = kwargs.get('config')
     if config is None:
         config = load_default_config()
         config.InteractiveShellEmbed = config.TerminalInteractiveShell
         kwargs['config'] = config
     using = kwargs.get('using', 'sync')
     if using :
         kwargs['config'].update({'TerminalInteractiveShell':{'loop_runner':using, 'colors':'NoColor', 'autoawait': using!='sync'}})
     #save ps1/ps2 if defined
     ps1 = None
     ps2 = None
     try:
         ps1 = sys.ps1
         ps2 = sys.ps2
     except AttributeError:
         pass
     #save previous instance
     saved_shell_instance = InteractiveShell._instance
     if saved_shell_instance is not None:
         cls = type(saved_shell_instance)
         cls.clear_instance()
     frame = sys._getframe(1)
     shell = InteractiveShellEmbed.instance(_init_location_id='%s:%s' % (
         frame.f_code.co_filename, frame.f_lineno), **kwargs)
     shell(header=header, stack_depth=2, compile_flags=compile_flags,
         _call_location_id='%s:%s' % (frame.f_code.co_filename, frame.f_lineno))
     InteractiveShellEmbed.clear_instance()
     #restore previous instance
     if saved_shell_instance is not None:
         cls = type(saved_shell_instance)
         cls.clear_instance()
         for subclass in cls._walk_mro():
             subclass._instance = saved_shell_instance
     if ps1 is not None:
         sys.ps1 = ps1
         sys.ps2 = ps2

docs/README.rst

@@ -1,64 +1,64 b''
 IPython Documentation
 ---------------------
 
 This directory contains the majority of the documentation for IPython.
 
 
 Deploy docs
 -----------
 
 Documentation is automatically deployed on ReadTheDocs on every push or merged
 Pull requests.
 
 
 Requirements
 ------------
 
 The documentation must be built using Python 3.
 
 In addition to :ref:`devinstall`,
 the following tools are needed to build the documentation:
 
  - sphinx
  - sphinx_rtd_theme
  - docrepr
 
 In a conda environment, or a Python 3 ``venv``, you should be able to run::
 
   cd ipython
-  pip install .[doc] -U
+  pip install -U -r docs/requirements.txt
 
 
 Build Commands
 --------------
 
 The documentation gets built using ``make``, and comes in several flavors.
 
 ``make html`` - build the API and narrative documentation web pages, this is
 the default ``make`` target, so running just ``make`` is equivalent to ``make
 html``.
 
 ``make html_noapi`` - same as above, but without running the auto-generated API
 docs. When you are working on the narrative documentation, the most time
 consuming portion  of the build process is the processing and rendering of the
 API documentation. This build target skips that.
 
 ``make pdf`` will compile a pdf from the documentation.
 
 You can run ``make help`` to see information on all possible make targets.
 
 To save time,
 the make targets above only process the files that have been changed since the
 previous docs build.
 To remove the previous docs build you can use ``make clean``.
 You can also combine ``clean`` with other `make` commands;
 for example,
 ``make clean html`` will do a complete rebuild of the docs or ``make clean pdf`` will do a complete build of the pdf.
 
 
 Continuous Integration
 ----------------------
 
 Documentation builds are included in the Travis-CI continuous integration process,
 so you can see the results of the docs build for any pull request at
 https://travis-ci.org/ipython/ipython/pull_requests.

docs/autogen_api.py

@@ -1,80 +1,83 b''
 #!/usr/bin/env python
 """Script to auto-generate our API docs.
 """
 
 import os
 import sys
 
 pjoin = os.path.join
 
 here = os.path.abspath(os.path.dirname(__file__))
 sys.path.append(pjoin(os.path.abspath(here), 'sphinxext'))
 
 from apigen import ApiDocWriter
 
 source = pjoin(here, 'source')
 
 #*****************************************************************************
 if __name__ == '__main__':
     package = 'IPython'
     outdir = pjoin(source, 'api', 'generated')
     docwriter = ApiDocWriter(package,rst_extension='.rst')
     # You have to escape the . here because . is a special char for regexps.
     # You must do make clean if you change this!
     docwriter.package_skip_patterns += [r'\.external$',
                                         # Extensions are documented elsewhere.
                                         r'\.extensions',
                                         # Magics are documented separately
                                         r'\.core\.magics',
                                         # This isn't API
                                         r'\.sphinxext',
                                         # Shims
                                         r'\.kernel',
                                         r'\.terminal\.pt_inputhooks',
                                         ]
 
     # The inputhook* modules often cause problems on import, such as trying to
     # load incompatible Qt bindings. It's easiest to leave them all out. The
     docwriter.module_skip_patterns += [
         r"\.lib\.inputhook.+",
         r"\.ipdoctest",
         r"\.testing\.plugin",
         # Backwards compat import for lib.lexers
         r"\.nbconvert\.utils\.lexers",
         # We document this manually.
         r"\.utils\.py3compat",
         # These are exposed in display
         r"\.core\.display",
         r"\.lib\.display",
         # Shims
         r"\.config",
         r"\.consoleapp",
         r"\.frontend$",
         r"\.html",
         r"\.nbconvert",
         r"\.nbformat",
         r"\.parallel",
         r"\.qt",
         # this is deprecated.
         r"\.utils\.version",
         # Private APIs (there should be a lot more here)
         r"\.terminal\.ptutils",
     ]
     # main API is in the inputhook module, which is documented.
 
     # These modules import functions and classes from other places to expose
     # them as part of the public API. They must have __all__ defined. The
     # non-API modules they import from should be excluded by the skip patterns
     # above.
-    docwriter.names_from__all__.update({
-        'IPython.display',
-    })
-    
+    docwriter.names_from__all__.update(
+        {
+            "IPython",
+            "IPython.display",
+        }
+    )
+
     # Now, generate the outputs
     docwriter.write_api_docs(outdir)
     # Write index with .txt extension - we can include it, but Sphinx won't try
     # to compile it
     docwriter.write_index(outdir, 'gen.txt',
                           relative_to = pjoin(source, 'api')
                           )
     print ('%d files written' % len(docwriter.written_modules))

docs/autogen_config.py

@@ -1,126 +1,127 b''
 #!/usr/bin/env python
 
 import inspect
 from pathlib import Path
 from IPython.terminal.ipapp import TerminalIPythonApp
 from ipykernel.kernelapp import IPKernelApp
 from traitlets import Undefined
 from collections import defaultdict
 
 here = (Path(__file__)).parent
 options = here / "source" / "config" / "options"
 generated = options / "config-generated.txt"
 
 import textwrap
 indent = lambda text,n: textwrap.indent(text,n*' ')
 
 
 def interesting_default_value(dv):
     if (dv is None) or (dv is Undefined):
         return False
     if isinstance(dv, (str, list, tuple, dict, set)):
         return bool(dv)
     return True
 
 def format_aliases(aliases):
     fmted = []
     for a in aliases:
         dashes = '-' if len(a) == 1 else '--'
         fmted.append('``%s%s``' % (dashes, a))
     return ', '.join(fmted)
 
 def class_config_rst_doc(cls, trait_aliases):
     """Generate rST documentation for this class' config options.
 
     Excludes traits defined on parent classes.
     """
     lines = []
     classname = cls.__name__
     for k, trait in sorted(cls.class_traits(config=True).items()):
         ttype = trait.__class__.__name__
 
         fullname = classname + '.' + trait.name
         lines += ['.. configtrait:: ' + fullname,
                   ''
                  ]
 
         help = trait.help.rstrip() or 'No description'
         lines.append(indent(inspect.cleandoc(help), 4) + '\n')
 
         # Choices or type
         if 'Enum' in ttype:
             # include Enum choices
             lines.append(indent(
                 ':options: ' + ', '.join('``%r``' % x for x in trait.values), 4))
         else:
             lines.append(indent(':trait type: ' + ttype, 4))
 
         # Default value
         # Ignore boring default values like None, [] or ''
         if interesting_default_value(trait.default_value):
             try:
                 dvr = trait.default_value_repr()
             except Exception:
                 dvr = None  # ignore defaults we can't construct
             if dvr is not None:
                 if len(dvr) > 64:
                     dvr = dvr[:61] + '...'
                 # Double up backslashes, so they get to the rendered docs
                 dvr = dvr.replace('\\n', '\\\\n')
                 lines.append(indent(':default: ``%s``' % dvr, 4))
 
         # Command line aliases
         if trait_aliases[fullname]:
             fmt_aliases = format_aliases(trait_aliases[fullname])
             lines.append(indent(':CLI option: ' + fmt_aliases, 4))
 
         # Blank line
         lines.append('')
 
     return '\n'.join(lines)
 
 def reverse_aliases(app):
     """Produce a mapping of trait names to lists of command line aliases.
     """
     res = defaultdict(list)
     for alias, trait in app.aliases.items():
         res[trait].append(alias)
 
     # Flags also often act as aliases for a boolean trait.
     # Treat flags which set one trait to True as aliases.
     for flag, (cfg, _) in app.flags.items():
         if len(cfg) == 1:
             classname = list(cfg)[0]
             cls_cfg = cfg[classname]
             if len(cls_cfg) == 1:
                 traitname = list(cls_cfg)[0]
                 if cls_cfg[traitname] is True:
                     res[classname+'.'+traitname].append(flag)
 
     return res
 
 def write_doc(name, title, app, preamble=None):
     trait_aliases = reverse_aliases(app)
     filename = options / (name + ".rst")
     with open(filename, "w", encoding="utf-8") as f:
+        f.write(".. _" + name + "_options:" + "\n\n")
         f.write(title + "\n")
         f.write(("=" * len(title)) + "\n")
         f.write("\n")
         if preamble is not None:
             f.write(preamble + '\n\n')
         #f.write(app.document_config_options())
 
         for c in app._classes_inc_parents():
             f.write(class_config_rst_doc(c, trait_aliases))
             f.write('\n')
 
 
 if __name__ == '__main__':
     # Touch this file for the make target
     Path(generated).write_text("", encoding="utf-8")
 
     write_doc('terminal', 'Terminal IPython options', TerminalIPythonApp())
     write_doc('kernel', 'IPython kernel options', IPKernelApp(),
         preamble=("These options can be used in :file:`ipython_kernel_config.py`. "
                   "The kernel also respects any options in `ipython_config.py`"),
     )

docs/autogen_shortcuts.py

@@ -1,221 +1,221 b''
 from dataclasses import dataclass
 from inspect import getsource
 from pathlib import Path
 from typing import cast, List, Union
 from html import escape as html_escape
 import re
 
 from prompt_toolkit.keys import KEY_ALIASES
 from prompt_toolkit.key_binding import KeyBindingsBase
 from prompt_toolkit.filters import Filter, Condition
 from prompt_toolkit.shortcuts import PromptSession
 
 from IPython.terminal.shortcuts import create_ipython_shortcuts, create_identifier
 from IPython.terminal.shortcuts.filters import KEYBINDING_FILTERS
 
 
 @dataclass
 class Shortcut:
     #: a sequence of keys (each element on the list corresponds to pressing one or more keys)
-    keys_sequence: list[str]
+    keys_sequence: List[str]
     filter: str
 
 
 @dataclass
 class Handler:
     description: str
     identifier: str
 
 
 @dataclass
 class Binding:
     handler: Handler
     shortcut: Shortcut
 
 
 class _NestedFilter(Filter):
     """Protocol reflecting non-public prompt_toolkit's `_AndList` and `_OrList`."""
 
     filters: List[Filter]
 
 
 class _Invert(Filter):
     """Protocol reflecting non-public prompt_toolkit's `_Invert`."""
 
     filter: Filter
 
 
 conjunctions_labels = {"_AndList": "&", "_OrList": "|"}
 
 ATOMIC_CLASSES = {"Never", "Always", "Condition"}
 
 
 HUMAN_NAMES_FOR_FILTERS = {
     filter_: name for name, filter_ in KEYBINDING_FILTERS.items()
 }
 
 
 def format_filter(
     filter_: Union[Filter, _NestedFilter, Condition, _Invert],
     is_top_level=True,
     skip=None,
 ) -> str:
     """Create easily readable description of the filter."""
     s = filter_.__class__.__name__
     if s == "Condition":
         func = cast(Condition, filter_).func
         if filter_ in HUMAN_NAMES_FOR_FILTERS:
             return HUMAN_NAMES_FOR_FILTERS[filter_]
         name = func.__name__
         if name == "<lambda>":
             source = getsource(func)
             return source.split("=")[0].strip()
         return func.__name__
     elif s == "_Invert":
         operand = cast(_Invert, filter_).filter
         if operand.__class__.__name__ in ATOMIC_CLASSES:
             return f"~{format_filter(operand, is_top_level=False)}"
         return f"~({format_filter(operand, is_top_level=False)})"
     elif s in conjunctions_labels:
         filters = cast(_NestedFilter, filter_).filters
         if filter_ in HUMAN_NAMES_FOR_FILTERS:
             return HUMAN_NAMES_FOR_FILTERS[filter_]
         conjunction = conjunctions_labels[s]
         glue = f" {conjunction} "
         result = glue.join(format_filter(x, is_top_level=False) for x in filters)
         if len(filters) > 1 and not is_top_level:
             result = f"({result})"
         return result
     elif s in ["Never", "Always"]:
         return s.lower()
     else:
         raise ValueError(f"Unknown filter type: {filter_}")
 
 
 def sentencize(s) -> str:
     """Extract first sentence"""
     s = re.split(r"\.\W", s.replace("\n", " ").strip())
     s = s[0] if len(s) else ""
     if not s.endswith("."):
         s += "."
     try:
         return " ".join(s.split())
     except AttributeError:
         return s
 
 
 class _DummyTerminal:
     """Used as a buffer to get prompt_toolkit bindings
     """
     handle_return = None
     input_transformer_manager = None
     display_completions = None
     editing_mode = "emacs"
     auto_suggest = None
 
 
 def bindings_from_prompt_toolkit(prompt_bindings: KeyBindingsBase) -> List[Binding]:
     """Collect bindings to a simple format that does not depend on prompt-toolkit internals"""
     bindings: List[Binding] = []
 
     for kb in prompt_bindings.bindings:
         bindings.append(
             Binding(
                 handler=Handler(
                     description=kb.handler.__doc__ or "",
                     identifier=create_identifier(kb.handler),
                 ),
                 shortcut=Shortcut(
                     keys_sequence=[
                         str(k.value) if hasattr(k, "value") else k for k in kb.keys
                     ],
                     filter=format_filter(kb.filter, skip={"has_focus_filter"}),
                 ),
             )
         )
     return bindings
 
 
 INDISTINGUISHABLE_KEYS = {**KEY_ALIASES, **{v: k for k, v in KEY_ALIASES.items()}}
 
 
 def format_prompt_keys(keys: str, add_alternatives=True) -> str:
     """Format prompt toolkit key with modifier into an RST representation."""
 
     def to_rst(key):
         escaped = key.replace("\\", "\\\\")
         return f":kbd:`{escaped}`"
 
-    keys_to_press: list[str]
+    keys_to_press: List[str]
 
     prefixes = {
         "c-s-": [to_rst("ctrl"), to_rst("shift")],
         "s-c-": [to_rst("ctrl"), to_rst("shift")],
         "c-": [to_rst("ctrl")],
         "s-": [to_rst("shift")],
     }
 
     for prefix, modifiers in prefixes.items():
         if keys.startswith(prefix):
             remainder = keys[len(prefix) :]
             keys_to_press = [*modifiers, to_rst(remainder)]
             break
     else:
         keys_to_press = [to_rst(keys)]
 
     result = " + ".join(keys_to_press)
 
     if keys in INDISTINGUISHABLE_KEYS and add_alternatives:
         alternative = INDISTINGUISHABLE_KEYS[keys]
 
         result = (
             result
             + " (or "
             + format_prompt_keys(alternative, add_alternatives=False)
             + ")"
         )
 
     return result
 
 
 if __name__ == '__main__':
     here = Path(__file__).parent
     dest = here / "source" / "config" / "shortcuts"
 
     ipy_bindings = create_ipython_shortcuts(_DummyTerminal())
 
     session = PromptSession(key_bindings=ipy_bindings)
     prompt_bindings = session.app.key_bindings
 
     assert prompt_bindings
     # Ensure that we collected the default shortcuts
     assert len(prompt_bindings.bindings) > len(ipy_bindings.bindings)
 
     bindings = bindings_from_prompt_toolkit(prompt_bindings)
 
     def sort_key(binding: Binding):
         return binding.handler.identifier, binding.shortcut.filter
 
     filters = []
     with (dest / "table.tsv").open("w", encoding="utf-8") as csv:
         for binding in sorted(bindings, key=sort_key):
             sequence = ", ".join(
                 [format_prompt_keys(keys) for keys in binding.shortcut.keys_sequence]
             )
             if binding.shortcut.filter == "always":
                 condition_label = "-"
             else:
                 # we cannot fit all the columns as the filters got too complex over time
                 condition_label = "ⓘ"
 
             csv.write(
                 "\t".join(
                     [
                         sequence,
                         sentencize(binding.handler.description)
                         + f" :raw-html:`<br>` `{binding.handler.identifier}`",
                         f':raw-html:`<span title="{html_escape(binding.shortcut.filter)}" style="cursor: help">{condition_label}</span>`',
                     ]
                 )
                 + "\n"
             )

docs/source/config/intro.rst

@@ -1,235 +1,235 b''
 =====================================
 Introduction to IPython configuration
 =====================================
 
 .. _setting_config:
 
 Setting configurable options
 ============================
 
 Many of IPython's classes have configurable attributes (see
 :doc:`options/index` for the list). These can be
 configured in several ways.
 
 Python configuration files
 --------------------------
 
 To create the blank configuration files, run::
 
     ipython profile create [profilename]
 
 If you leave out the profile name, the files will be created for the
 ``default`` profile (see :ref:`profiles`). These will typically be located in
 :file:`~/.ipython/profile_default/`, and will be named
 :file:`ipython_config.py`, for historical reasons you may also find files
 named with IPython prefix instead of Jupyter:
 :file:`ipython_notebook_config.py`, etc. The settings in
 :file:`ipython_config.py` apply to all IPython commands.
 
 By default, configuration files are fully featured Python scripts that can
 execute arbitrary code, the main usage is to set value on the configuration
 object ``c`` which exist in your configuration file.
 
 You can then configure class attributes like this::
 
     c.InteractiveShell.automagic = False
 
 Be careful with spelling--incorrect names will simply be ignored, with
 no error. 
 
 To add to a collection which may have already been defined elsewhere or have
 default values, you can use methods like those found on lists, dicts and
 sets: append, extend, :meth:`~traitlets.config.LazyConfigValue.prepend` (like
 extend, but at the front), add and update (which works both for dicts and
 sets)::
 
     c.InteractiveShellApp.extensions.append('Cython')
 
 .. versionadded:: 2.0
    list, dict and set methods for config values
 
 Example configuration file
 ``````````````````````````
 
 ::
 
     # sample ipython_config.py
 
     c.TerminalIPythonApp.display_banner = True
     c.InteractiveShellApp.log_level = 20
     c.InteractiveShellApp.extensions = [
         'myextension'
     ]
     c.InteractiveShellApp.exec_lines = [
         'import numpy',
         'import scipy'
     ]
     c.InteractiveShellApp.exec_files = [
         'mycode.py',
         'fancy.ipy'
     ]
     c.InteractiveShell.colors = 'LightBG'
     c.InteractiveShell.xmode = 'Context'
     c.TerminalInteractiveShell.confirm_exit = False
     c.TerminalInteractiveShell.editor = 'nano'
 
     c.PrefilterManager.multi_line_specials = True
 
     c.AliasManager.user_aliases = [
      ('la', 'ls -al')
     ]
 
 JSON Configuration files
 ------------------------
 
 In case where executability of configuration can be problematic, or
 configurations need to be modified programmatically, IPython also support a
 limited set of functionalities via ``.json`` configuration files. 
 
 You can defined most of the configuration options via a json object which
 hierarchy represent the value you would normally set on the ``c`` object of
 ``.py`` configuration files. The following ``ipython_config.json`` file::
 
     {
         "InteractiveShell": {
             "colors": "LightBG",
         },
         "InteractiveShellApp": {
             "extensions": [
                 "myextension"
             ]
         },
         "TerminalInteractiveShell": {
             "editor": "nano"
         }
     }
 
 Is equivalent to the following ``ipython_config.py``::
 
     c.InteractiveShellApp.extensions = [
         'myextension'
     ]
 
     c.InteractiveShell.colors = 'LightBG'
     c.TerminalInteractiveShell.editor = 'nano'
 
 
 Command line arguments
 ----------------------
 
 Every configurable value can be set from the command line, using this
 syntax::
 
     ipython --ClassName.attribute=value
 
 Many frequently used options have short aliases and flags, such as
 ``--matplotlib`` (to integrate with a matplotlib GUI event loop) or
 ``--pdb`` (automatic post-mortem debugging of exceptions).
 
 To see all of these abbreviated options, run::
 
     ipython --help
     jupyter notebook --help
     # etc.
 
 Options specified at the command line, in either format, override
 options set in a configuration file.
 
 The config magic
 ----------------
 
 You can also modify config from inside IPython, using a magic command::
 
     %config IPCompleter.greedy = True
 
 At present, this only affects the current session - changes you make to
 config are not saved anywhere. Also, some options are only read when
 IPython starts, so they can't be changed like this.
 
 .. _configure_start_ipython:
 
 Running IPython from Python
 ----------------------------
 
 If you are using :ref:`embedding` to start IPython from a normal 
 python file, you can set configuration options the same way as in a 
-config file by creating a traitlets config object and passing it to 
+config file by creating a traitlets :class:`Config` object and passing it to 
 start_ipython like in the example below.
 
 .. literalinclude:: ../../../examples/Embedding/start_ipython_config.py
     :language: python
 
 .. _profiles:
 
 Profiles
 ========
 
 IPython can use multiple profiles, with separate configuration and
 history. By default, if you don't specify a profile, IPython always runs
 in the ``default`` profile. To use a new profile::
 
     ipython profile create foo   # create the profile foo
     ipython --profile=foo        # start IPython using the new profile
 
 Profiles are typically stored in :ref:`ipythondir`, but you can also keep
 a profile in the current working directory, for example to distribute it
 with a project. To find a profile directory on the filesystem::
 
     ipython locate profile foo
 
 .. _ipythondir:
 
 The IPython directory
 =====================
 
 IPython stores its files---config, command history and extensions---in
 the directory :file:`~/.ipython/` by default.
 
 .. envvar:: IPYTHONDIR
 
    If set, this environment variable should be the path to a directory,
    which IPython will use for user data. IPython will create it if it
    does not exist.
 
 .. option:: --ipython-dir=<path>
 
    This command line option can also be used to override the default
    IPython directory.
 
 To see where IPython is looking for the IPython directory, use the command
 ``ipython locate``, or the Python function :func:`IPython.paths.get_ipython_dir`.
 
 
 Systemwide configuration
 ========================
 
 It can be useful to deploy systemwide ipython or ipykernel configuration
 when managing environment for many users. At startup time IPython and
 IPykernel will search for configuration file in multiple systemwide
 locations, mainly:
 
   - ``/etc/ipython/``
   - ``/usr/local/etc/ipython/``
 
 When the global install is a standalone python distribution it may also
 search in distribution specific location, for example:
 
   - ``$ANACONDA_LOCATION/etc/ipython/``
 
 In those locations, Terminal IPython will look for a file called
 ``ipython_config.py`` and ``ipython_config.json``, ipykernel will look for
 ``ipython_kernel_config.py`` and ``ipython_kernel.json``.
 
 Configuration files are loaded in order and merged with configuration on
 later location taking precedence on earlier locations (that is to say a user
 can overwrite a systemwide configuration option).
 
 You can see all locations in which IPython is looking for configuration files
 by starting ipython in debug mode::
 
     $ ipython --debug -c 'exit()'
 
 Identically with ipykernel though the command is currently blocking until
 this process is killed with ``Ctrl-\``::
  
     $ python -m ipykernel --debug

docs/source/config/options/index.rst

@@ -1,12 +1,12 b''
 ===============
 IPython options
 ===============
 
 Any of the options listed here can be set in config files, at the
-command line, or from inside IPython. See :ref:`setting_config` for
-details.
+command line, from inside IPython, or using a traitlets :class:`Config` object.
+See :ref:`setting_config` for details.
 
 .. toctree::
 
    terminal
    kernel