Merge branch 'main' into latex_rendering_tempdir

IPython/tests/test_shortcuts.py

@@ -0,0 +1,40 b''
+import pytest
+from IPython.terminal.shortcuts import _apply_autosuggest
+
+from unittest.mock import Mock
+
+
+def make_event(text, cursor, suggestion):
+    event = Mock()
+    event.current_buffer = Mock()
+    event.current_buffer.suggestion = Mock()
+    event.current_buffer.cursor_position = cursor
+    event.current_buffer.suggestion.text = suggestion
+    event.current_buffer.document = Mock()
+    event.current_buffer.document.get_end_of_line_position = Mock(return_value=0)
+    event.current_buffer.document.text = text
+    event.current_buffer.document.cursor_position = cursor
+    return event
+
+
[e-mail](
+    "text, cursor, suggestion, called",
+    [
+        ("123456", 6, "123456789", True),
+        ("123456", 3, "123456789", False),
+        ("123456   \n789", 6, "123456789", True),
+    ],
+)
+def test_autosuggest_at_EOL(text, cursor, suggestion, called):
+    """
+    test that autosuggest is only applied at end of line.
+    """
+
+    event = make_event(text, cursor, suggestion)
+    event.current_buffer.insert_text = Mock()
+    _apply_autosuggest(event)
+    if called:
+        event.current_buffer.insert_text.assert_called()
+    else:
+        event.current_buffer.insert_text.assert_not_called()
+        # event.current_buffer.document.get_end_of_line_position.assert_called()

.github/workflows/mypy.yml

@@ -1,34 +1,34 b''
 name: Run MyPy
 
 on:
   push:
-    branches: [ master, 7.x]
+    branches: [ main, 7.x]
   pull_request:
-    branches: [ master, 7.x]
+    branches: [ main, 7.x]
 
 jobs:
   build:
 
     runs-on: ubuntu-latest
     strategy:
       matrix:
         python-version: [3.8]
 
     steps:
     - uses: actions/checkout@v2
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v2
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
         pip install mypy pyflakes flake8
     - name: Lint with mypy
       run: |
         mypy -p IPython.terminal
         mypy -p IPython.core.magics
     - name: Lint with pyflakes
       run: |
         flake8  IPython/core/magics/script.py
         flake8  IPython/core/magics/packaging.py

.github/workflows/python-package.yml

@@ -1,40 +1,40 b''
 # This workflow will install Python dependencies, run tests and lint with a variety of Python versions
 # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
 
 name: Python package
 
 on:
   push:
-    branches: [ master, 7.x ]
+    branches: [ main, 7.x ]
   pull_request:
-    branches: [ master, 7.x ]
+    branches: [ main, 7.x ]
 
 jobs:
   formatting:
 
     runs-on: ubuntu-latest
     timeout-minutes: 5
     strategy:
       matrix:
         python-version: [3.8]
 
     steps:
     - uses: actions/checkout@v2
       with:
         fetch-depth: 0
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v2
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
         pip install darker black==21.12b0
     - name: Lint with darker
       run: |
         darker -r 60625f241f298b5039cb2debc365db38aa7bb522 --check --diff . || (
         echo "Changes need auto-formatting. Run:"
         echo "    darker -r 60625f241f298b5039cb2debc365db38aa7bb522"
         echo "then commit and push changes to fix."
         exit 1
         )

.github/workflows/test.yml

@@ -1,81 +1,80 b''
 name: Run tests
 
 on:
   push:
     branches:
       - main
-      - master
       - '*.x'
   pull_request:
   # Run weekly on Monday at 1:23 UTC
   schedule:
   - cron:  '23 1 * * 1'
   workflow_dispatch:
 
 
 jobs:
   test:
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, windows-latest]
         python-version: ["3.8", "3.9", "3.10"]
         deps: [test_extra]
         # Test all on ubuntu, test ends on macos
         include:
           - os: macos-latest
             python-version: "3.8"
             deps: test_extra
           - os: macos-latest
             python-version: "3.10"
             deps: test_extra
           # Tests minimal dependencies set
           - os: ubuntu-latest
             python-version: "3.10"
             deps: test
           # Tests latest development Python version
           - os: ubuntu-latest
             python-version: "3.11-dev"
             deps: test
           # Installing optional dependencies stuff takes ages on PyPy
           - os: ubuntu-latest
             python-version: "pypy-3.8"
             deps: test
           - os: windows-latest
             python-version: "pypy-3.8"
             deps: test
           - os: macos-latest
             python-version: "pypy-3.8"
             deps: test
 
     steps:
     - uses: actions/checkout@v2
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v2
       with:
         python-version: ${{ matrix.python-version }}
         cache: pip
     - name: Install latex
       if: runner.os == 'Linux' && matrix.deps == 'test_extra'
       run: echo "disable latex for now, issues in mirros" #sudo apt-get -yq -o Acquire::Retries=3 --no-install-suggests --no-install-recommends install texlive dvipng
     - name: Install and update Python dependencies
       run: |
         python -m pip install --upgrade pip setuptools wheel build
         python -m pip install --upgrade -e .[${{ matrix.deps }}]
         python -m pip install --upgrade check-manifest pytest-cov
     - name: Try building with Python build
       if: runner.os != 'Windows'  # setup.py does not support sdist on Windows
       run: |
         python -m build
         shasum -a 256 dist/*
     - name: Check manifest
       if: runner.os != 'Windows'  # setup.py does not support sdist on Windows
       run: check-manifest
     - name: pytest
       env:
         COLUMNS: 120
       run: |
         pytest --color=yes -raXxs ${{ startsWith(matrix.python-version, 'pypy') && ' ' || '--cov --cov-report=xml' }}
     - name: Upload coverage to Codecov
       uses: codecov/codecov-action@v2

CONTRIBUTING.md

@@ -1,90 +1,90 b''
 ## Triaging Issues
 
 On the IPython repository,  we strive to trust users and give them responsibility.
 By using one of our bots, any user can close issues or add/remove
 labels by mentioning the bot and asking it to do things on your behalf.
 
 To close an issue (or PR), even if you did not create it, use the following:
 
 > @meeseeksdev close
 
 This command can be in the middle of another comment, but must start on its
 own line. 
 
 To add labels to an issue, ask the bot to `tag` with a comma-separated list of
 tags to add:
 
 > @meeseeksdev tag windows, documentation
 
 Only already pre-created tags can be added.  So far, the list is limited to:
 `async/await`, `backported`, `help wanted`, `documentation`, `notebook`,
 `tab-completion`, `windows`
 
 To remove a label, use the `untag` command:
 
 > @meeseeksdev untag windows, documentation
 
 We'll be adding additional capabilities for the bot and will share them here
 when they are ready to be used.
 
 ## Opening an Issue
 
 When opening a new Issue, please take the following steps:
 
 1. Search GitHub and/or Google for your issue to avoid duplicate reports.
    Keyword searches for your error messages are most helpful.
-2. If possible, try updating to master and reproducing your issue,
+2. If possible, try updating to main and reproducing your issue,
    because we may have already fixed it.
 3. Try to include a minimal reproducible test case.
 4. Include relevant system information.  Start with the output of:
 
         python -c "import IPython; print(IPython.sys_info())"
 
    And include any relevant package versions, depending on the issue, such as
    matplotlib, numpy, Qt, Qt bindings (PyQt/PySide), tornado, web browser, etc.
 
 ## Pull Requests
 
 Some guidelines on contributing to IPython:
 
 * All work is submitted via Pull Requests.
 * Pull Requests can be submitted as soon as there is code worth discussing.
   Pull Requests track the branch, so you can continue to work after the PR is submitted.
   Review and discussion can begin well before the work is complete,
   and the more discussion the better.
   The worst case is that the PR is closed.
-* Pull Requests should generally be made against master
+* Pull Requests should generally be made against main
 * Pull Requests should be tested, if feasible:
     - bugfixes should include regression tests.
     - new behavior should at least get minimal exercise.
 * New features and backwards-incompatible changes should be documented by adding
   a new file to the [pr](docs/source/whatsnew/pr) directory, see [the README.md
   there](docs/source/whatsnew/pr/README.md) for details.
 * Don't make 'cleanup' pull requests just to change code style.
   We don't follow any style guide strictly, and we consider formatting changes
   unnecessary noise.
   If you're making functional changes, you can clean up the specific pieces of
   code you're working on.
 
 [Travis](http://travis-ci.org/#!/ipython/ipython) does a pretty good job testing
 IPython and Pull Requests, but it may make sense to manually perform tests,
 particularly for PRs that affect `IPython.parallel` or Windows.
 
 For more detailed information, see our [GitHub Workflow](https://github.com/ipython/ipython/wiki/Dev:-GitHub-workflow).
 
 ## Running Tests
 
 All the tests can be run by using
 ```shell
 pytest
 ```
 
 All the tests for a single module (for example **test_alias**) can be run by using the fully qualified path to the module.
 ```shell
 pytest IPython/core/tests/test_alias.py
 ```
 
 Only a single test (for example **test_alias_lifecycle**) within a single file can be run by adding the specific test after a `::` at the end:
 ```shell
 pytest IPython/core/tests/test_alias.py::test_alias_lifecycle
 ```

IPython/__init__.py

@@ -1,156 +1,156 b''
 """
 IPython: tools for interactive and parallel computing in Python.
 
 https://ipython.org
 """
 #-----------------------------------------------------------------------------
 #  Copyright (c) 2008-2011, IPython Development Team.
 #  Copyright (c) 2001-2007, Fernando Perez <[e-mail]>
 #  Copyright (c) 2001, Janko Hauser <[e-mail]>
 #  Copyright (c) 2001, Nathaniel Gray <[e-mail]>
 #
 #  Distributed under the terms of the Modified BSD License.
 #
 #  The full license is in the file COPYING.txt, distributed with this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import os
 import sys
 
 #-----------------------------------------------------------------------------
 # Setup everything
 #-----------------------------------------------------------------------------
 
 # Don't forget to also update setup.py when this changes!
 if sys.version_info < (3, 8):
     raise ImportError(
         """
 IPython 8+ supports Python 3.8 and above, following NEP 29.
 When using Python 2.7, please install IPython 5.x LTS Long Term Support version.
 Python 3.3 and 3.4 were supported up to IPython 6.x.
 Python 3.5 was supported with IPython 7.0 to 7.9.
 Python 3.6 was supported with IPython up to 7.16.
 Python 3.7 was still supported with the 7.x branch.
 
 See IPython `README.rst` file for more information:
 
-    https://github.com/ipython/ipython/blob/master/README.rst
+    https://github.com/ipython/ipython/blob/main/README.rst
 
 """
     )
 
 #-----------------------------------------------------------------------------
 # Setup the top level names
 #-----------------------------------------------------------------------------
 
 from .core.getipython import get_ipython
 from .core import release
 from .core.application import Application
 from .terminal.embed import embed
 
 from .core.interactiveshell import InteractiveShell
 from .utils.sysinfo import sys_info
 from .utils.frame import extract_module_locals
 
 # Release data
 __author__ = '%s <%s>' % (release.author, release.author_email)
 __license__  = release.license
 __version__  = release.version
 version_info = release.version_info
 # list of CVEs that should have been patched in this release.
 # this is informational and should not be relied upon.
 __patched_cves__ = {"CVE-2022-21699"}
 
 
 def embed_kernel(module=None, local_ns=None, **kwargs):
     """Embed and start an IPython kernel in a given scope.
 
     If you don't want the kernel to initialize the namespace
     from the scope of the surrounding function,
     and/or you want to load full IPython configuration,
     you probably want `IPython.start_kernel()` instead.
 
     Parameters
     ----------
     module : types.ModuleType, optional
         The module to load into IPython globals (default: caller)
     local_ns : dict, optional
         The namespace to load into IPython user namespace (default: caller)
     **kwargs : various, optional
         Further keyword args are relayed to the IPKernelApp constructor,
         allowing configuration of the Kernel.  Will only have an effect
         on the first embed_kernel call for a given process.
     """
     
     (caller_module, caller_locals) = extract_module_locals(1)
     if module is None:
         module = caller_module
     if local_ns is None:
         local_ns = caller_locals
     
     # Only import .zmq when we really need it
     from ipykernel.embed import embed_kernel as real_embed_kernel
     real_embed_kernel(module=module, local_ns=local_ns, **kwargs)
 
 def start_ipython(argv=None, **kwargs):
     """Launch a normal IPython instance (as opposed to embedded)
 
     `IPython.embed()` puts a shell in a particular calling scope,
     such as a function or method for debugging purposes,
     which is often not desirable.
 
     `start_ipython()` does full, regular IPython initialization,
     including loading startup files, configuration, etc.
     much of which is skipped by `embed()`.
 
     This is a public API method, and will survive implementation changes.
 
     Parameters
     ----------
     argv : list or None, optional
         If unspecified or None, IPython will parse command-line options from sys.argv.
         To prevent any command-line parsing, pass an empty list: `argv=[]`.
     user_ns : dict, optional
         specify this dictionary to initialize the IPython user namespace with particular values.
     **kwargs : various, optional
         Any other kwargs will be passed to the Application constructor,
         such as `config`.
     """
     from IPython.terminal.ipapp import launch_new_instance
     return launch_new_instance(argv=argv, **kwargs)
 
 def start_kernel(argv=None, **kwargs):
     """Launch a normal IPython kernel instance (as opposed to embedded)
 
     `IPython.embed_kernel()` puts a shell in a particular calling scope,
     such as a function or method for debugging purposes,
     which is often not desirable.
 
     `start_kernel()` does full, regular IPython initialization,
     including loading startup files, configuration, etc.
     much of which is skipped by `embed()`.
 
     Parameters
     ----------
     argv : list or None, optional
         If unspecified or None, IPython will parse command-line options from sys.argv.
         To prevent any command-line parsing, pass an empty list: `argv=[]`.
     user_ns : dict, optional
         specify this dictionary to initialize the IPython user namespace with particular values.
     **kwargs : various, optional
         Any other kwargs will be passed to the Application constructor,
         such as `config`.
     """
     import warnings
 
     warnings.warn(
         "start_kernel is deprecated since IPython 8.0, use from `ipykernel.kernelapp.launch_new_instance`",
         DeprecationWarning,
         stacklevel=2,
     )
     from ipykernel.kernelapp import launch_new_instance
     return launch_new_instance(argv=argv, **kwargs)

IPython/core/completer.py

@@ -1,2272 +1,2276 b''
 """Completion for IPython.
 
 This module started as fork of the rlcompleter module in the Python standard
 library.  The original enhancements made to rlcompleter have been sent
 upstream and were accepted as of Python 2.3,
 
 This module now support a wide variety of completion mechanism both available
 for normal classic Python code, as well as completer for IPython specific
 Syntax like magics.
 
 Latex and Unicode completion
 ============================
 
 IPython and compatible frontends not only can complete your code, but can help
 you to input a wide range of characters. In particular we allow you to insert
 a unicode character using the tab completion mechanism.
 
 Forward latex/unicode completion
 --------------------------------
 
 Forward completion allows you to easily type a unicode character using its latex
 name, or unicode long description. To do so type a backslash follow by the
 relevant name and press tab:
 
 
 Using latex completion:
 
 .. code::
 
     \\alpha<tab>
     α
 
 or using unicode completion:
 
 
 .. code::
 
     \\GREEK SMALL LETTER ALPHA<tab>
     α
 
 
 Only valid Python identifiers will complete. Combining characters (like arrow or
 dots) are also available, unlike latex they need to be put after the their
 counterpart that is to say, ``F\\\\vec<tab>`` is correct, not ``\\\\vec<tab>F``.
 
 Some browsers are known to display combining characters incorrectly.
 
 Backward latex completion
 -------------------------
 
 It is sometime challenging to know how to type a character, if you are using
 IPython, or any compatible frontend you can prepend backslash to the character
 and press ``<tab>`` to expand it to its latex form.
 
 .. code::
 
     \\α<tab>
     \\alpha
 
 
 Both forward and backward completions can be deactivated by setting the
 ``Completer.backslash_combining_completions`` option to ``False``.
 
 
 Experimental
 ============
 
 Starting with IPython 6.0, this module can make use of the Jedi library to
 generate completions both using static analysis of the code, and dynamically
 inspecting multiple namespaces. Jedi is an autocompletion and static analysis
 for Python. The APIs attached to this new mechanism is unstable and will
 raise unless use in an :any:`provisionalcompleter` context manager.
 
 You will find that the following are experimental:
 
     - :any:`provisionalcompleter`
     - :any:`IPCompleter.completions`
     - :any:`Completion`
     - :any:`rectify_completions`
 
 .. note::
 
     better name for :any:`rectify_completions` ?
 
 We welcome any feedback on these new API, and we also encourage you to try this
 module in debug mode (start IPython with ``--Completer.debug=True``) in order
 to have extra logging information if :any:`jedi` is crashing, or if current
 IPython completer pending deprecations are returning results not yet handled
 by :any:`jedi`
 
 Using Jedi for tab completion allow snippets like the following to work without
 having to execute any code:
 
    >>> myvar = ['hello', 42]
    ... myvar[1].bi<tab>
 
 Tab completion will be able to infer that ``myvar[1]`` is a real number without
 executing any code unlike the previously available ``IPCompleter.greedy``
 option.
 
 Be sure to update :any:`jedi` to the latest stable version or to try the
 current development version to get better completions.
 """
 
 
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 #
 # Some of this code originated from rlcompleter in the Python standard library
 # Copyright (C) 2001 Python Software Foundation, www.python.org
 
 
 import builtins as builtin_mod
 import glob
 import inspect
 import itertools
 import keyword
 import os
 import re
 import string
 import sys
 import time
 import unicodedata
 import uuid
 import warnings
 from contextlib import contextmanager
 from importlib import import_module
 from types import SimpleNamespace
 from typing import Iterable, Iterator, List, Tuple, Union, Any, Sequence, Dict, NamedTuple, Pattern, Optional
 
 from IPython.core.error import TryNext
 from IPython.core.inputtransformer2 import ESC_MAGIC
 from IPython.core.latex_symbols import latex_symbols, reverse_latex_symbol
 from IPython.core.oinspect import InspectColors
 from IPython.testing.skipdoctest import skip_doctest
 from IPython.utils import generics
 from IPython.utils.dir2 import dir2, get_real_method
 from IPython.utils.path import ensure_dir_exists
 from IPython.utils.process import arg_split
 from traitlets import Bool, Enum, Int, List as ListTrait, Unicode, default, observe
 from traitlets.config.configurable import Configurable
 
 import __main__
 
 # skip module docstests
 __skip_doctest__ = True
 
 try:
     import jedi
     jedi.settings.case_insensitive_completion = False
     import jedi.api.helpers
     import jedi.api.classes
     JEDI_INSTALLED = True
 except ImportError:
     JEDI_INSTALLED = False
 #-----------------------------------------------------------------------------
 # Globals
 #-----------------------------------------------------------------------------
 
 # ranges where we have most of the valid unicode names. We could be more finer
 # grained but is it worth it for performance  While unicode have character in the
 # range 0, 0x110000, we seem to have name for about 10% of those. (131808 as I
 # write this). With below range we cover them all, with a density of ~67%
 # biggest next gap we consider only adds up about 1% density and there are 600
 # gaps that would need hard coding.
 _UNICODE_RANGES = [(32, 0x3134b), (0xe0001, 0xe01f0)]
 
 # Public API
 __all__ = ['Completer','IPCompleter']
 
 if sys.platform == 'win32':
     PROTECTABLES = ' '
 else:
     PROTECTABLES = ' ()[]{}?=\\|;:\'#*"^&'
 
 # Protect against returning an enormous number of completions which the frontend
 # may have trouble processing.
 MATCHES_LIMIT = 500
 
 
 class ProvisionalCompleterWarning(FutureWarning):
     """
     Exception raise by an experimental feature in this module.
 
     Wrap code in :any:`provisionalcompleter` context manager if you
     are certain you want to use an unstable feature.
     """
     pass
 
 warnings.filterwarnings('error', category=ProvisionalCompleterWarning)
 
 
 @skip_doctest
 @contextmanager
 def provisionalcompleter(action='ignore'):
     """
     This context manager has to be used in any place where unstable completer
     behavior and API may be called.
 
     >>> with provisionalcompleter():
     ...     completer.do_experimental_things() # works
 
     >>> completer.do_experimental_things() # raises.
 
     .. note::
 
         Unstable
 
         By using this context manager you agree that the API in use may change
         without warning, and that you won't complain if they do so.
 
         You also understand that, if the API is not to your liking, you should report
         a bug to explain your use case upstream.
 
         We'll be happy to get your feedback, feature requests, and improvements on
         any of the unstable APIs!
     """
     with warnings.catch_warnings():
         warnings.filterwarnings(action, category=ProvisionalCompleterWarning)
         yield
 
 
 def has_open_quotes(s):
     """Return whether a string has open quotes.
 
     This simply counts whether the number of quote characters of either type in
     the string is odd.
 
     Returns
     -------
     If there is an open quote, the quote character is returned.  Else, return
     False.
     """
     # We check " first, then ', so complex cases with nested quotes will get
     # the " to take precedence.
     if s.count('"') % 2:
         return '"'
     elif s.count("'") % 2:
         return "'"
     else:
         return False
 
 
 def protect_filename(s, protectables=PROTECTABLES):
     """Escape a string to protect certain characters."""
     if set(s) & set(protectables):
         if sys.platform == "win32":
             return '"' + s + '"'
         else:
             return "".join(("\\" + c if c in protectables else c) for c in s)
     else:
         return s
 
 
 def expand_user(path:str) -> Tuple[str, bool, str]:
     """Expand ``~``-style usernames in strings.
 
     This is similar to :func:`os.path.expanduser`, but it computes and returns
     extra information that will be useful if the input was being used in
     computing completions, and you wish to return the completions with the
     original '~' instead of its expanded value.
 
     Parameters
     ----------
     path : str
         String to be expanded.  If no ~ is present, the output is the same as the
         input.
 
     Returns
     -------
     newpath : str
         Result of ~ expansion in the input path.
     tilde_expand : bool
         Whether any expansion was performed or not.
     tilde_val : str
         The value that ~ was replaced with.
     """
     # Default values
     tilde_expand = False
     tilde_val = ''
     newpath = path
 
     if path.startswith('~'):
         tilde_expand = True
         rest = len(path)-1
         newpath = os.path.expanduser(path)
         if rest:
             tilde_val = newpath[:-rest]
         else:
             tilde_val = newpath
 
     return newpath, tilde_expand, tilde_val
 
 
 def compress_user(path:str, tilde_expand:bool, tilde_val:str) -> str:
     """Does the opposite of expand_user, with its outputs.
     """
     if tilde_expand:
         return path.replace(tilde_val, '~')
     else:
         return path
 
 
 def completions_sorting_key(word):
     """key for sorting completions
 
     This does several things:
 
     - Demote any completions starting with underscores to the end
     - Insert any %magic and %%cellmagic completions in the alphabetical order
       by their name
     """
     prio1, prio2 = 0, 0
 
     if word.startswith('__'):
         prio1 = 2
     elif word.startswith('_'):
         prio1 = 1
 
     if word.endswith('='):
         prio1 = -1
 
     if word.startswith('%%'):
         # If there's another % in there, this is something else, so leave it alone
         if not "%" in word[2:]:
             word = word[2:]
             prio2 = 2
     elif word.startswith('%'):
         if not "%" in word[1:]:
             word = word[1:]
             prio2 = 1
 
     return prio1, word, prio2
 
 
 class _FakeJediCompletion:
     """
     This is a workaround to communicate to the UI that Jedi has crashed and to
     report a bug. Will be used only id :any:`IPCompleter.debug` is set to true.
 
     Added in IPython 6.0 so should likely be removed for 7.0
 
     """
 
     def __init__(self, name):
 
         self.name = name
         self.complete = name
         self.type = 'crashed'
         self.name_with_symbols = name
         self.signature = ''
         self._origin = 'fake'
 
     def __repr__(self):
         return '<Fake completion object jedi has crashed>'
 
 
 class Completion:
     """
     Completion object used and return by IPython completers.
 
     .. warning::
 
         Unstable
 
         This function is unstable, API may change without warning.
         It will also raise unless use in proper context manager.
 
     This act as a middle ground :any:`Completion` object between the
     :any:`jedi.api.classes.Completion` object and the Prompt Toolkit completion
     object. While Jedi need a lot of information about evaluator and how the
     code should be ran/inspected, PromptToolkit (and other frontend) mostly
     need user facing information.
 
     - Which range should be replaced replaced by what.
     - Some metadata (like completion type), or meta information to displayed to
       the use user.
 
     For debugging purpose we can also store the origin of the completion (``jedi``,
     ``IPython.python_matches``, ``IPython.magics_matches``...).
     """
 
     __slots__ = ['start', 'end', 'text', 'type', 'signature', '_origin']
 
     def __init__(self, start: int, end: int, text: str, *, type: str=None, _origin='', signature='') -> None:
         warnings.warn("``Completion`` is a provisional API (as of IPython 6.0). "
                       "It may change without warnings. "
                       "Use in corresponding context manager.",
                       category=ProvisionalCompleterWarning, stacklevel=2)
 
         self.start = start
         self.end = end
         self.text = text
         self.type = type
         self.signature = signature
         self._origin = _origin
 
     def __repr__(self):
         return '<Completion start=%s end=%s text=%r type=%r, signature=%r,>' % \
                 (self.start, self.end, self.text, self.type or '?', self.signature or '?')
 
     def __eq__(self, other)->Bool:
         """
         Equality and hash do not hash the type (as some completer may not be
         able to infer the type), but are use to (partially) de-duplicate
         completion.
 
         Completely de-duplicating completion is a bit tricker that just
         comparing as it depends on surrounding text, which Completions are not
         aware of.
         """
         return self.start == other.start and \
             self.end == other.end and \
             self.text == other.text
 
     def __hash__(self):
         return hash((self.start, self.end, self.text))
 
 
 _IC = Iterable[Completion]
 
 
 def _deduplicate_completions(text: str, completions: _IC)-> _IC:
     """
     Deduplicate a set of completions.
 
     .. warning::
 
         Unstable
 
         This function is unstable, API may change without warning.
 
     Parameters
     ----------
     text : str
         text that should be completed.
     completions : Iterator[Completion]
         iterator over the completions to deduplicate
 
     Yields
     ------
     `Completions` objects
     Completions coming from multiple sources, may be different but end up having
     the same effect when applied to ``text``. If this is the case, this will
     consider completions as equal and only emit the first encountered.
     Not folded in `completions()` yet for debugging purpose, and to detect when
     the IPython completer does return things that Jedi does not, but should be
     at some point.
     """
     completions = list(completions)
     if not completions:
         return
 
     new_start = min(c.start for c in completions)
     new_end = max(c.end for c in completions)
 
     seen = set()
     for c in completions:
         new_text = text[new_start:c.start] + c.text + text[c.end:new_end]
         if new_text not in seen:
             yield c
             seen.add(new_text)
 
 
 def rectify_completions(text: str, completions: _IC, *, _debug: bool = False) -> _IC:
     """
     Rectify a set of completions to all have the same ``start`` and ``end``
 
     .. warning::
 
         Unstable
 
         This function is unstable, API may change without warning.
         It will also raise unless use in proper context manager.
 
     Parameters
     ----------
     text : str
         text that should be completed.
     completions : Iterator[Completion]
         iterator over the completions to rectify
     _debug : bool
         Log failed completion
 
     Notes
     -----
     :any:`jedi.api.classes.Completion` s returned by Jedi may not have the same start and end, though
     the Jupyter Protocol requires them to behave like so. This will readjust
     the completion to have the same ``start`` and ``end`` by padding both
     extremities with surrounding text.
 
     During stabilisation should support a ``_debug`` option to log which
     completion are return by the IPython completer and not found in Jedi in
     order to make upstream bug report.
     """
     warnings.warn("`rectify_completions` is a provisional API (as of IPython 6.0). "
                  "It may change without warnings. "
                  "Use in corresponding context manager.",
                   category=ProvisionalCompleterWarning, stacklevel=2)
 
     completions = list(completions)
     if not completions:
         return
     starts = (c.start for c in completions)
     ends = (c.end for c in completions)
 
     new_start = min(starts)
     new_end = max(ends)
 
     seen_jedi = set()
     seen_python_matches = set()
     for c in completions:
         new_text = text[new_start:c.start] + c.text + text[c.end:new_end]
         if c._origin == 'jedi':
             seen_jedi.add(new_text)
         elif c._origin == 'IPCompleter.python_matches':
             seen_python_matches.add(new_text)
         yield Completion(new_start, new_end, new_text, type=c.type, _origin=c._origin, signature=c.signature)
     diff = seen_python_matches.difference(seen_jedi)
     if diff and _debug:
         print('IPython.python matches have extras:', diff)
 
 
 if sys.platform == 'win32':
     DELIMS = ' \t\n`!@#$^&*()=+[{]}|;\'",<>?'
 else:
     DELIMS = ' \t\n`!@#$^&*()=+[{]}\\|;:\'",<>?'
 
 GREEDY_DELIMS = ' =\r\n'
 
 
 class CompletionSplitter(object):
     """An object to split an input line in a manner similar to readline.
 
     By having our own implementation, we can expose readline-like completion in
     a uniform manner to all frontends.  This object only needs to be given the
     line of text to be split and the cursor position on said line, and it
     returns the 'word' to be completed on at the cursor after splitting the
     entire line.
 
     What characters are used as splitting delimiters can be controlled by
     setting the ``delims`` attribute (this is a property that internally
     automatically builds the necessary regular expression)"""
 
     # Private interface
 
     # A string of delimiter characters.  The default value makes sense for
     # IPython's most typical usage patterns.
     _delims = DELIMS
 
     # The expression (a normal string) to be compiled into a regular expression
     # for actual splitting.  We store it as an attribute mostly for ease of
     # debugging, since this type of code can be so tricky to debug.
     _delim_expr = None
 
     # The regular expression that does the actual splitting
     _delim_re = None
 
     def __init__(self, delims=None):
         delims = CompletionSplitter._delims if delims is None else delims
         self.delims = delims
 
     @property
     def delims(self):
         """Return the string of delimiter characters."""
         return self._delims
 
     @delims.setter
     def delims(self, delims):
         """Set the delimiters for line splitting."""
         expr = '[' + ''.join('\\'+ c for c in delims) + ']'
         self._delim_re = re.compile(expr)
         self._delims = delims
         self._delim_expr = expr
 
     def split_line(self, line, cursor_pos=None):
         """Split a line of text with a cursor at the given position.
         """
         l = line if cursor_pos is None else line[:cursor_pos]
         return self._delim_re.split(l)[-1]
 
 
 
 class Completer(Configurable):
 
     greedy = Bool(False,
         help="""Activate greedy completion
         PENDING DEPRECATION. this is now mostly taken care of with Jedi.
 
         This will enable completion on elements of lists, results of function calls, etc.,
         but can be unsafe because the code is actually evaluated on TAB.
         """,
     ).tag(config=True)
 
     use_jedi = Bool(default_value=JEDI_INSTALLED,
                     help="Experimental: Use Jedi to generate autocompletions. "
                     "Default to True if jedi is installed.").tag(config=True)
 
     jedi_compute_type_timeout = Int(default_value=400,
         help="""Experimental: restrict time (in milliseconds) during which Jedi can compute types.
         Set to 0 to stop computing types. Non-zero value lower than 100ms may hurt
         performance by preventing jedi to build its cache.
         """).tag(config=True)
 
     debug = Bool(default_value=False,
                  help='Enable debug for the Completer. Mostly print extra '
                       'information for experimental jedi integration.')\
                       .tag(config=True)
 
     backslash_combining_completions = Bool(True,
         help="Enable unicode completions, e.g. \\alpha<tab> . "
              "Includes completion of latex commands, unicode names, and expanding "
              "unicode characters back to latex commands.").tag(config=True)
 
     def __init__(self, namespace=None, global_namespace=None, **kwargs):
         """Create a new completer for the command line.
 
         Completer(namespace=ns, global_namespace=ns2) -> completer instance.
 
         If unspecified, the default namespace where completions are performed
         is __main__ (technically, __main__.__dict__). Namespaces should be
         given as dictionaries.
 
         An optional second namespace can be given.  This allows the completer
         to handle cases where both the local and global scopes need to be
         distinguished.
         """
 
         # Don't bind to namespace quite yet, but flag whether the user wants a
         # specific namespace or to use __main__.__dict__. This will allow us
         # to bind to __main__.__dict__ at completion time, not now.
         if namespace is None:
             self.use_main_ns = True
         else:
             self.use_main_ns = False
             self.namespace = namespace
 
         # The global namespace, if given, can be bound directly
         if global_namespace is None:
             self.global_namespace = {}
         else:
             self.global_namespace = global_namespace
 
         self.custom_matchers = []
 
         super(Completer, self).__init__(**kwargs)
 
     def complete(self, text, state):
         """Return the next possible completion for 'text'.
 
         This is called successively with state == 0, 1, 2, ... until it
         returns None.  The completion should begin with 'text'.
 
         """
         if self.use_main_ns:
             self.namespace = __main__.__dict__
 
         if state == 0:
             if "." in text:
                 self.matches = self.attr_matches(text)
             else:
                 self.matches = self.global_matches(text)
         try:
             return self.matches[state]
         except IndexError:
             return None
 
     def global_matches(self, text):
         """Compute matches when text is a simple name.
 
         Return a list of all keywords, built-in functions and names currently
         defined in self.namespace or self.global_namespace that match.
 
         """
         matches = []
         match_append = matches.append
         n = len(text)
-        for lst in [keyword.kwlist,
-                    builtin_mod.__dict__.keys(),
-                    self.namespace.keys(),
-                    self.global_namespace.keys()]:
+        for lst in [
+            keyword.kwlist,
+            builtin_mod.__dict__.keys(),
+            list(self.namespace.keys()),
+            list(self.global_namespace.keys()),
+        ]:
             for word in lst:
                 if word[:n] == text and word != "__builtins__":
                     match_append(word)
 
         snake_case_re = re.compile(r"[^_]+(_[^_]+)+?\Z")
-        for lst in [self.namespace.keys(),
-                    self.global_namespace.keys()]:
-            shortened = {"_".join([sub[0] for sub in word.split('_')]) : word
-                         for word in lst if snake_case_re.match(word)}
+        for lst in [list(self.namespace.keys()), list(self.global_namespace.keys())]:
+            shortened = {
+                "_".join([sub[0] for sub in word.split("_")]): word
+                for word in lst
+                if snake_case_re.match(word)
+            }
             for word in shortened.keys():
                 if word[:n] == text and word != "__builtins__":
                     match_append(shortened[word])
         return matches
 
     def attr_matches(self, text):
         """Compute matches when text contains a dot.
 
         Assuming the text is of the form NAME.NAME....[NAME], and is
         evaluatable in self.namespace or self.global_namespace, it will be
         evaluated and its attributes (as revealed by dir()) are used as
         possible completions.  (For class instances, class members are
         also considered.)
 
         WARNING: this can still invoke arbitrary C code, if an object
         with a __getattr__ hook is evaluated.
 
         """
 
         # Another option, seems to work great. Catches things like ''.<tab>
         m = re.match(r"(\S+(\.\w+)*)\.(\w*)$", text)
 
         if m:
             expr, attr = m.group(1, 3)
         elif self.greedy:
             m2 = re.match(r"(.+)\.(\w*)$", self.line_buffer)
             if not m2:
                 return []
             expr, attr = m2.group(1,2)
         else:
             return []
 
         try:
             obj = eval(expr, self.namespace)
         except:
             try:
                 obj = eval(expr, self.global_namespace)
             except:
                 return []
 
         if self.limit_to__all__ and hasattr(obj, '__all__'):
             words = get__all__entries(obj)
         else:
             words = dir2(obj)
 
         try:
             words = generics.complete_object(obj, words)
         except TryNext:
             pass
         except AssertionError:
             raise
         except Exception:
             # Silence errors from completion function
             #raise # dbg
             pass
         # Build match list to return
         n = len(attr)
         return [u"%s.%s" % (expr, w) for w in words if w[:n] == attr ]
 
 
 def get__all__entries(obj):
     """returns the strings in the __all__ attribute"""
     try:
         words = getattr(obj, '__all__')
     except:
         return []
 
     return [w for w in words if isinstance(w, str)]
 
 
 def match_dict_keys(keys: List[Union[str, bytes, Tuple[Union[str, bytes]]]], prefix: str, delims: str,
                     extra_prefix: Optional[Tuple[str, bytes]]=None) -> Tuple[str, int, List[str]]:
     """Used by dict_key_matches, matching the prefix to a list of keys
 
     Parameters
     ----------
     keys
         list of keys in dictionary currently being completed.
     prefix
         Part of the text already typed by the user. E.g. `mydict[b'fo`
     delims
         String of delimiters to consider when finding the current key.
     extra_prefix : optional
         Part of the text already typed in multi-key index cases. E.g. for
         `mydict['foo', "bar", 'b`, this would be `('foo', 'bar')`.
 
     Returns
     -------
     A tuple of three elements: ``quote``, ``token_start``, ``matched``, with
     ``quote`` being the quote that need to be used to close current string.
     ``token_start`` the position where the replacement should start occurring,
     ``matches`` a list of replacement/completion
 
     """
     prefix_tuple = extra_prefix if extra_prefix else ()
     Nprefix = len(prefix_tuple)
     def filter_prefix_tuple(key):
         # Reject too short keys
         if len(key) <= Nprefix:
             return False
         # Reject keys with non str/bytes in it
         for k in key:
             if not isinstance(k, (str, bytes)):
                 return False
         # Reject keys that do not match the prefix
         for k, pt in zip(key, prefix_tuple):
             if k != pt:
                 return False
         # All checks passed!
         return True
 
     filtered_keys:List[Union[str,bytes]] = []
     def _add_to_filtered_keys(key):
         if isinstance(key, (str, bytes)):
             filtered_keys.append(key)
 
     for k in keys:
         if isinstance(k, tuple):
             if filter_prefix_tuple(k):
                 _add_to_filtered_keys(k[Nprefix])
         else:
             _add_to_filtered_keys(k)
 
     if not prefix:
         return '', 0, [repr(k) for k in filtered_keys]
     quote_match = re.search('["\']', prefix)
     assert quote_match is not None # silence mypy
     quote = quote_match.group()
     try:
         prefix_str = eval(prefix + quote, {})
     except Exception:
         return '', 0, []
 
     pattern = '[^' + ''.join('\\' + c for c in delims) + ']*$'
     token_match = re.search(pattern, prefix, re.UNICODE)
     assert token_match is not None # silence mypy
     token_start = token_match.start()
     token_prefix = token_match.group()
 
     matched:List[str] = []
     for key in filtered_keys:
         try:
             if not key.startswith(prefix_str):
                 continue
         except (AttributeError, TypeError, UnicodeError):
             # Python 3+ TypeError on b'a'.startswith('a') or vice-versa
             continue
 
         # reformat remainder of key to begin with prefix
         rem = key[len(prefix_str):]
         # force repr wrapped in '
         rem_repr = repr(rem + '"') if isinstance(rem, str) else repr(rem + b'"')
         rem_repr = rem_repr[1 + rem_repr.index("'"):-2]
         if quote == '"':
             # The entered prefix is quoted with ",
             # but the match is quoted with '.
             # A contained " hence needs escaping for comparison:
             rem_repr = rem_repr.replace('"', '\\"')
 
         # then reinsert prefix from start of token
         matched.append('%s%s' % (token_prefix, rem_repr))
     return quote, token_start, matched
 
 
 def cursor_to_position(text:str, line:int, column:int)->int:
     """
     Convert the (line,column) position of the cursor in text to an offset in a
     string.
 
     Parameters
     ----------
     text : str
         The text in which to calculate the cursor offset
     line : int
         Line of the cursor; 0-indexed
     column : int
         Column of the cursor 0-indexed
 
     Returns
     -------
     Position of the cursor in ``text``, 0-indexed.
 
     See Also
     --------
     position_to_cursor : reciprocal of this function
 
     """
     lines = text.split('\n')
     assert line <= len(lines), '{} <= {}'.format(str(line), str(len(lines)))
 
     return sum(len(l) + 1 for l in lines[:line]) + column
 
 def position_to_cursor(text:str, offset:int)->Tuple[int, int]:
     """
     Convert the position of the cursor in text (0 indexed) to a line
     number(0-indexed) and a column number (0-indexed) pair
 
     Position should be a valid position in ``text``.
 
     Parameters
     ----------
     text : str
         The text in which to calculate the cursor offset
     offset : int
         Position of the cursor in ``text``, 0-indexed.
 
     Returns
     -------
     (line, column) : (int, int)
         Line of the cursor; 0-indexed, column of the cursor 0-indexed
 
     See Also
     --------
     cursor_to_position : reciprocal of this function
 
     """
 
     assert 0 <= offset <= len(text) , "0 <= %s <= %s" % (offset , len(text))
 
     before = text[:offset]
     blines = before.split('\n')  # ! splitnes trim trailing \n
     line = before.count('\n')
     col = len(blines[-1])
     return line, col
 
 
 def _safe_isinstance(obj, module, class_name):
     """Checks if obj is an instance of module.class_name if loaded
     """
     return (module in sys.modules and
             isinstance(obj, getattr(import_module(module), class_name)))
 
 def back_unicode_name_matches(text:str) -> Tuple[str, Sequence[str]]:
     """Match Unicode characters back to Unicode name
 
     This does  ``☃`` -> ``\\snowman``
 
     Note that snowman is not a valid python3 combining character but will be expanded.
     Though it will not recombine back to the snowman character by the completion machinery.
 
     This will not either back-complete standard sequences like \\n, \\b ...
 
     Returns
     =======
 
     Return a tuple with two elements:
 
     - The Unicode character that was matched (preceded with a backslash), or
         empty string,
     - a sequence (of 1), name for the match Unicode character, preceded by
         backslash, or empty if no match.
     
     """
     if len(text)<2:
         return '', ()
     maybe_slash = text[-2]
     if maybe_slash != '\\':
         return '', ()
 
     char = text[-1]
     # no expand on quote for completion in strings.
     # nor backcomplete standard ascii keys
     if char in string.ascii_letters or char in ('"',"'"):
         return '', ()
     try :
         unic = unicodedata.name(char)
         return '\\'+char,('\\'+unic,)
     except KeyError:
         pass
     return '', ()
 
 def back_latex_name_matches(text:str) -> Tuple[str, Sequence[str]] :
     """Match latex characters back to unicode name
 
     This does ``\\ℵ`` -> ``\\aleph``
 
     """
     if len(text)<2:
         return '', ()
     maybe_slash = text[-2]
     if maybe_slash != '\\':
         return '', ()
 
 
     char = text[-1]
     # no expand on quote for completion in strings.
     # nor backcomplete standard ascii keys
     if char in string.ascii_letters or char in ('"',"'"):
         return '', ()
     try :
         latex = reverse_latex_symbol[char]
         # '\\' replace the \ as well
         return '\\'+char,[latex]
     except KeyError:
         pass
     return '', ()
 
 
 def _formatparamchildren(parameter) -> str:
     """
     Get parameter name and value from Jedi Private API
 
     Jedi does not expose a simple way to get `param=value` from its API.
 
     Parameters
     ----------
     parameter
         Jedi's function `Param`
 
     Returns
     -------
     A string like 'a', 'b=1', '*args', '**kwargs'
 
     """
     description = parameter.description
     if not description.startswith('param '):
         raise ValueError('Jedi function parameter description have change format.'
                          'Expected "param ...", found %r".' % description)
     return description[6:]
 
 def _make_signature(completion)-> str:
     """
     Make the signature from a jedi completion
 
     Parameters
     ----------
     completion : jedi.Completion
         object does not complete a function type
 
     Returns
     -------
     a string consisting of the function signature, with the parenthesis but
     without the function name. example:
     `(a, *args, b=1, **kwargs)`
 
     """
 
     # it looks like this might work on jedi 0.17
     if hasattr(completion, 'get_signatures'):
         signatures = completion.get_signatures()
         if not signatures:
             return  '(?)'
 
         c0 = completion.get_signatures()[0]
         return '('+c0.to_string().split('(', maxsplit=1)[1]
 
     return '(%s)'% ', '.join([f for f in (_formatparamchildren(p) for signature in completion.get_signatures()
                                           for p in signature.defined_names()) if f])
 
 
 class _CompleteResult(NamedTuple):
     matched_text : str
     matches: Sequence[str]
     matches_origin: Sequence[str]
     jedi_matches: Any
 
 
 class IPCompleter(Completer):
     """Extension of the completer class with IPython-specific features"""
 
     __dict_key_regexps: Optional[Dict[bool,Pattern]] = None
 
     @observe('greedy')
     def _greedy_changed(self, change):
         """update the splitter and readline delims when greedy is changed"""
         if change['new']:
             self.splitter.delims = GREEDY_DELIMS
         else:
             self.splitter.delims = DELIMS
 
     dict_keys_only = Bool(False,
         help="""Whether to show dict key matches only""")
 
     merge_completions = Bool(True,
         help="""Whether to merge completion results into a single list
 
         If False, only the completion results from the first non-empty
         completer will be returned.
         """
     ).tag(config=True)
     omit__names = Enum((0,1,2), default_value=2,
         help="""Instruct the completer to omit private method names
 
         Specifically, when completing on ``object.<tab>``.
 
         When 2 [default]: all names that start with '_' will be excluded.
 
         When 1: all 'magic' names (``__foo__``) will be excluded.
 
         When 0: nothing will be excluded.
         """
     ).tag(config=True)
     limit_to__all__ = Bool(False,
         help="""
         DEPRECATED as of version 5.0.
 
         Instruct the completer to use __all__ for the completion
 
         Specifically, when completing on ``object.<tab>``.
 
         When True: only those names in obj.__all__ will be included.
 
         When False [default]: the __all__ attribute is ignored
         """,
     ).tag(config=True)
 
     profile_completions = Bool(
         default_value=False,
         help="If True, emit profiling data for completion subsystem using cProfile."
     ).tag(config=True)
 
     profiler_output_dir = Unicode(
         default_value=".completion_profiles",
         help="Template for path at which to output profile data for completions."
     ).tag(config=True)
 
     @observe('limit_to__all__')
     def _limit_to_all_changed(self, change):
         warnings.warn('`IPython.core.IPCompleter.limit_to__all__` configuration '
             'value has been deprecated since IPython 5.0, will be made to have '
             'no effects and then removed in future version of IPython.',
             UserWarning)
 
     def __init__(
         self, shell=None, namespace=None, global_namespace=None, config=None, **kwargs
     ):
         """IPCompleter() -> completer
 
         Return a completer object.
 
         Parameters
         ----------
         shell
             a pointer to the ipython shell itself.  This is needed
             because this completer knows about magic functions, and those can
             only be accessed via the ipython instance.
         namespace : dict, optional
             an optional dict where completions are performed.
         global_namespace : dict, optional
             secondary optional dict for completions, to
             handle cases (such as IPython embedded inside functions) where
             both Python scopes are visible.
         config : Config
             traitlet's config object
         **kwargs
             passed to super class unmodified.
         """
 
         self.magic_escape = ESC_MAGIC
         self.splitter = CompletionSplitter()
 
         # _greedy_changed() depends on splitter and readline being defined:
         super().__init__(
             namespace=namespace,
             global_namespace=global_namespace,
             config=config,
             **kwargs
         )
 
         # List where completion matches will be stored
         self.matches = []
         self.shell = shell
         # Regexp to split filenames with spaces in them
         self.space_name_re = re.compile(r'([^\\] )')
         # Hold a local ref. to glob.glob for speed
         self.glob = glob.glob
 
         # Determine if we are running on 'dumb' terminals, like (X)Emacs
         # buffers, to avoid completion problems.
         term = os.environ.get('TERM','xterm')
         self.dumb_terminal = term in ['dumb','emacs']
 
         # Special handling of backslashes needed in win32 platforms
         if sys.platform == "win32":
             self.clean_glob = self._clean_glob_win32
         else:
             self.clean_glob = self._clean_glob
 
         #regexp to parse docstring for function signature
         self.docstring_sig_re = re.compile(r'^[\w|\s.]+\(([^)]*)\).*')
         self.docstring_kwd_re = re.compile(r'[\s|\[]*(\w+)(?:\s*=\s*.*)')
         #use this if positional argument name is also needed
         #= re.compile(r'[\s|\[]*(\w+)(?:\s*=?\s*.*)')
 
         self.magic_arg_matchers = [
             self.magic_config_matches,
             self.magic_color_matches,
         ]
 
         # This is set externally by InteractiveShell
         self.custom_completers = None
 
         # This is a list of names of unicode characters that can be completed
         # into their corresponding unicode value. The list is large, so we
         # lazily initialize it on first use. Consuming code should access this
         # attribute through the `@unicode_names` property.
         self._unicode_names = None
 
     @property
     def matchers(self) -> List[Any]:
         """All active matcher routines for completion"""
         if self.dict_keys_only:
             return [self.dict_key_matches]
 
         if self.use_jedi:
             return [
                 *self.custom_matchers,
                 self.dict_key_matches,
                 self.file_matches,
                 self.magic_matches,
             ]
         else:
             return [
                 *self.custom_matchers,
                 self.dict_key_matches,
                 self.python_matches,
                 self.file_matches,
                 self.magic_matches,
                 self.python_func_kw_matches,
             ]
 
     def all_completions(self, text:str) -> List[str]:
         """
         Wrapper around the completion methods for the benefit of emacs.
         """
         prefix = text.rpartition('.')[0]
         with provisionalcompleter():
             return ['.'.join([prefix, c.text]) if prefix and self.use_jedi else c.text
                     for c in self.completions(text, len(text))]
 
         return self.complete(text)[1]
 
     def _clean_glob(self, text:str):
         return self.glob("%s*" % text)
 
     def _clean_glob_win32(self, text:str):
         return [f.replace("\\","/")
                 for f in self.glob("%s*" % text)]
 
     def file_matches(self, text:str)->List[str]:
         """Match filenames, expanding ~USER type strings.
 
         Most of the seemingly convoluted logic in this completer is an
         attempt to handle filenames with spaces in them.  And yet it's not
         quite perfect, because Python's readline doesn't expose all of the
         GNU readline details needed for this to be done correctly.
 
         For a filename with a space in it, the printed completions will be
         only the parts after what's already been typed (instead of the
         full completions, as is normally done).  I don't think with the
         current (as of Python 2.3) Python readline it's possible to do
         better."""
 
         # chars that require escaping with backslash - i.e. chars
         # that readline treats incorrectly as delimiters, but we
         # don't want to treat as delimiters in filename matching
         # when escaped with backslash
         if text.startswith('!'):
             text = text[1:]
             text_prefix = u'!'
         else:
             text_prefix = u''
 
         text_until_cursor = self.text_until_cursor
         # track strings with open quotes
         open_quotes = has_open_quotes(text_until_cursor)
 
         if '(' in text_until_cursor or '[' in text_until_cursor:
             lsplit = text
         else:
             try:
                 # arg_split ~ shlex.split, but with unicode bugs fixed by us
                 lsplit = arg_split(text_until_cursor)[-1]
             except ValueError:
                 # typically an unmatched ", or backslash without escaped char.
                 if open_quotes:
                     lsplit = text_until_cursor.split(open_quotes)[-1]
                 else:
                     return []
             except IndexError:
                 # tab pressed on empty line
                 lsplit = ""
 
         if not open_quotes and lsplit != protect_filename(lsplit):
             # if protectables are found, do matching on the whole escaped name
             has_protectables = True
             text0,text = text,lsplit
         else:
             has_protectables = False
             text = os.path.expanduser(text)
 
         if text == "":
             return [text_prefix + protect_filename(f) for f in self.glob("*")]
 
         # Compute the matches from the filesystem
         if sys.platform == 'win32':
             m0 = self.clean_glob(text)
         else:
             m0 = self.clean_glob(text.replace('\\', ''))
 
         if has_protectables:
             # If we had protectables, we need to revert our changes to the
             # beginning of filename so that we don't double-write the part
             # of the filename we have so far
             len_lsplit = len(lsplit)
             matches = [text_prefix + text0 +
                        protect_filename(f[len_lsplit:]) for f in m0]
         else:
             if open_quotes:
                 # if we have a string with an open quote, we don't need to
                 # protect the names beyond the quote (and we _shouldn't_, as
                 # it would cause bugs when the filesystem call is made).
                 matches = m0 if sys.platform == "win32" else\
                     [protect_filename(f, open_quotes) for f in m0]
             else:
                 matches = [text_prefix +
                            protect_filename(f) for f in m0]
 
         # Mark directories in input list by appending '/' to their names.
         return [x+'/' if os.path.isdir(x) else x for x in matches]
 
     def magic_matches(self, text:str):
         """Match magics"""
         # Get all shell magics now rather than statically, so magics loaded at
         # runtime show up too.
         lsm = self.shell.magics_manager.lsmagic()
         line_magics = lsm['line']
         cell_magics = lsm['cell']
         pre = self.magic_escape
         pre2 = pre+pre
 
         explicit_magic = text.startswith(pre)
 
         # Completion logic:
         # - user gives %%: only do cell magics
         # - user gives %: do both line and cell magics
         # - no prefix: do both
         # In other words, line magics are skipped if the user gives %% explicitly
         #
         # We also exclude magics that match any currently visible names:
         # https://github.com/ipython/ipython/issues/4877, unless the user has
         # typed a %:
         # https://github.com/ipython/ipython/issues/10754
         bare_text = text.lstrip(pre)
         global_matches = self.global_matches(bare_text)
         if not explicit_magic:
             def matches(magic):
                 """
                 Filter magics, in particular remove magics that match
                 a name present in global namespace.
                 """
                 return ( magic.startswith(bare_text) and
                          magic not in global_matches )
         else:
             def matches(magic):
                 return magic.startswith(bare_text)
 
         comp = [ pre2+m for m in cell_magics if matches(m)]
         if not text.startswith(pre2):
             comp += [ pre+m for m in line_magics if matches(m)]
 
         return comp
 
     def magic_config_matches(self, text:str) -> List[str]:
         """ Match class names and attributes for %config magic """
         texts = text.strip().split()
 
         if len(texts) > 0 and (texts[0] == 'config' or texts[0] == '%config'):
             # get all configuration classes
             classes = sorted(set([ c for c in self.shell.configurables
                                    if c.__class__.class_traits(config=True)
                                    ]), key=lambda x: x.__class__.__name__)
             classnames = [ c.__class__.__name__ for c in classes ]
 
             # return all classnames if config or %config is given
             if len(texts) == 1:
                 return classnames
 
             # match classname
             classname_texts = texts[1].split('.')
             classname = classname_texts[0]
             classname_matches = [ c for c in classnames
                                   if c.startswith(classname) ]
 
             # return matched classes or the matched class with attributes
             if texts[1].find('.') < 0:
                 return classname_matches
             elif len(classname_matches) == 1 and \
                             classname_matches[0] == classname:
                 cls = classes[classnames.index(classname)].__class__
                 help = cls.class_get_help()
                 # strip leading '--' from cl-args:
                 help = re.sub(re.compile(r'^--', re.MULTILINE), '', help)
                 return [ attr.split('=')[0]
                          for attr in help.strip().splitlines()
                          if attr.startswith(texts[1]) ]
         return []
 
     def magic_color_matches(self, text:str) -> List[str] :
         """ Match color schemes for %colors magic"""
         texts = text.split()
         if text.endswith(' '):
             # .split() strips off the trailing whitespace. Add '' back
             # so that: '%colors ' -> ['%colors', '']
             texts.append('')
 
         if len(texts) == 2 and (texts[0] == 'colors' or texts[0] == '%colors'):
             prefix = texts[1]
             return [ color for color in InspectColors.keys()
                      if color.startswith(prefix) ]
         return []
 
     def _jedi_matches(self, cursor_column:int, cursor_line:int, text:str) -> Iterable[Any]:
         """
         Return a list of :any:`jedi.api.Completions` object from a ``text`` and
         cursor position.
 
         Parameters
         ----------
         cursor_column : int
             column position of the cursor in ``text``, 0-indexed.
         cursor_line : int
             line position of the cursor in ``text``, 0-indexed
         text : str
             text to complete
 
         Notes
         -----
         If ``IPCompleter.debug`` is ``True`` may return a :any:`_FakeJediCompletion`
         object containing a string with the Jedi debug information attached.
         """
         namespaces = [self.namespace]
         if self.global_namespace is not None:
             namespaces.append(self.global_namespace)
 
         completion_filter = lambda x:x
         offset = cursor_to_position(text, cursor_line, cursor_column)
         # filter output if we are completing for object members
         if offset:
             pre = text[offset-1]
             if pre == '.':
                 if self.omit__names == 2:
                     completion_filter = lambda c:not c.name.startswith('_')
                 elif self.omit__names == 1:
                     completion_filter = lambda c:not (c.name.startswith('__') and c.name.endswith('__'))
                 elif self.omit__names == 0:
                     completion_filter = lambda x:x
                 else:
                     raise ValueError("Don't understand self.omit__names == {}".format(self.omit__names))
 
         interpreter = jedi.Interpreter(text[:offset], namespaces)
         try_jedi = True
 
         try:
             # find the first token in the current tree -- if it is a ' or " then we are in a string
             completing_string = False
             try:
                 first_child = next(c for c in interpreter._get_module().tree_node.children if hasattr(c, 'value'))
             except StopIteration:
                 pass
             else:
                 # note the value may be ', ", or it may also be ''' or """, or
                 # in some cases, """what/you/typed..., but all of these are
                 # strings.
                 completing_string = len(first_child.value) > 0 and first_child.value[0] in {"'", '"'}
 
             # if we are in a string jedi is likely not the right candidate for
             # now. Skip it.
             try_jedi = not completing_string
         except Exception as e:
             # many of things can go wrong, we are using private API just don't crash.
             if self.debug:
                 print("Error detecting if completing a non-finished string :", e, '|')
 
         if not try_jedi:
             return []
         try:
             return filter(completion_filter, interpreter.complete(column=cursor_column, line=cursor_line + 1))
         except Exception as e:
             if self.debug:
                 return [_FakeJediCompletion('Oops Jedi has crashed, please report a bug with the following:\n"""\n%s\ns"""' % (e))]
             else:
                 return []
 
     def python_matches(self, text:str)->List[str]:
         """Match attributes or global python names"""
         if "." in text:
             try:
                 matches = self.attr_matches(text)
                 if text.endswith('.') and self.omit__names:
                     if self.omit__names == 1:
                         # true if txt is _not_ a __ name, false otherwise:
                         no__name = (lambda txt:
                                     re.match(r'.*\.__.*?__',txt) is None)
                     else:
                         # true if txt is _not_ a _ name, false otherwise:
                         no__name = (lambda txt:
                                     re.match(r'\._.*?',txt[txt.rindex('.'):]) is None)
                     matches = filter(no__name, matches)
             except NameError:
                 # catches <undefined attributes>.<tab>
                 matches = []
         else:
             matches = self.global_matches(text)
         return matches
 
     def _default_arguments_from_docstring(self, doc):
         """Parse the first line of docstring for call signature.
 
         Docstring should be of the form 'min(iterable[, key=func])\n'.
         It can also parse cython docstring of the form
         'Minuit.migrad(self, int ncall=10000, resume=True, int nsplit=1)'.
         """
         if doc is None:
             return []
 
         #care only the firstline
         line = doc.lstrip().splitlines()[0]
 
         #p = re.compile(r'^[\w|\s.]+\(([^)]*)\).*')
         #'min(iterable[, key=func])\n' -> 'iterable[, key=func]'
         sig = self.docstring_sig_re.search(line)
         if sig is None:
             return []
         # iterable[, key=func]' -> ['iterable[' ,' key=func]']
         sig = sig.groups()[0].split(',')
         ret = []
         for s in sig:
             #re.compile(r'[\s|\[]*(\w+)(?:\s*=\s*.*)')
             ret += self.docstring_kwd_re.findall(s)
         return ret
 
     def _default_arguments(self, obj):
         """Return the list of default arguments of obj if it is callable,
         or empty list otherwise."""
         call_obj = obj
         ret = []
         if inspect.isbuiltin(obj):
             pass
         elif not (inspect.isfunction(obj) or inspect.ismethod(obj)):
             if inspect.isclass(obj):
                 #for cython embedsignature=True the constructor docstring
                 #belongs to the object itself not __init__
                 ret += self._default_arguments_from_docstring(
                             getattr(obj, '__doc__', ''))
                 # for classes, check for __init__,__new__
                 call_obj = (getattr(obj, '__init__', None) or
                        getattr(obj, '__new__', None))
             # for all others, check if they are __call__able
             elif hasattr(obj, '__call__'):
                 call_obj = obj.__call__
         ret += self._default_arguments_from_docstring(
                  getattr(call_obj, '__doc__', ''))
 
         _keeps = (inspect.Parameter.KEYWORD_ONLY,
                   inspect.Parameter.POSITIONAL_OR_KEYWORD)
 
         try:
             sig = inspect.signature(obj)
             ret.extend(k for k, v in sig.parameters.items() if
                        v.kind in _keeps)
         except ValueError:
             pass
 
         return list(set(ret))
 
     def python_func_kw_matches(self, text):
         """Match named parameters (kwargs) of the last open function"""
 
         if "." in text: # a parameter cannot be dotted
             return []
         try: regexp = self.__funcParamsRegex
         except AttributeError:
             regexp = self.__funcParamsRegex = re.compile(r'''
                 '.*?(?<!\\)' |    # single quoted strings or
                 ".*?(?<!\\)" |    # double quoted strings or
                 \w+          |    # identifier
                 \S                # other characters
                 ''', re.VERBOSE | re.DOTALL)
         # 1. find the nearest identifier that comes before an unclosed
         # parenthesis before the cursor
         # e.g. for "foo (1+bar(x), pa<cursor>,a=1)", the candidate is "foo"
         tokens = regexp.findall(self.text_until_cursor)
         iterTokens = reversed(tokens); openPar = 0
 
         for token in iterTokens:
             if token == ')':
                 openPar -= 1
             elif token == '(':
                 openPar += 1
                 if openPar > 0:
                     # found the last unclosed parenthesis
                     break
         else:
             return []
         # 2. Concatenate dotted names ("foo.bar" for "foo.bar(x, pa" )
         ids = []
         isId = re.compile(r'\w+$').match
 
         while True:
             try:
                 ids.append(next(iterTokens))
                 if not isId(ids[-1]):
                     ids.pop(); break
                 if not next(iterTokens) == '.':
                     break
             except StopIteration:
                 break
 
         # Find all named arguments already assigned to, as to avoid suggesting
         # them again
         usedNamedArgs = set()
         par_level = -1
         for token, next_token in zip(tokens, tokens[1:]):
             if token == '(':
                 par_level += 1
             elif token == ')':
                 par_level -= 1
 
             if par_level != 0:
                 continue
 
             if next_token != '=':
                 continue
 
             usedNamedArgs.add(token)
 
         argMatches = []
         try:
             callableObj = '.'.join(ids[::-1])
             namedArgs = self._default_arguments(eval(callableObj,
                                                     self.namespace))
 
             # Remove used named arguments from the list, no need to show twice
             for namedArg in set(namedArgs) - usedNamedArgs:
                 if namedArg.startswith(text):
                     argMatches.append("%s=" %namedArg)
         except:
             pass
 
         return argMatches
 
     @staticmethod
     def _get_keys(obj: Any) -> List[Any]:
         # Objects can define their own completions by defining an
         # _ipy_key_completions_() method.
         method = get_real_method(obj, '_ipython_key_completions_')
         if method is not None:
             return method()
 
         # Special case some common in-memory dict-like types
         if isinstance(obj, dict) or\
            _safe_isinstance(obj, 'pandas', 'DataFrame'):
             try:
                 return list(obj.keys())
             except Exception:
                 return []
         elif _safe_isinstance(obj, 'numpy', 'ndarray') or\
              _safe_isinstance(obj, 'numpy', 'void'):
             return obj.dtype.names or []
         return []
 
     def dict_key_matches(self, text:str) -> List[str]:
         "Match string keys in a dictionary, after e.g. 'foo[' "
 
 
         if self.__dict_key_regexps is not None:
             regexps = self.__dict_key_regexps
         else:
             dict_key_re_fmt = r'''(?x)
             (  # match dict-referring expression wrt greedy setting
                 %s
             )
             \[   # open bracket
             \s*  # and optional whitespace
             # Capture any number of str-like objects (e.g. "a", "b", 'c')
             ((?:[uUbB]?  # string prefix (r not handled)
                 (?:
                     '(?:[^']|(?<!\\)\\')*'
                 |
                     "(?:[^"]|(?<!\\)\\")*"
                 )
                 \s*,\s*
             )*)
             ([uUbB]?  # string prefix (r not handled)
                 (?:   # unclosed string
                     '(?:[^']|(?<!\\)\\')*
                 |
                     "(?:[^"]|(?<!\\)\\")*
                 )
             )?
             $
             '''
             regexps = self.__dict_key_regexps = {
                 False: re.compile(dict_key_re_fmt % r'''
                                   # identifiers separated by .
                                   (?!\d)\w+
                                   (?:\.(?!\d)\w+)*
                                   '''),
                 True: re.compile(dict_key_re_fmt % '''
                                  .+
                                  ''')
             }
 
         match = regexps[self.greedy].search(self.text_until_cursor)
 
         if match is None:
             return []
 
         expr, prefix0, prefix = match.groups()
         try:
             obj = eval(expr, self.namespace)
         except Exception:
             try:
                 obj = eval(expr, self.global_namespace)
             except Exception:
                 return []
 
         keys = self._get_keys(obj)
         if not keys:
             return keys
 
         extra_prefix = eval(prefix0) if prefix0 != '' else None
 
         closing_quote, token_offset, matches = match_dict_keys(keys, prefix, self.splitter.delims, extra_prefix=extra_prefix)
         if not matches:
             return matches
 
         # get the cursor position of
         # - the text being completed
         # - the start of the key text
         # - the start of the completion
         text_start = len(self.text_until_cursor) - len(text)
         if prefix:
             key_start = match.start(3)
             completion_start = key_start + token_offset
         else:
             key_start = completion_start = match.end()
 
         # grab the leading prefix, to make sure all completions start with `text`
         if text_start > key_start:
             leading = ''
         else:
             leading = text[text_start:completion_start]
 
         # the index of the `[` character
         bracket_idx = match.end(1)
 
         # append closing quote and bracket as appropriate
         # this is *not* appropriate if the opening quote or bracket is outside
         # the text given to this method
         suf = ''
         continuation = self.line_buffer[len(self.text_until_cursor):]
         if key_start > text_start and closing_quote:
             # quotes were opened inside text, maybe close them
             if continuation.startswith(closing_quote):
                 continuation = continuation[len(closing_quote):]
             else:
                 suf += closing_quote
         if bracket_idx > text_start:
             # brackets were opened inside text, maybe close them
             if not continuation.startswith(']'):
                 suf += ']'
 
         return [leading + k + suf for k in matches]
 
     @staticmethod
     def unicode_name_matches(text:str) -> Tuple[str, List[str]] :
         """Match Latex-like syntax for unicode characters base
         on the name of the character.
 
         This does  ``\\GREEK SMALL LETTER ETA`` -> ``η``
 
         Works only on valid python 3 identifier, or on combining characters that
         will combine to form a valid identifier.
         """
         slashpos = text.rfind('\\')
         if slashpos > -1:
             s = text[slashpos+1:]
             try :
                 unic = unicodedata.lookup(s)
                 # allow combining chars
                 if ('a'+unic).isidentifier():
                     return '\\'+s,[unic]
             except KeyError:
                 pass
         return '', []
 
 
     def latex_matches(self, text:str) -> Tuple[str, Sequence[str]]:
         """Match Latex syntax for unicode characters.
 
         This does both ``\\alp`` -> ``\\alpha`` and ``\\alpha`` -> ``α``
         """
         slashpos = text.rfind('\\')
         if slashpos > -1:
             s = text[slashpos:]
             if s in latex_symbols:
                 # Try to complete a full latex symbol to unicode
                 # \\alpha -> α
                 return s, [latex_symbols[s]]
             else:
                 # If a user has partially typed a latex symbol, give them
                 # a full list of options \al -> [\aleph, \alpha]
                 matches = [k for k in latex_symbols if k.startswith(s)]
                 if matches:
                     return s, matches
         return '', ()
 
     def dispatch_custom_completer(self, text):
         if not self.custom_completers:
             return
 
         line = self.line_buffer
         if not line.strip():
             return None
 
         # Create a little structure to pass all the relevant information about
         # the current completion to any custom completer.
         event = SimpleNamespace()
         event.line = line
         event.symbol = text
         cmd = line.split(None,1)[0]
         event.command = cmd
         event.text_until_cursor = self.text_until_cursor
 
         # for foo etc, try also to find completer for %foo
         if not cmd.startswith(self.magic_escape):
             try_magic = self.custom_completers.s_matches(
                 self.magic_escape + cmd)
         else:
             try_magic = []
 
         for c in itertools.chain(self.custom_completers.s_matches(cmd),
                  try_magic,
                  self.custom_completers.flat_matches(self.text_until_cursor)):
             try:
                 res = c(event)
                 if res:
                     # first, try case sensitive match
                     withcase = [r for r in res if r.startswith(text)]
                     if withcase:
                         return withcase
                     # if none, then case insensitive ones are ok too
                     text_low = text.lower()
                     return [r for r in res if r.lower().startswith(text_low)]
             except TryNext:
                 pass
             except KeyboardInterrupt:
                 """
                 If custom completer take too long,
                 let keyboard interrupt abort and return nothing.
                 """
                 break
 
         return None
 
     def completions(self, text: str, offset: int)->Iterator[Completion]:
         """
         Returns an iterator over the possible completions
 
         .. warning::
 
             Unstable
 
             This function is unstable, API may change without warning.
             It will also raise unless use in proper context manager.
 
         Parameters
         ----------
         text : str
             Full text of the current input, multi line string.
         offset : int
             Integer representing the position of the cursor in ``text``. Offset
             is 0-based indexed.
 
         Yields
         ------
         Completion
 
         Notes
         -----
         The cursor on a text can either be seen as being "in between"
         characters or "On" a character depending on the interface visible to
         the user. For consistency the cursor being on "in between" characters X
         and Y is equivalent to the cursor being "on" character Y, that is to say
         the character the cursor is on is considered as being after the cursor.
 
         Combining characters may span more that one position in the
         text.
 
         .. note::
 
             If ``IPCompleter.debug`` is :any:`True` will yield a ``--jedi/ipython--``
             fake Completion token to distinguish completion returned by Jedi
             and usual IPython completion.
 
         .. note::
 
             Completions are not completely deduplicated yet. If identical
             completions are coming from different sources this function does not
             ensure that each completion object will only be present once.
         """
         warnings.warn("_complete is a provisional API (as of IPython 6.0). "
                       "It may change without warnings. "
                       "Use in corresponding context manager.",
                       category=ProvisionalCompleterWarning, stacklevel=2)
 
         seen = set()
         profiler:Optional[cProfile.Profile]
         try:
             if self.profile_completions:
                 import cProfile
                 profiler = cProfile.Profile()
                 profiler.enable()
             else:
                 profiler = None
 
             for c in self._completions(text, offset, _timeout=self.jedi_compute_type_timeout/1000):
                 if c and (c in seen):
                     continue
                 yield c
                 seen.add(c)
         except KeyboardInterrupt:
             """if completions take too long and users send keyboard interrupt,
             do not crash and return ASAP. """
             pass
         finally:
             if profiler is not None:
                 profiler.disable()
                 ensure_dir_exists(self.profiler_output_dir)
                 output_path = os.path.join(self.profiler_output_dir, str(uuid.uuid4()))
                 print("Writing profiler output to", output_path)
                 profiler.dump_stats(output_path)
 
     def _completions(self, full_text: str, offset: int, *, _timeout) -> Iterator[Completion]:
         """
         Core completion module.Same signature as :any:`completions`, with the
         extra `timeout` parameter (in seconds).
 
         Computing jedi's completion ``.type`` can be quite expensive (it is a
         lazy property) and can require some warm-up, more warm up than just
         computing the ``name`` of a completion. The warm-up can be :
 
             - Long warm-up the first time a module is encountered after
             install/update: actually build parse/inference tree.
 
             - first time the module is encountered in a session: load tree from
             disk.
 
         We don't want to block completions for tens of seconds so we give the
         completer a "budget" of ``_timeout`` seconds per invocation to compute
         completions types, the completions that have not yet been computed will
         be marked as "unknown" an will have a chance to be computed next round
         are things get cached.
 
         Keep in mind that Jedi is not the only thing treating the completion so
         keep the timeout short-ish as if we take more than 0.3 second we still
         have lots of processing to do.
 
         """
         deadline = time.monotonic() + _timeout
 
 
         before = full_text[:offset]
         cursor_line, cursor_column = position_to_cursor(full_text, offset)
 
         matched_text, matches, matches_origin, jedi_matches = self._complete(
             full_text=full_text, cursor_line=cursor_line, cursor_pos=cursor_column)
 
         iter_jm = iter(jedi_matches)
         if _timeout:
             for jm in iter_jm:
                 try:
                     type_ = jm.type
                 except Exception:
                     if self.debug:
                         print("Error in Jedi getting type of ", jm)
                     type_ = None
                 delta = len(jm.name_with_symbols) - len(jm.complete)
                 if type_ == 'function':
                     signature = _make_signature(jm)
                 else:
                     signature = ''
                 yield Completion(start=offset - delta,
                                  end=offset,
                                  text=jm.name_with_symbols,
                                  type=type_,
                                  signature=signature,
                                  _origin='jedi')
 
                 if time.monotonic() > deadline:
                     break
 
         for jm in iter_jm:
             delta = len(jm.name_with_symbols) - len(jm.complete)
             yield Completion(start=offset - delta,
                              end=offset,
                              text=jm.name_with_symbols,
                              type='<unknown>',  # don't compute type for speed
                              _origin='jedi',
                              signature='')
 
 
         start_offset = before.rfind(matched_text)
 
         # TODO:
         # Suppress this, right now just for debug.
         if jedi_matches and matches and self.debug:
             yield Completion(start=start_offset, end=offset, text='--jedi/ipython--',
                              _origin='debug', type='none', signature='')
 
         # I'm unsure if this is always true, so let's assert and see if it
         # crash
         assert before.endswith(matched_text)
         for m, t in zip(matches, matches_origin):
             yield Completion(start=start_offset, end=offset, text=m, _origin=t, signature='', type='<unknown>')
 
 
     def complete(self, text=None, line_buffer=None, cursor_pos=None) -> Tuple[str, Sequence[str]]:
         """Find completions for the given text and line context.
 
         Note that both the text and the line_buffer are optional, but at least
         one of them must be given.
 
         Parameters
         ----------
         text : string, optional
             Text to perform the completion on.  If not given, the line buffer
             is split using the instance's CompletionSplitter object.
         line_buffer : string, optional
             If not given, the completer attempts to obtain the current line
             buffer via readline.  This keyword allows clients which are
             requesting for text completions in non-readline contexts to inform
             the completer of the entire text.
         cursor_pos : int, optional
             Index of the cursor in the full line buffer.  Should be provided by
             remote frontends where kernel has no access to frontend state.
 
         Returns
         -------
         Tuple of two items:
         text : str
             Text that was actually used in the completion.
         matches : list
             A list of completion matches.
 
         Notes
         -----
             This API is likely to be deprecated and replaced by
             :any:`IPCompleter.completions` in the future.
 
         """
         warnings.warn('`Completer.complete` is pending deprecation since '
                 'IPython 6.0 and will be replaced by `Completer.completions`.',
                       PendingDeprecationWarning)
         # potential todo, FOLD the 3rd throw away argument of _complete
         # into the first 2 one.
         return self._complete(line_buffer=line_buffer, cursor_pos=cursor_pos, text=text, cursor_line=0)[:2]
 
     def _complete(self, *, cursor_line, cursor_pos, line_buffer=None, text=None,
                   full_text=None) -> _CompleteResult:
         """
         Like complete but can also returns raw jedi completions as well as the
         origin of the completion text. This could (and should) be made much
         cleaner but that will be simpler once we drop the old (and stateful)
         :any:`complete` API.
 
         With current provisional API, cursor_pos act both (depending on the
         caller) as the offset in the ``text`` or ``line_buffer``, or as the
         ``column`` when passing multiline strings this could/should be renamed
         but would add extra noise.
 
         Parameters
         ----------
         cursor_line
             Index of the line the cursor is on. 0 indexed.
         cursor_pos
             Position of the cursor in the current line/line_buffer/text. 0
             indexed.
         line_buffer : optional, str
             The current line the cursor is in, this is mostly due to legacy
             reason that readline could only give a us the single current line.
             Prefer `full_text`.
         text : str
             The current "token" the cursor is in, mostly also for historical
             reasons. as the completer would trigger only after the current line
             was parsed.
         full_text : str
             Full text of the current cell.
 
         Returns
         -------
         A tuple of N elements which are (likely):
             matched_text: ? the text that the complete matched
             matches: list of completions ?
             matches_origin: ? list same length as matches, and where each completion came from
             jedi_matches: list of Jedi matches, have it's own structure.
         """
 
 
         # if the cursor position isn't given, the only sane assumption we can
         # make is that it's at the end of the line (the common case)
         if cursor_pos is None:
             cursor_pos = len(line_buffer) if text is None else len(text)
 
         if self.use_main_ns:
             self.namespace = __main__.__dict__
 
         # if text is either None or an empty string, rely on the line buffer
         if (not line_buffer) and full_text:
             line_buffer = full_text.split('\n')[cursor_line]
         if not text: # issue #11508: check line_buffer before calling split_line
             text = self.splitter.split_line(line_buffer, cursor_pos)  if line_buffer else ''
 
         if self.backslash_combining_completions:
             # allow deactivation of these on windows.
             base_text = text if not line_buffer else line_buffer[:cursor_pos]
 
             for meth in (self.latex_matches,
                          self.unicode_name_matches,
                          back_latex_name_matches,
                          back_unicode_name_matches,
                          self.fwd_unicode_match):
                 name_text, name_matches = meth(base_text)
                 if name_text:
                     return _CompleteResult(name_text, name_matches[:MATCHES_LIMIT], \
                            [meth.__qualname__]*min(len(name_matches), MATCHES_LIMIT), ())
 
 
         # If no line buffer is given, assume the input text is all there was
         if line_buffer is None:
             line_buffer = text
 
         self.line_buffer = line_buffer
         self.text_until_cursor = self.line_buffer[:cursor_pos]
 
         # Do magic arg matches
         for matcher in self.magic_arg_matchers:
             matches = list(matcher(line_buffer))[:MATCHES_LIMIT]
             if matches:
                 origins = [matcher.__qualname__] * len(matches)
                 return _CompleteResult(text, matches, origins, ())
 
         # Start with a clean slate of completions
         matches = []
 
         # FIXME: we should extend our api to return a dict with completions for
         # different types of objects.  The rlcomplete() method could then
         # simply collapse the dict into a list for readline, but we'd have
         # richer completion semantics in other environments.
         is_magic_prefix = len(text) > 0 and text[0] == "%"
         completions: Iterable[Any] = []
         if self.use_jedi and not is_magic_prefix:
             if not full_text:
                 full_text = line_buffer
             completions = self._jedi_matches(
                 cursor_pos, cursor_line, full_text)
 
         if self.merge_completions:
             matches = []
             for matcher in self.matchers:
                 try:
                     matches.extend([(m, matcher.__qualname__)
                                     for m in matcher(text)])
                 except:
                     # Show the ugly traceback if the matcher causes an
                     # exception, but do NOT crash the kernel!
                     sys.excepthook(*sys.exc_info())
         else:
             for matcher in self.matchers:
                 matches = [(m, matcher.__qualname__)
                             for m in matcher(text)]
                 if matches:
                     break
                     
         seen = set()
         filtered_matches = set()
         for m in matches:
             t, c = m
             if t not in seen:
                 filtered_matches.add(m)
                 seen.add(t)
 
         _filtered_matches = sorted(filtered_matches, key=lambda x: completions_sorting_key(x[0]))
 
         custom_res = [(m, 'custom') for m in self.dispatch_custom_completer(text) or []]
         
         _filtered_matches = custom_res or _filtered_matches
         
         _filtered_matches = _filtered_matches[:MATCHES_LIMIT]
         _matches = [m[0] for m in _filtered_matches]
         origins = [m[1] for m in _filtered_matches]
 
         self.matches = _matches
 
         return _CompleteResult(text, _matches, origins, completions)
         
     def fwd_unicode_match(self, text:str) -> Tuple[str, Sequence[str]]:
         """
         Forward match a string starting with a backslash with a list of
         potential Unicode completions.
 
         Will compute list list of Unicode character names on first call and cache it.
 
         Returns
         -------
         At tuple with:
             - matched text (empty if no matches)
             - list of potential completions, empty tuple  otherwise)
         """
         # TODO: self.unicode_names is here a list we traverse each time with ~100k elements.
         # We could do a faster match using a Trie.
 
         # Using pygtrie the following seem to work:
 
         #     s = PrefixSet()
 
         #     for c in range(0,0x10FFFF + 1):
         #         try:
         #             s.add(unicodedata.name(chr(c)))
         #         except ValueError:
         #             pass
         #     [''.join(k) for k in s.iter(prefix)]
 
         # But need to be timed and adds an extra dependency.
 
         slashpos = text.rfind('\\')
         # if text starts with slash
         if slashpos > -1:
             # PERF: It's important that we don't access self._unicode_names
             # until we're inside this if-block. _unicode_names is lazily
             # initialized, and it takes a user-noticeable amount of time to
             # initialize it, so we don't want to initialize it unless we're
             # actually going to use it.
             s = text[slashpos + 1 :]
             sup = s.upper()
             candidates = [x for x in self.unicode_names if x.startswith(sup)]
             if candidates:
                 return s, candidates
             candidates = [x for x in self.unicode_names if sup in x]
             if candidates:
                 return s, candidates
             splitsup = sup.split(" ")
             candidates = [
                 x for x in self.unicode_names if all(u in x for u in splitsup)
             ]
             if candidates:
                 return s, candidates
 
             return "", ()
 
         # if text does not start with slash
         else:
             return '', ()
 
     @property
     def unicode_names(self) -> List[str]:
         """List of names of unicode code points that can be completed.
 
         The list is lazily initialized on first access.
         """
         if self._unicode_names is None:
             names = []
             for c in range(0,0x10FFFF + 1):
                 try:
                     names.append(unicodedata.name(chr(c)))
                 except ValueError:
                     pass
             self._unicode_names = _unicode_name_compute(_UNICODE_RANGES)
 
         return self._unicode_names
 
 def _unicode_name_compute(ranges:List[Tuple[int,int]]) -> List[str]:
     names = []
     for start,stop in ranges:
         for c in range(start, stop) :
             try:
                 names.append(unicodedata.name(chr(c)))
             except ValueError:
                 pass
     return names

IPython/lib/tests/test_pretty.py

@@ -1,536 +1,539 b''
 # coding: utf-8
 """Tests for IPython.lib.pretty."""
 
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 
 from collections import Counter, defaultdict, deque, OrderedDict, UserList
 import os
 import pytest
 import types
 import string
 import sys
 import unittest
 
 import pytest
 
 from IPython.lib import pretty
 
 from io import StringIO
 
 
 class MyList(object):
     def __init__(self, content):
         self.content = content
     def _repr_pretty_(self, p, cycle):
         if cycle:
             p.text("MyList(...)")
         else:
             with p.group(3, "MyList(", ")"):
                 for (i, child) in enumerate(self.content):
                     if i:
                         p.text(",")
                         p.breakable()
                     else:
                         p.breakable("")
                     p.pretty(child)
 
 
 class MyDict(dict):
     def _repr_pretty_(self, p, cycle):
         p.text("MyDict(...)")
 
 class MyObj(object):
     def somemethod(self):
         pass
 
 
 class Dummy1(object):
     def _repr_pretty_(self, p, cycle):
         p.text("Dummy1(...)")
 
 class Dummy2(Dummy1):
     _repr_pretty_ = None
 
 class NoModule(object):
     pass
 
 NoModule.__module__ = None
 
 class Breaking(object):
     def _repr_pretty_(self, p, cycle):
         with p.group(4,"TG: ",":"):
             p.text("Breaking(")
             p.break_()
             p.text(")")
 
 class BreakingRepr(object):
     def __repr__(self):
         return "Breaking(\n)"
 
 class BadRepr(object):
     def __repr__(self):
         return 1/0
 
 
 def test_indentation():
     """Test correct indentation in groups"""
     count = 40
     gotoutput = pretty.pretty(MyList(range(count)))
     expectedoutput = "MyList(\n" + ",\n".join("   %d" % i for i in range(count)) + ")"
 
     assert gotoutput == expectedoutput
 
 
 def test_dispatch():
     """
     Test correct dispatching: The _repr_pretty_ method for MyDict
     must be found before the registered printer for dict.
     """
     gotoutput = pretty.pretty(MyDict())
     expectedoutput = "MyDict(...)"
 
     assert gotoutput == expectedoutput
 
 
 def test_callability_checking():
     """
     Test that the _repr_pretty_ method is tested for callability and skipped if
     not.
     """
     gotoutput = pretty.pretty(Dummy2())
     expectedoutput = "Dummy1(...)"
 
     assert gotoutput == expectedoutput
 
 
 @pytest.mark.parametrize(
     "obj,expected_output",
     zip(
         [
             set(),
             frozenset(),
             set([1]),
             frozenset([1]),
             set([1, 2]),
             frozenset([1, 2]),
             set([-1, -2, -3]),
         ],
         [
             "set()",
             "frozenset()",
             "{1}",
             "frozenset({1})",
             "{1, 2}",
             "frozenset({1, 2})",
             "{-3, -2, -1}",
         ],
     ),
 )
 def test_sets(obj, expected_output):
     """
     Test that set and frozenset use Python 3 formatting.
     """
     got_output = pretty.pretty(obj)
     assert got_output == expected_output
 
 
 def test_pprint_heap_allocated_type():
     """
     Test that pprint works for heap allocated types.
     """
     module_name = "xxlimited" if sys.version_info < (3, 10) else "xxlimited_35"
+    expected_output = (
+        "xxlimited.Null" if sys.version_info < (3, 10, 6) else "xxlimited_35.Null"
+    )
     xxlimited = pytest.importorskip(module_name)
     output = pretty.pretty(xxlimited.Null)
-    assert output == "xxlimited.Null"
+    assert output == expected_output
 
 
 def test_pprint_nomod():
     """
     Test that pprint works for classes with no __module__.
     """
     output = pretty.pretty(NoModule)
     assert output == "NoModule"
 
 
 def test_pprint_break():
     """
     Test that p.break_ produces expected output
     """
     output = pretty.pretty(Breaking())
     expected = "TG: Breaking(\n    ):"
     assert output == expected
 
 def test_pprint_break_repr():
     """
     Test that p.break_ is used in repr
     """
     output = pretty.pretty([[BreakingRepr()]])
     expected = "[[Breaking(\n  )]]"
     assert output == expected
 
     output = pretty.pretty([[BreakingRepr()]*2])
     expected = "[[Breaking(\n  ),\n  Breaking(\n  )]]"
     assert output == expected
 
 def test_bad_repr():
     """Don't catch bad repr errors"""
     with pytest.raises(ZeroDivisionError):
         pretty.pretty(BadRepr())
 
 class BadException(Exception):
     def __str__(self):
         return -1
 
 class ReallyBadRepr(object):
     __module__ = 1
     @property
     def __class__(self):
         raise ValueError("I am horrible")
 
     def __repr__(self):
         raise BadException()
 
 def test_really_bad_repr():
     with pytest.raises(BadException):
         pretty.pretty(ReallyBadRepr())
 
 
 class SA(object):
     pass
 
 class SB(SA):
     pass
 
 class TestsPretty(unittest.TestCase):
 
     def test_super_repr(self):
         # "<super: module_name.SA, None>"
         output = pretty.pretty(super(SA))
         self.assertRegex(output, r"<super: \S+.SA, None>")
 
         # "<super: module_name.SA, <module_name.SB at 0x...>>"
         sb = SB()
         output = pretty.pretty(super(SA, sb))
         self.assertRegex(output, r"<super: \S+.SA,\s+<\S+.SB at 0x\S+>>")
 
 
     def test_long_list(self):
         lis = list(range(10000))
         p = pretty.pretty(lis)
         last2 = p.rsplit('\n', 2)[-2:]
         self.assertEqual(last2, [' 999,', ' ...]'])
 
     def test_long_set(self):
         s = set(range(10000))
         p = pretty.pretty(s)
         last2 = p.rsplit('\n', 2)[-2:]
         self.assertEqual(last2, [' 999,', ' ...}'])
 
     def test_long_tuple(self):
         tup = tuple(range(10000))
         p = pretty.pretty(tup)
         last2 = p.rsplit('\n', 2)[-2:]
         self.assertEqual(last2, [' 999,', ' ...)'])
 
     def test_long_dict(self):
         d = { n:n for n in range(10000) }
         p = pretty.pretty(d)
         last2 = p.rsplit('\n', 2)[-2:]
         self.assertEqual(last2, [' 999: 999,', ' ...}'])
 
     def test_unbound_method(self):
         output = pretty.pretty(MyObj.somemethod)
         self.assertIn('MyObj.somemethod', output)
 
 
 class MetaClass(type):
     def __new__(cls, name):
         return type.__new__(cls, name, (object,), {'name': name})
 
     def __repr__(self):
         return "[CUSTOM REPR FOR CLASS %s]" % self.name
 
 
 ClassWithMeta = MetaClass('ClassWithMeta')
 
 
 def test_metaclass_repr():
     output = pretty.pretty(ClassWithMeta)
     assert output == "[CUSTOM REPR FOR CLASS ClassWithMeta]"
 
 
 def test_unicode_repr():
     u = u"üniçodé"
     ustr = u
 
     class C(object):
         def __repr__(self):
             return ustr
 
     c = C()
     p = pretty.pretty(c)
     assert p == u
     p = pretty.pretty([c])
     assert p == "[%s]" % u
 
 
 def test_basic_class():
     def type_pprint_wrapper(obj, p, cycle):
         if obj is MyObj:
             type_pprint_wrapper.called = True
         return pretty._type_pprint(obj, p, cycle)
     type_pprint_wrapper.called = False
 
     stream = StringIO()
     printer = pretty.RepresentationPrinter(stream)
     printer.type_pprinters[type] = type_pprint_wrapper
     printer.pretty(MyObj)
     printer.flush()
     output = stream.getvalue()
 
     assert output == "%s.MyObj" % __name__
     assert type_pprint_wrapper.called is True
 
 
 def test_collections_userlist():
     # Create userlist with cycle
     a = UserList()
     a.append(a)
 
     cases = [
         (UserList(), "UserList([])"),
         (
             UserList(i for i in range(1000, 1020)),
             "UserList([1000,\n"
             "          1001,\n"
             "          1002,\n"
             "          1003,\n"
             "          1004,\n"
             "          1005,\n"
             "          1006,\n"
             "          1007,\n"
             "          1008,\n"
             "          1009,\n"
             "          1010,\n"
             "          1011,\n"
             "          1012,\n"
             "          1013,\n"
             "          1014,\n"
             "          1015,\n"
             "          1016,\n"
             "          1017,\n"
             "          1018,\n"
             "          1019])",
         ),
         (a, "UserList([UserList(...)])"),
     ]
     for obj, expected in cases:
         assert pretty.pretty(obj) == expected
 
 
 # TODO : pytest.mark.parametrise once nose is gone.
 def test_collections_defaultdict():
     # Create defaultdicts with cycles
     a = defaultdict()
     a.default_factory = a
     b = defaultdict(list)
     b['key'] = b
 
     # Dictionary order cannot be relied on, test against single keys.
     cases = [
         (defaultdict(list), 'defaultdict(list, {})'),
         (defaultdict(list, {'key': '-' * 50}),
          "defaultdict(list,\n"
          "            {'key': '--------------------------------------------------'})"),
         (a, 'defaultdict(defaultdict(...), {})'),
         (b, "defaultdict(list, {'key': defaultdict(...)})"),
     ]
     for obj, expected in cases:
         assert pretty.pretty(obj) == expected
 
 
 # TODO : pytest.mark.parametrise once nose is gone.
 def test_collections_ordereddict():
     # Create OrderedDict with cycle
     a = OrderedDict()
     a['key'] = a
 
     cases = [
         (OrderedDict(), 'OrderedDict()'),
         (OrderedDict((i, i) for i in range(1000, 1010)),
          'OrderedDict([(1000, 1000),\n'
          '             (1001, 1001),\n'
          '             (1002, 1002),\n'
          '             (1003, 1003),\n'
          '             (1004, 1004),\n'
          '             (1005, 1005),\n'
          '             (1006, 1006),\n'
          '             (1007, 1007),\n'
          '             (1008, 1008),\n'
          '             (1009, 1009)])'),
         (a, "OrderedDict([('key', OrderedDict(...))])"),
     ]
     for obj, expected in cases:
         assert pretty.pretty(obj) == expected
 
 
 # TODO : pytest.mark.parametrise once nose is gone.
 def test_collections_deque():
     # Create deque with cycle
     a = deque()
     a.append(a)
 
     cases = [
         (deque(), 'deque([])'),
         (deque(i for i in range(1000, 1020)),
          'deque([1000,\n'
          '       1001,\n'
          '       1002,\n'
          '       1003,\n'
          '       1004,\n'
          '       1005,\n'
          '       1006,\n'
          '       1007,\n'
          '       1008,\n'
          '       1009,\n'
          '       1010,\n'
          '       1011,\n'
          '       1012,\n'
          '       1013,\n'
          '       1014,\n'
          '       1015,\n'
          '       1016,\n'
          '       1017,\n'
          '       1018,\n'
          '       1019])'),
         (a, 'deque([deque(...)])'),
     ]
     for obj, expected in cases:
         assert pretty.pretty(obj) == expected
 
 
 # TODO : pytest.mark.parametrise once nose is gone.
 def test_collections_counter():
     class MyCounter(Counter):
         pass
     cases = [
         (Counter(), 'Counter()'),
         (Counter(a=1), "Counter({'a': 1})"),
         (MyCounter(a=1), "MyCounter({'a': 1})"),
     ]
     for obj, expected in cases:
         assert pretty.pretty(obj) == expected
 
 # TODO : pytest.mark.parametrise once nose is gone.
 def test_mappingproxy():
     MP = types.MappingProxyType
     underlying_dict = {}
     mp_recursive = MP(underlying_dict)
     underlying_dict[2] = mp_recursive
     underlying_dict[3] = underlying_dict
 
     cases = [
         (MP({}), "mappingproxy({})"),
         (MP({None: MP({})}), "mappingproxy({None: mappingproxy({})})"),
         (MP({k: k.upper() for k in string.ascii_lowercase}),
          "mappingproxy({'a': 'A',\n"
          "              'b': 'B',\n"
          "              'c': 'C',\n"
          "              'd': 'D',\n"
          "              'e': 'E',\n"
          "              'f': 'F',\n"
          "              'g': 'G',\n"
          "              'h': 'H',\n"
          "              'i': 'I',\n"
          "              'j': 'J',\n"
          "              'k': 'K',\n"
          "              'l': 'L',\n"
          "              'm': 'M',\n"
          "              'n': 'N',\n"
          "              'o': 'O',\n"
          "              'p': 'P',\n"
          "              'q': 'Q',\n"
          "              'r': 'R',\n"
          "              's': 'S',\n"
          "              't': 'T',\n"
          "              'u': 'U',\n"
          "              'v': 'V',\n"
          "              'w': 'W',\n"
          "              'x': 'X',\n"
          "              'y': 'Y',\n"
          "              'z': 'Z'})"),
         (mp_recursive, "mappingproxy({2: {...}, 3: {2: {...}, 3: {...}}})"),
         (underlying_dict,
          "{2: mappingproxy({2: {...}, 3: {...}}), 3: {...}}"),
     ]
     for obj, expected in cases:
         assert pretty.pretty(obj) == expected
 
 
 # TODO : pytest.mark.parametrise once nose is gone.
 def test_simplenamespace():
     SN = types.SimpleNamespace
 
     sn_recursive = SN()
     sn_recursive.first = sn_recursive
     sn_recursive.second = sn_recursive
     cases = [
         (SN(), "namespace()"),
         (SN(x=SN()), "namespace(x=namespace())"),
         (SN(a_long_name=[SN(s=string.ascii_lowercase)]*3, a_short_name=None),
          "namespace(a_long_name=[namespace(s='abcdefghijklmnopqrstuvwxyz'),\n"
          "                       namespace(s='abcdefghijklmnopqrstuvwxyz'),\n"
          "                       namespace(s='abcdefghijklmnopqrstuvwxyz')],\n"
          "          a_short_name=None)"),
         (sn_recursive, "namespace(first=namespace(...), second=namespace(...))"),
     ]
     for obj, expected in cases:
         assert pretty.pretty(obj) == expected
 
 
 def test_pretty_environ():
     dict_repr = pretty.pretty(dict(os.environ))
     # reindent to align with 'environ' prefix
     dict_indented = dict_repr.replace('\n', '\n' + (' ' * len('environ')))
     env_repr = pretty.pretty(os.environ)
     assert env_repr == "environ" + dict_indented
 
 
 def test_function_pretty():
     "Test pretty print of function"
     # posixpath is a pure python module, its interface is consistent
     # across Python distributions
     import posixpath
 
     assert pretty.pretty(posixpath.join) == "<function posixpath.join(a, *p)>"
 
     # custom function
     def meaning_of_life(question=None):
         if question:
             return 42
         return "Don't panic"
 
     assert "meaning_of_life(question=None)" in pretty.pretty(meaning_of_life)
 
 
 class OrderedCounter(Counter, OrderedDict):
     'Counter that remembers the order elements are first encountered'
 
     def __repr__(self):
         return '%s(%r)' % (self.__class__.__name__, OrderedDict(self))
 
     def __reduce__(self):
         return self.__class__, (OrderedDict(self),)
 
 class MySet(set):  # Override repr of a basic type
     def __repr__(self):
         return 'mine'
 
 def test_custom_repr():
     """A custom repr should override a pretty printer for a parent type"""
     oc = OrderedCounter("abracadabra")
     assert "OrderedCounter(OrderedDict" in pretty.pretty(oc)
 
     assert pretty.pretty(MySet()) == "mine"

IPython/terminal/shortcuts.py

@@ -1,544 +1,551 b''
 """
 Module to define and register Terminal IPython shortcuts with
 :mod:`prompt_toolkit`
 """
 
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 import warnings
 import signal
 import sys
 import re
 import os
 from typing import Callable
 
 
 from prompt_toolkit.application.current import get_app
 from prompt_toolkit.enums import DEFAULT_BUFFER, SEARCH_BUFFER
 from prompt_toolkit.filters import (has_focus, has_selection, Condition,
     vi_insert_mode, emacs_insert_mode, has_completions, vi_mode)
 from prompt_toolkit.key_binding.bindings.completion import display_completions_like_readline
 from prompt_toolkit.key_binding import KeyBindings
 from prompt_toolkit.key_binding.bindings import named_commands as nc
 from prompt_toolkit.key_binding.vi_state import InputMode, ViState
 
 from IPython.utils.decorators import undoc
 
 @undoc
 @Condition
 def cursor_in_leading_ws():
     before = get_app().current_buffer.document.current_line_before_cursor
     return (not before) or before.isspace()
 
 
+# Needed for to accept autosuggestions in vi insert mode
+def _apply_autosuggest(event):
+    """
+    Apply autosuggestion if at end of line.
+    """
+    b = event.current_buffer
+    d = b.document
+    after_cursor = d.text[d.cursor_position :]
+    lines = after_cursor.split("\n")
+    end_of_current_line = lines[0].strip()
+    suggestion = b.suggestion
+    if (suggestion is not None) and (suggestion.text) and (end_of_current_line == ""):
+        b.insert_text(suggestion.text)
+    else:
+        nc.end_of_line(event)
+
 def create_ipython_shortcuts(shell):
     """Set up the prompt_toolkit keyboard shortcuts for IPython"""
 
     kb = KeyBindings()
     insert_mode = vi_insert_mode | emacs_insert_mode
 
     if getattr(shell, 'handle_return', None):
         return_handler = shell.handle_return(shell)
     else:
         return_handler = newline_or_execute_outer(shell)
 
     kb.add('enter', filter=(has_focus(DEFAULT_BUFFER)
                             & ~has_selection
                             & insert_mode
                         ))(return_handler)
 
     def reformat_and_execute(event):
         reformat_text_before_cursor(event.current_buffer, event.current_buffer.document, shell)
         event.current_buffer.validate_and_handle()
 
     kb.add('escape', 'enter', filter=(has_focus(DEFAULT_BUFFER)
                             & ~has_selection
                             & insert_mode
                                       ))(reformat_and_execute)
 
     kb.add("c-\\")(quit)
 
     kb.add('c-p', filter=(vi_insert_mode & has_focus(DEFAULT_BUFFER))
                 )(previous_history_or_previous_completion)
 
     kb.add('c-n', filter=(vi_insert_mode & has_focus(DEFAULT_BUFFER))
                 )(next_history_or_next_completion)
 
     kb.add('c-g', filter=(has_focus(DEFAULT_BUFFER) & has_completions)
                 )(dismiss_completion)
 
     kb.add('c-c', filter=has_focus(DEFAULT_BUFFER))(reset_buffer)
 
     kb.add('c-c', filter=has_focus(SEARCH_BUFFER))(reset_search_buffer)
 
     supports_suspend = Condition(lambda: hasattr(signal, 'SIGTSTP'))
     kb.add('c-z', filter=supports_suspend)(suspend_to_bg)
 
     # Ctrl+I == Tab
     kb.add('tab', filter=(has_focus(DEFAULT_BUFFER)
                           & ~has_selection
                           & insert_mode
                           & cursor_in_leading_ws
                         ))(indent_buffer)
     kb.add('c-o', filter=(has_focus(DEFAULT_BUFFER) & emacs_insert_mode)
            )(newline_autoindent_outer(shell.input_transformer_manager))
 
     kb.add('f2', filter=has_focus(DEFAULT_BUFFER))(open_input_in_editor)
 
     @Condition
     def auto_match():
         return shell.auto_match
 
     focused_insert = (vi_insert_mode | emacs_insert_mode) & has_focus(DEFAULT_BUFFER)
     _preceding_text_cache = {}
     _following_text_cache = {}
 
     def preceding_text(pattern):
         try:
             return _preceding_text_cache[pattern]
         except KeyError:
             pass
         m = re.compile(pattern)
 
         def _preceding_text():
             app = get_app()
             return bool(m.match(app.current_buffer.document.current_line_before_cursor))
 
         condition = Condition(_preceding_text)
         _preceding_text_cache[pattern] = condition
         return condition
 
     def following_text(pattern):
         try:
             return _following_text_cache[pattern]
         except KeyError:
             pass
         m = re.compile(pattern)
 
         def _following_text():
             app = get_app()
             return bool(m.match(app.current_buffer.document.current_line_after_cursor))
 
         condition = Condition(_following_text)
         _following_text_cache[pattern] = condition
         return condition
 
     # auto match
     @kb.add("(", filter=focused_insert & auto_match & following_text(r"[,)}\]]|$"))
     def _(event):
         event.current_buffer.insert_text("()")
         event.current_buffer.cursor_left()
 
     @kb.add("[", filter=focused_insert & auto_match & following_text(r"[,)}\]]|$"))
     def _(event):
         event.current_buffer.insert_text("[]")
         event.current_buffer.cursor_left()
 
     @kb.add("{", filter=focused_insert & auto_match & following_text(r"[,)}\]]|$"))
     def _(event):
         event.current_buffer.insert_text("{}")
         event.current_buffer.cursor_left()
 
     @kb.add(
         '"',
         filter=focused_insert
         & auto_match
         & preceding_text(r'^([^"]+|"[^"]*")*$')
         & following_text(r"[,)}\]]|$"),
     )
     def _(event):
         event.current_buffer.insert_text('""')
         event.current_buffer.cursor_left()
 
     @kb.add(
         "'",
         filter=focused_insert
         & auto_match
         & preceding_text(r"^([^']+|'[^']*')*$")
         & following_text(r"[,)}\]]|$"),
     )
     def _(event):
         event.current_buffer.insert_text("''")
         event.current_buffer.cursor_left()
 
     # raw string
     @kb.add(
         "(", filter=focused_insert & auto_match & preceding_text(r".*(r|R)[\"'](-*)$")
     )
     def _(event):
         matches = re.match(
             r".*(r|R)[\"'](-*)",
             event.current_buffer.document.current_line_before_cursor,
         )
         dashes = matches.group(2) or ""
         event.current_buffer.insert_text("()" + dashes)
         event.current_buffer.cursor_left(len(dashes) + 1)
 
     @kb.add(
         "[", filter=focused_insert & auto_match & preceding_text(r".*(r|R)[\"'](-*)$")
     )
     def _(event):
         matches = re.match(
             r".*(r|R)[\"'](-*)",
             event.current_buffer.document.current_line_before_cursor,
         )
         dashes = matches.group(2) or ""
         event.current_buffer.insert_text("[]" + dashes)
         event.current_buffer.cursor_left(len(dashes) + 1)
 
     @kb.add(
         "{", filter=focused_insert & auto_match & preceding_text(r".*(r|R)[\"'](-*)$")
     )
     def _(event):
         matches = re.match(
             r".*(r|R)[\"'](-*)",
             event.current_buffer.document.current_line_before_cursor,
         )
         dashes = matches.group(2) or ""
         event.current_buffer.insert_text("{}" + dashes)
         event.current_buffer.cursor_left(len(dashes) + 1)
 
     # just move cursor
     @kb.add(")", filter=focused_insert & auto_match & following_text(r"^\)"))
     @kb.add("]", filter=focused_insert & auto_match & following_text(r"^\]"))
     @kb.add("}", filter=focused_insert & auto_match & following_text(r"^\}"))
     @kb.add('"', filter=focused_insert & auto_match & following_text('^"'))
     @kb.add("'", filter=focused_insert & auto_match & following_text("^'"))
     def _(event):
         event.current_buffer.cursor_right()
 
     @kb.add(
         "backspace",
         filter=focused_insert
         & preceding_text(r".*\($")
         & auto_match
         & following_text(r"^\)"),
     )
     @kb.add(
         "backspace",
         filter=focused_insert
         & preceding_text(r".*\[$")
         & auto_match
         & following_text(r"^\]"),
     )
     @kb.add(
         "backspace",
         filter=focused_insert
         & preceding_text(r".*\{$")
         & auto_match
         & following_text(r"^\}"),
     )
     @kb.add(
         "backspace",
         filter=focused_insert
         & preceding_text('.*"$')
         & auto_match
         & following_text('^"'),
     )
     @kb.add(
         "backspace",
         filter=focused_insert
         & preceding_text(r".*'$")
         & auto_match
         & following_text(r"^'"),
     )
     def _(event):
         event.current_buffer.delete()
         event.current_buffer.delete_before_cursor()
 
     if shell.display_completions == "readlinelike":
         kb.add(
             "c-i",
             filter=(
                 has_focus(DEFAULT_BUFFER)
                 & ~has_selection
                 & insert_mode
                 & ~cursor_in_leading_ws
             ),
         )(display_completions_like_readline)
 
     if sys.platform == "win32":
         kb.add("c-v", filter=(has_focus(DEFAULT_BUFFER) & ~vi_mode))(win_paste)
 
     @Condition
     def ebivim():
         return shell.emacs_bindings_in_vi_insert_mode
 
     focused_insert_vi = has_focus(DEFAULT_BUFFER) & vi_insert_mode
 
-    # Needed for to accept autosuggestions in vi insert mode
-    def _apply_autosuggest(event):
-        b = event.current_buffer
-        suggestion = b.suggestion
-        if suggestion is not None and suggestion.text:
-            b.insert_text(suggestion.text)
-        else:
-            nc.end_of_line(event)
-
     @kb.add("end", filter=has_focus(DEFAULT_BUFFER) & (ebivim | ~vi_insert_mode))
     def _(event):
         _apply_autosuggest(event)
 
     @kb.add("c-e", filter=focused_insert_vi & ebivim)
     def _(event):
         _apply_autosuggest(event)
 
     @kb.add("c-f", filter=focused_insert_vi)
     def _(event):
         b = event.current_buffer
         suggestion = b.suggestion
         if suggestion:
             b.insert_text(suggestion.text)
         else:
             nc.forward_char(event)
 
     @kb.add("escape", "f", filter=focused_insert_vi & ebivim)
     def _(event):
         b = event.current_buffer
         suggestion = b.suggestion
         if suggestion:
             t = re.split(r"(\S+\s+)", suggestion.text)
             b.insert_text(next((x for x in t if x), ""))
         else:
             nc.forward_word(event)
 
     # Simple Control keybindings
     key_cmd_dict = {
         "c-a": nc.beginning_of_line,
         "c-b": nc.backward_char,
         "c-k": nc.kill_line,
         "c-w": nc.backward_kill_word,
         "c-y": nc.yank,
         "c-_": nc.undo,
     }
 
     for key, cmd in key_cmd_dict.items():
         kb.add(key, filter=focused_insert_vi & ebivim)(cmd)
 
     # Alt and Combo Control keybindings
     keys_cmd_dict = {
         # Control Combos
         ("c-x", "c-e"): nc.edit_and_execute,
         ("c-x", "e"): nc.edit_and_execute,
         # Alt
         ("escape", "b"): nc.backward_word,
         ("escape", "c"): nc.capitalize_word,
         ("escape", "d"): nc.kill_word,
         ("escape", "h"): nc.backward_kill_word,
         ("escape", "l"): nc.downcase_word,
         ("escape", "u"): nc.uppercase_word,
         ("escape", "y"): nc.yank_pop,
         ("escape", "."): nc.yank_last_arg,
     }
 
     for keys, cmd in keys_cmd_dict.items():
         kb.add(*keys, filter=focused_insert_vi & ebivim)(cmd)
 
     def get_input_mode(self):
         app = get_app()
         app.ttimeoutlen = shell.ttimeoutlen
         app.timeoutlen = shell.timeoutlen
 
         return self._input_mode
 
     def set_input_mode(self, mode):
         shape = {InputMode.NAVIGATION: 2, InputMode.REPLACE: 4}.get(mode, 6)
         cursor = "\x1b[{} q".format(shape)
 
         sys.stdout.write(cursor)
         sys.stdout.flush()
 
         self._input_mode = mode
 
     if shell.editing_mode == "vi" and shell.modal_cursor:
         ViState._input_mode = InputMode.INSERT
         ViState.input_mode = property(get_input_mode, set_input_mode)
 
     return kb
 
 
 def reformat_text_before_cursor(buffer, document, shell):
     text = buffer.delete_before_cursor(len(document.text[:document.cursor_position]))
     try:
         formatted_text = shell.reformat_handler(text)
         buffer.insert_text(formatted_text)
     except Exception as e:
         buffer.insert_text(text)
 
 
 def newline_or_execute_outer(shell):
 
     def newline_or_execute(event):
         """When the user presses return, insert a newline or execute the code."""
         b = event.current_buffer
         d = b.document
 
         if b.complete_state:
             cc = b.complete_state.current_completion
             if cc:
                 b.apply_completion(cc)
             else:
                 b.cancel_completion()
             return
 
         # If there's only one line, treat it as if the cursor is at the end.
         # See https://github.com/ipython/ipython/issues/10425
         if d.line_count == 1:
             check_text = d.text
         else:
             check_text = d.text[:d.cursor_position]
         status, indent = shell.check_complete(check_text)
        
         # if all we have after the cursor is whitespace: reformat current text
         # before cursor
         after_cursor = d.text[d.cursor_position:]
         reformatted = False
         if not after_cursor.strip():
             reformat_text_before_cursor(b, d, shell)
             reformatted = True
         if not (d.on_last_line or
                 d.cursor_position_row >= d.line_count - d.empty_line_count_at_the_end()
                 ):
             if shell.autoindent:
                 b.insert_text('\n' + indent)
             else:
                 b.insert_text('\n')
             return
 
         if (status != 'incomplete') and b.accept_handler:
             if not reformatted:
                 reformat_text_before_cursor(b, d, shell)
             b.validate_and_handle()
         else:
             if shell.autoindent:
                 b.insert_text('\n' + indent)
             else:
                 b.insert_text('\n')
     return newline_or_execute
 
 
 def previous_history_or_previous_completion(event):
     """
     Control-P in vi edit mode on readline is history next, unlike default prompt toolkit.
 
     If completer is open this still select previous completion.
     """
     event.current_buffer.auto_up()
 
 
 def next_history_or_next_completion(event):
     """
     Control-N in vi edit mode on readline is history previous, unlike default prompt toolkit.
 
     If completer is open this still select next completion.
     """
     event.current_buffer.auto_down()
 
 
 def dismiss_completion(event):
     b = event.current_buffer
     if b.complete_state:
         b.cancel_completion()
 
 
 def reset_buffer(event):
     b = event.current_buffer
     if b.complete_state:
         b.cancel_completion()
     else:
         b.reset()
 
 
 def reset_search_buffer(event):
     if event.current_buffer.document.text:
         event.current_buffer.reset()
     else:
         event.app.layout.focus(DEFAULT_BUFFER)
 
 def suspend_to_bg(event):
     event.app.suspend_to_background()
 
 def quit(event):
     """
     On platforms that support SIGQUIT, send SIGQUIT to the current process.
     On other platforms, just exit the process with a message.
     """
     sigquit = getattr(signal, "SIGQUIT", None)
     if sigquit is not None:
         os.kill(0, signal.SIGQUIT)
     else:
         sys.exit("Quit")
 
 def indent_buffer(event):
     event.current_buffer.insert_text(' ' * 4)
 
 @undoc
 def newline_with_copy_margin(event):
     """
     DEPRECATED since IPython 6.0
 
     See :any:`newline_autoindent_outer` for a replacement.
 
     Preserve margin and cursor position when using
     Control-O to insert a newline in EMACS mode
     """
     warnings.warn("`newline_with_copy_margin(event)` is deprecated since IPython 6.0. "
       "see `newline_autoindent_outer(shell)(event)` for a replacement.",
                   DeprecationWarning, stacklevel=2)
 
     b = event.current_buffer
     cursor_start_pos = b.document.cursor_position_col
     b.newline(copy_margin=True)
     b.cursor_up(count=1)
     cursor_end_pos = b.document.cursor_position_col
     if cursor_start_pos != cursor_end_pos:
         pos_diff = cursor_start_pos - cursor_end_pos
         b.cursor_right(count=pos_diff)
 
 def newline_autoindent_outer(inputsplitter) -> Callable[..., None]:
     """
     Return a function suitable for inserting a indented newline after the cursor.
 
     Fancier version of deprecated ``newline_with_copy_margin`` which should
     compute the correct indentation of the inserted line. That is to say, indent
     by 4 extra space after a function definition, class definition, context
     manager... And dedent by 4 space after ``pass``, ``return``, ``raise ...``.
     """
 
     def newline_autoindent(event):
         """insert a newline after the cursor indented appropriately."""
         b = event.current_buffer
         d = b.document
 
         if b.complete_state:
             b.cancel_completion()
         text = d.text[:d.cursor_position] + '\n'
         _, indent = inputsplitter.check_complete(text)
         b.insert_text('\n' + (' ' * (indent or 0)), move_cursor=False)
 
     return newline_autoindent
 
 
 def open_input_in_editor(event):
     event.app.current_buffer.open_in_editor()
 
 
 if sys.platform == 'win32':
     from IPython.core.error import TryNext
     from IPython.lib.clipboard import (ClipboardEmpty,
                                        win32_clipboard_get,
                                        tkinter_clipboard_get)
 
     @undoc
     def win_paste(event):
         try:
             text = win32_clipboard_get()
         except TryNext:
             try:
                 text = tkinter_clipboard_get()
             except (TryNext, ClipboardEmpty):
                 return
         except ClipboardEmpty:
             return
         event.current_buffer.insert_text(text.replace("\t", " " * 4))

README.rst

@@ -1,171 +1,171 b''
-.. image:: https://codecov.io/github/ipython/ipython/coverage.svg?branch=master
-    :target: https://codecov.io/github/ipython/ipython?branch=master
+.. image:: https://codecov.io/github/ipython/ipython/coverage.svg?branch=main
+    :target: https://codecov.io/github/ipython/ipython?branch=main
 
 .. image:: https://img.shields.io/pypi/v/IPython.svg
     :target: https://pypi.python.org/pypi/ipython
 
 .. image:: https://github.com/ipython/ipython/actions/workflows/test.yml/badge.svg
     :target: https://github.com/ipython/ipython/actions/workflows/test.yml)
 
 .. image:: https://www.codetriage.com/ipython/ipython/badges/users.svg
     :target: https://www.codetriage.com/ipython/ipython/
 
 .. image:: https://raster.shields.io/badge/Follows-NEP29-brightgreen.png
     :target: https://numpy.org/neps/nep-0029-deprecation_policy.html
 
 
 ===========================================
  IPython: Productive Interactive Computing
 ===========================================
 
 Overview
 ========
 
 Welcome to IPython.  Our full documentation is available on `ipython.readthedocs.io
 <https://ipython.readthedocs.io/en/stable/>`_ and contains information on how to install, use, and
 contribute to the project.
 IPython (Interactive Python) is a command shell for interactive computing in multiple programming languages, originally developed for the Python programming language, that offers introspection, rich media, shell syntax, tab completion, and history.
 
 **IPython versions and Python Support**
 
 Starting with IPython 7.10, IPython follows `NEP 29 <https://numpy.org/neps/nep-0029-deprecation_policy.html>`_
 
 **IPython 7.17+** requires Python version 3.7 and above.
 
 **IPython 7.10+** requires Python version 3.6 and above.
 
 **IPython 7.0** requires Python version 3.5 and above.
 
 **IPython 6.x** requires Python version 3.3 and above.
 
 **IPython 5.x LTS** is the compatible release for Python 2.7.
 If you require Python 2 support, you **must** use IPython 5.x LTS. Please
 update your project configurations and requirements as necessary.
 
 
 The Notebook, Qt console and a number of other pieces are now parts of *Jupyter*.
 See the `Jupyter installation docs <https://jupyter.readthedocs.io/en/latest/install.html>`__
 if you want to use these.
 
 Main features of IPython
 ========================
 Comprehensive object introspection.
 
 Input history, persistent across sessions.
 
 Caching of output results during a session with automatically generated references.
 
 Extensible tab completion, with support by default for completion of python variables and keywords, filenames and function keywords.
 
 Extensible system of ‘magic’ commands for controlling the environment and performing many tasks related to IPython or the operating system.
 
 A rich configuration system with easy switching between different setups (simpler than changing $PYTHONSTARTUP environment variables every time).
 
 Session logging and reloading.
 
 Extensible syntax processing for special purpose situations.
 
 Access to the system shell with user-extensible alias system.
 
 Easily embeddable in other Python programs and GUIs.
 
 Integrated access to the pdb debugger and the Python profiler.
 
 
 Development and Instant running
 ===============================
 
 You can find the latest version of the development documentation on `readthedocs
 <https://ipython.readthedocs.io/en/latest/>`_.
 
 You can run IPython from this directory without even installing it system-wide
 by typing at the terminal::
 
    $ python -m IPython
 
 Or see the `development installation docs
 <https://ipython.readthedocs.io/en/latest/install/install.html#installing-the-development-version>`_
 for the latest revision on read the docs.
 
 Documentation and installation instructions for older version of IPython can be
 found on the `IPython website <https://ipython.org/documentation.html>`_
 
 
 
 IPython requires Python version 3 or above
 ==========================================
 
 Starting with version 6.0, IPython does not support Python 2.7, 3.0, 3.1, or
 3.2.
 
 For a version compatible with Python 2.7, please install the 5.x LTS Long Term
 Support version.
 
 If you are encountering this error message you are likely trying to install or
 use IPython from source. You need to checkout the remote 5.x branch. If you are
 using git the following should work::
 
   $ git fetch origin
   $ git checkout 5.x
 
 If you encounter this error message with a regular install of IPython, then you
 likely need to update your package manager, for example if you are using `pip`
 check the version of pip with::
 
   $ pip --version
 
 You will need to update pip to the version 9.0.1 or greater. If you are not using
 pip, please inquiry with the maintainers of the package for your package
 manager.
 
 For more information see one of our blog posts:
 
     https://blog.jupyter.org/release-of-ipython-5-0-8ce60b8d2e8e
 
 As well as the following Pull-Request for discussion:
 
     https://github.com/ipython/ipython/pull/9900
 
 This error does also occur if you are invoking ``setup.py`` directly – which you
 should not – or are using ``easy_install`` If this is the case, use ``pip
 install .`` instead of ``setup.py install`` , and ``pip install -e .`` instead
 of ``setup.py develop`` If you are depending on IPython as a dependency you may
 also want to have a conditional dependency on IPython depending on the Python
 version::
 
     install_req = ['ipython']
     if sys.version_info[0] < 3 and 'bdist_wheel' not in sys.argv:
         install_req.remove('ipython')
         install_req.append('ipython<6')
 
     setup(
         ...
         install_requires=install_req
     )
 
 Alternatives to IPython
 =======================
 
 IPython may not be to your taste; if that's the case there might be similar
 project that you might want to use:
 
 - The classic Python REPL.
 - `bpython <https://bpython-interpreter.org/>`_
 - `mypython <https://www.asmeurer.com/mypython/>`_
 - `ptpython and ptipython <https://pypi.org/project/ptpython/>`_
 - `Xonsh <https://xon.sh/>`_
 
 Ignoring commits with git blame.ignoreRevsFile
 ==============================================
 
 As of git 2.23, it is possible to make formatting changes without breaking
 ``git blame``. See the `git documentation
 <https://git-scm.com/docs/git-config#Documentation/git-config.txt-blameignoreRevsFile>`_
 for more details.
 
 To use this feature you must:
 
 - Install git >= 2.23
 - Configure your local git repo by running:
    - POSIX: ``tools\configure-git-blame-ignore-revs.sh``
    - Windows:  ``tools\configure-git-blame-ignore-revs.bat``

docs/source/coredev/index.rst

@@ -1,320 +1,320 b''
 .. _core_developer_guide:
 
 =================================
 Guide for IPython core Developers
 =================================
 
 This guide documents the development of IPython itself.  Alternatively,
 developers of third party tools and libraries that use IPython should see the
 :doc:`../development/index`.
 
 
 For instructions on how to make a developer install see :ref:`devinstall`.
 
 Backporting Pull requests
 =========================
 
-All pull requests should usually be made against ``master``, if a Pull Request
+All pull requests should usually be made against ``main``, if a Pull Request
 need to be backported to an earlier release; then it should be tagged with the
 correct ``milestone``.
 
 If you tag a pull request with a milestone **before** merging the pull request,
-and the base ref is ``master``, then our backport bot should automatically create
+and the base ref is ``main``, then our backport bot should automatically create
 a corresponding pull-request that backport on the correct branch.
 
 If you have write access to the IPython repository you can also just mention the
 **backport bot** to do the work for you. The bot is evolving so instructions may
 be different. At the time of this writing you can use::
 
     @meeseeksdev[bot] backport [to] <branchname>
 
 The bot will attempt to backport the current pull-request and issue a PR if
 possible. 
 
 .. note::
 
     The ``@`` and ``[bot]`` when mentioning the bot should be optional and can
     be omitted.
 
 If the pull request cannot be automatically backported, the bot should tell you
 so on the PR and apply a "Need manual backport" tag to the origin PR.
 
 .. _release_process:
 
 IPython release process
 =======================
 
 This document contains the process that is used to create an IPython release.
 
 Conveniently, the ``release`` script in the ``tools`` directory of the ``IPython``
 repository automates most of the release process. This document serves as a
 handy reminder and checklist for the release manager.
 
 During the release process, you might need the extra following dependencies:
 
  - ``keyring`` to access your GitHub authentication tokens
  - ``graphviz`` to generate some graphs in the documentation
  - ``ghpro`` to generate the stats
 
 Make sure you have all the required dependencies to run the tests as well.
 
 You can try to ``source tools/release_helper.sh`` when releasing via bash, it 
 should guide you through most of the process.
 
 
 1. Set Environment variables
 ----------------------------
 
 Set environment variables to document previous release tag, current
 release milestone, current release version, and git tag.
 
 These variables may be used later to copy/paste as answers to the script
 questions instead of typing the appropriate command when the time comes. These
 variables are not used by the scripts directly; therefore, there is no need to
 ``export`` them. The format for bash is as follows, but note that these values
 are just an example valid only for the 5.0 release; you'll need to update them
 for the release you are actually making::
 
     PREV_RELEASE=4.2.1
     MILESTONE=5.0
     VERSION=5.0.0
-    BRANCH=master
+    BRANCH=main
 
 For `reproducibility of builds <https://reproducible-builds.org/specs/source-date-epoch/>`_,
 we recommend setting ``SOURCE_DATE_EPOCH`` prior to running the build; record the used value
 of ``SOURCE_DATE_EPOCH`` as it may not be available from build artifact. You
 should be able to use ``date +%s`` to get a formatted timestamp::
 
     SOURCE_DATE_EPOCH=$(date +%s)
 
 
 2. Create GitHub stats and finish release note
 ----------------------------------------------
 
 .. note::
 
     This step is optional if making a Beta or RC release.
 
 .. note::
 
     Before generating the GitHub stats, verify that all closed issues and pull
     requests have `appropriate milestones
     <https://github.com/ipython/ipython/wiki/Dev:-GitHub-workflow#milestones>`_.
     `This search
     <https://github.com/ipython/ipython/issues?q=is%3Aclosed+no%3Amilestone+is%3Aissue>`_
     should return no results before creating the GitHub stats.
 
 If a major release:
 
     - merge any pull request notes into what's new::
 
           python tools/update_whatsnew.py
 
     - update ``docs/source/whatsnew/development.rst``, to ensure it covers
       the major release features
 
     - move the contents of ``development.rst`` to ``versionX.rst`` where ``X`` is
       the numerical release version
 
     - generate summary of GitHub contributions, which can be done with::
 
           python tools/github_stats.py --milestone $MILESTONE > stats.rst
 
       which may need some manual cleanup of ``stats.rst``. Add the cleaned
       ``stats.rst`` results to ``docs/source/whatsnew/github-stats-X.rst``
       where ``X`` is the numerical release version (don't forget to add it to
       the git repository as well). If creating a major release, make a new
       ``github-stats-X.rst`` file; if creating a minor release, the content
       from ``stats.rst`` may simply be added to the top of an existing
       ``github-stats-X.rst`` file.
 
     - Edit ``docs/source/whatsnew/index.rst`` to list the new ``github-stats-X``
       file you just created.
 
     - You do not need to temporarily remove the first entry called
       ``development``, nor re-add it after the release, it will automatically be
       hidden when releasing a stable version of IPython (if ``_version_extra``
       in ``release.py`` is an empty string.
 
       Make sure that the stats file has a header or it won't be rendered in
       the final documentation.
 
 To find duplicates and update `.mailmap`, use::
 
     git log --format="%aN <%aE>" $PREV_RELEASE... | sort -u -f
 
 If a minor release you might need to do some of the above points manually, and
 forward port the changes.
 
 3. Make sure the repository is clean
 ------------------------------------
 
 of any file that could be problematic.
    Remove all non-tracked files with:
 
    .. code::
 
        git clean -xfdi
 
    This will ask for confirmation before removing all untracked files. Make
    sure the ``dist/`` folder is clean to avoid any stale builds from
    previous build attempts.
 
 
 4. Update the release version number
 ------------------------------------
 
 Edit ``IPython/core/release.py`` to have the current version.
 
 in particular, update version number and ``_version_extra`` content in
 ``IPython/core/release.py``.
 
 Step 5 will validate your changes automatically, but you might still want to
 make sure the version number matches pep440.
 
 In particular, ``rc`` and ``beta`` are not separated by ``.`` or the ``sdist``
 and ``bdist`` will appear as different releases. For example, a valid version
 number for a release candidate (rc) release is: ``1.3rc1``. Notice that there
 is no separator between the '3' and the 'r'. Check the environment variable
 ``$VERSION`` as well.
 
 You will likely just have to modify/comment/uncomment one of the lines setting
 ``_version_extra``
 
 
 5. Run the `tools/build_release` script
 ---------------------------------------
 
 Running ``tools/build_release`` does all the file checking and building that
 the real release script will do. This makes test installations, checks that
 the build procedure runs OK, and tests other steps in the release process.
 
 The ``build_release`` script will in particular verify that the version number
 match PEP 440, in order to avoid surprise at the time of build upload.
 
 We encourage creating a test build of the docs as well. 
 
 6. Create and push the new tag
 ------------------------------
 
 Commit the changes to release.py::
 
     git commit -am "release $VERSION" -S
     git push origin $BRANCH
 
 (omit the ``-S`` if you are no signing the package)
 
 Create and push the tag::
 
     git tag -am "release $VERSION" "$VERSION" -s
     git push origin $VERSION
 
 (omit the ``-s`` if you are no signing the package)
 
 Update release.py back to ``x.y-dev`` or ``x.y-maint`` commit and push::
 
     git commit -am "back to development" -S
     git push origin $BRANCH
 
 (omit the ``-S`` if you are no signing the package)
 
 Now checkout the tag we just made::
 
     git checkout $VERSION
 
 7. Run the release script
 -------------------------
 
 Run the ``release`` script, this step requires having a current wheel, Python
 >=3.4 and Python 2.7.::
 
     ./tools/release
 
 This makes the tarballs and wheels, and puts them under the ``dist/``
 folder. Be sure to test the ``wheels``  and the ``sdist`` locally before
 uploading them to PyPI. We do not use an universal wheel as each wheel
 installs an ``ipython2`` or ``ipython3`` script, depending on the version of
 Python it is built for. Using an universal wheel would prevent this.
 
 Check the shasum of files with::
 
     shasum -a 256 dist/*
 
 and takes notes of them you might need them to update the conda-forge recipes.
 Rerun the command and check the hash have not changed::
 
     ./tools/release
     shasum -a 256 dist/*
 
 Use the following to actually upload the result of the build::
 
     ./tools/release upload
 
 It should posts them to ``archive.ipython.org`` and to PyPI.
 
 PyPI/Warehouse will automatically hide previous releases. If you are uploading
 a non-stable version, make sure to log-in to PyPI and un-hide previous version.
 
 
 8. Draft a short release announcement
 -------------------------------------
 
 The announcement should include:
 
 - release highlights
 - a link to the html version of the *What's new* section of the documentation
 - a link to upgrade or installation tips (if necessary)
 
 Post the announcement to the mailing list and or blog, and link from Twitter.
 
 .. note::
 
     If you are doing a RC or Beta, you can likely skip the next steps.
 
 9. Update milestones on GitHub
 -------------------------------
 
 These steps will bring milestones up to date:
 
 - close the just released milestone
 - open a new milestone for the next release (x, y+1), if the milestone doesn't
   exist already
 
 10. Update the IPython website
 ------------------------------
 
 The IPython website should document the new release:
 
 - add release announcement (news, announcements)
 - update current version and download links
 - update links on the documentation page (especially if a major release)
 
 11. Update readthedocs
 ----------------------
 
 Make sure to update readthedocs and set the latest tag as stable, as well as
 checking that previous release is still building under its own tag.
 
 12. Update the Conda-Forge feedstock
 ------------------------------------
 
 Follow the instructions on `the repository <https://github.com/conda-forge/ipython-feedstock>`_
 
 13. Celebrate!
 --------------
 
 Celebrate the release and please thank the contributors for their work. Great
 job!
 
 
 
 Old Documentation
 =================
 
 Out of date documentation is still available and have been kept for archival purposes.
 
 .. note::
 
   Developers documentation used to be on the IPython wiki, but are now out of
   date. The wiki is though still available for historical reasons: `Old IPython
   GitHub Wiki.  <https://github.com/ipython/ipython/wiki/Dev:-Index>`_

examples/IPython Kernel/Terminal Usage.ipynb

@@ -1,285 +1,285 b''
 {
  "cells": [
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
     "# A few things that work best/only at the IPython terminal or Qt console clients"
    ]
   },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
     "## Running code with `%run`"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": 1,
    "metadata": {
     "collapsed": false
    },
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
       "Writing script.py\n"
      ]
     }
    ],
    "source": [
     "%%writefile script.py\n",
     "x = 10\n",
     "y = 20\n",
     "z = x+y\n",
     "print('z is: %s' % z)"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": 2,
    "metadata": {
     "collapsed": false
    },
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
       "z is: 30\n"
      ]
     }
    ],
    "source": [
     "%run script"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": 3,
    "metadata": {
     "collapsed": false
    },
    "outputs": [
     {
      "data": {
       "text/plain": [
        "10"
       ]
      },
      "execution_count": 3,
      "metadata": {},
      "output_type": "execute_result"
     }
    ],
    "source": [
     "x"
    ]
   },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
     "## Event loop and GUI integration"
    ]
   },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
     "The `%gui` magic enables the integration of GUI event loops with the interactive execution loop, allowing you to run GUI code without blocking IPython.\n",
     "\n",
     "Consider for example the execution of Qt-based code. Once we enable the Qt gui support:"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": 4,
    "metadata": {
     "collapsed": false
    },
    "outputs": [],
    "source": [
     "%gui qt"
    ]
   },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
     "We can define a simple Qt application class (simplified version from [this Qt tutorial](http://zetcode.com/tutorials/pyqt4/firstprograms)):"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": 5,
    "metadata": {
     "collapsed": false
    },
    "outputs": [],
    "source": [
     "import sys\n",
     "from PyQt4 import QtGui, QtCore\n",
     "\n",
     "class SimpleWindow(QtGui.QWidget):\n",
     "    def __init__(self, parent=None):\n",
     "        QtGui.QWidget.__init__(self, parent)\n",
     "\n",
     "        self.setGeometry(300, 300, 200, 80)\n",
     "        self.setWindowTitle('Hello World')\n",
     "\n",
     "        quit = QtGui.QPushButton('Close', self)\n",
     "        quit.setGeometry(10, 10, 60, 35)\n",
     "\n",
     "        self.connect(quit, QtCore.SIGNAL('clicked()'),\n",
     "                     self, QtCore.SLOT('close()'))"
    ]
   },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
     "And now we can instantiate it:"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": 6,
    "metadata": {
     "collapsed": false
    },
    "outputs": [],
    "source": [
     "app = QtCore.QCoreApplication.instance()\n",
     "if app is None:\n",
     "    app = QtGui.QApplication([])\n",
     "\n",
     "sw = SimpleWindow()\n",
     "sw.show()\n",
     "\n",
     "from IPython.lib.guisupport import start_event_loop_qt4\n",
     "start_event_loop_qt4(app)"
    ]
   },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
     "But IPython still remains responsive:"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": 7,
    "metadata": {
     "collapsed": false
    },
    "outputs": [
     {
      "data": {
       "text/plain": [
        "12"
       ]
      },
      "execution_count": 7,
      "metadata": {},
      "output_type": "execute_result"
     }
    ],
    "source": [
     "10+2"
    ]
   },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "The `%gui` magic can be similarly used to control Wx, Tk, glut and pyglet applications, [as can be seen in our examples](https://github.com/ipython/ipython/tree/master/examples/lib)."
+    "The `%gui` magic can be similarly used to control Wx, Tk, glut and pyglet applications, [as can be seen in our examples](https://github.com/ipython/ipython/tree/main/examples/lib)."
    ]
   },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
     "## Embedding IPython in a terminal application"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": 8,
    "metadata": {
     "collapsed": false
    },
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
       "Writing simple-embed.py\n"
      ]
     }
    ],
    "source": [
     "%%writefile simple-embed.py\n",
     "# This shows how to use the new top-level embed function.  It is a simpler\n",
     "# API that manages the creation of the embedded shell.\n",
     "\n",
     "from IPython import embed\n",
     "\n",
     "a = 10\n",
     "b = 20\n",
     "\n",
     "embed(header='First time', banner1='')\n",
     "\n",
     "c = 30\n",
     "d = 40\n",
     "\n",
     "embed(header='The second time')"
    ]
   },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
     "The example in kernel-embedding shows how to embed a full kernel into an application and how to connect to this kernel from an external process."
    ]
   },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
     "## Logging terminal sessions and transitioning to a notebook"
    ]
   },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
     "The `%logstart` magic lets you log a terminal session with various degrees of control, and the `%notebook` one will convert an interactive console session into a notebook with all input cells already created for you (but no output)."
    ]
   }
  ],
  "metadata": {
   "kernelspec": {
    "display_name": "Python 3",
    "language": "python",
    "name": "python3"
   },
   "language_info": {
    "codemirror_mode": {
     "name": "ipython",
     "version": 3
    },
    "file_extension": ".py",
    "mimetype": "text/x-python",
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
    "version": "3.4.2"
   }
  },
  "nbformat": 4,
  "nbformat_minor": 0
 }

setup.py

@@ -1,148 +1,148 b''
 # -*- coding: utf-8 -*-
 """Setup script for IPython.
 
 Under Posix environments it works like a typical setup.py script.
 Under Windows, the command sdist is not supported, since IPython
 requires utilities which are not available under Windows."""
 
 #-----------------------------------------------------------------------------
 #  Copyright (c) 2008-2011, IPython Development Team.
 #  Copyright (c) 2001-2007, Fernando Perez <[e-mail]>
 #  Copyright (c) 2001, Janko Hauser <[e-mail]>
 #  Copyright (c) 2001, Nathaniel Gray <[e-mail]>
 #
 #  Distributed under the terms of the Modified BSD License.
 #
 #  The full license is in the file COPYING.rst, distributed with this software.
 #-----------------------------------------------------------------------------
 
 import os
 import sys
 
 # **Python version check**
 #
 # This check is also made in IPython/__init__, don't forget to update both when
 # changing Python version requirements.
 if sys.version_info < (3, 8):
     pip_message = 'This may be due to an out of date pip. Make sure you have pip >= 9.0.1.'
     try:
         import pip
         pip_version = tuple([int(x) for x in pip.__version__.split('.')[:3]])
         if pip_version < (9, 0, 1) :
             pip_message = 'Your pip version is out of date, please install pip >= 9.0.1. '\
             'pip {} detected.'.format(pip.__version__)
         else:
             # pip is new enough - it must be something else
             pip_message = ''
     except Exception:
         pass
 
 
     error = """
 IPython 8+ supports Python 3.8 and above, following NEP 29.
 When using Python 2.7, please install IPython 5.x LTS Long Term Support version.
 Python 3.3 and 3.4 were supported up to IPython 6.x.
 Python 3.5 was supported with IPython 7.0 to 7.9.
 Python 3.6 was supported with IPython up to 7.16.
 Python 3.7 was still supported with the 7.x branch.
 
 See IPython `README.rst` file for more information:
 
-    https://github.com/ipython/ipython/blob/master/README.rst
+    https://github.com/ipython/ipython/blob/main/README.rst
 
 Python {py} detected.
 {pip}
 """.format(
         py=sys.version_info, pip=pip_message
     )
 
     print(error, file=sys.stderr)
     sys.exit(1)
 
 # At least we're on the python version we need, move on.
 
 from setuptools import setup
 
 # Our own imports
 sys.path.insert(0, ".")
 
 from setupbase import target_update
 
 from setupbase import (
     setup_args,
     check_package_data_first,
     find_data_files,
     git_prebuild,
     install_symlinked,
     install_lib_symlink,
     install_scripts_for_symlink,
     unsymlink,
 )
 
 #-------------------------------------------------------------------------------
 # Handle OS specific things
 #-------------------------------------------------------------------------------
 
 if os.name in ('nt','dos'):
     os_name = 'windows'
 else:
     os_name = os.name
 
 # Under Windows, 'sdist' has not been supported.  Now that the docs build with
 # Sphinx it might work, but let's not turn it on until someone confirms that it
 # actually works.
 if os_name == 'windows' and 'sdist' in sys.argv:
     print('The sdist command is not available under Windows.  Exiting.')
     sys.exit(1)
 
 
 #-------------------------------------------------------------------------------
 # Things related to the IPython documentation
 #-------------------------------------------------------------------------------
 
 # update the manuals when building a source dist
 if len(sys.argv) >= 2 and sys.argv[1] in ('sdist','bdist_rpm'):
 
     # List of things to be updated. Each entry is a triplet of args for
     # target_update()
     to_update = [
         (
             "docs/man/ipython.1.gz",
             ["docs/man/ipython.1"],
             "cd docs/man && python -m gzip --best ipython.1",
         ),
     ]
 
 
     [ target_update(*t) for t in to_update ]
 
 #---------------------------------------------------------------------------
 # Find all the packages, package data, and data_files
 #---------------------------------------------------------------------------
 
 data_files = find_data_files()
 
 setup_args['data_files'] = data_files
 
 #---------------------------------------------------------------------------
 # custom distutils commands
 #---------------------------------------------------------------------------
 # imports here, so they are after setuptools import if there was one
 from setuptools.command.sdist import sdist
 
 setup_args['cmdclass'] = {
     'build_py': \
             check_package_data_first(git_prebuild('IPython')),
     'sdist' : git_prebuild('IPython', sdist),
     'symlink': install_symlinked,
     'install_lib_symlink': install_lib_symlink,
     'install_scripts_sym': install_scripts_for_symlink,
     'unsymlink': unsymlink,
 }
 
 #---------------------------------------------------------------------------
 # Do the actual setup now
 #---------------------------------------------------------------------------
 
 if __name__ == "__main__":
     setup(**setup_args)

tools/github_stats.py

@@ -1,230 +1,230 b''
 #!/usr/bin/env python
 """Simple tools to query github.com and gather stats about issues.
 
 To generate a report for IPython 2.0, run:
 
     python github_stats.py --milestone 2.0 --since-tag rel-1.0.0
 """
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 
 import sys
 
 from argparse import ArgumentParser
 from datetime import datetime, timedelta
 from subprocess import check_output
 
 from gh_api import (
     get_paged_request, make_auth_header, get_pull_request, is_pull_request,
     get_milestone_id, get_issues_list, get_authors,
 )
 #-----------------------------------------------------------------------------
 # Globals
 #-----------------------------------------------------------------------------
 
 ISO8601 = "%Y-%m-%dT%H:%M:%SZ"
 PER_PAGE = 100
 
 #-----------------------------------------------------------------------------
 # Functions
 #-----------------------------------------------------------------------------
 
 def round_hour(dt):
     return dt.replace(minute=0,second=0,microsecond=0)
 
 def _parse_datetime(s):
     """Parse dates in the format returned by the Github API."""
     if s:
         return datetime.strptime(s, ISO8601)
     else:
         return datetime.fromtimestamp(0)
 
 def issues2dict(issues):
     """Convert a list of issues to a dict, keyed by issue number."""
     idict = {}
     for i in issues:
         idict[i['number']] = i
     return idict
 
 def split_pulls(all_issues, project="ipython/ipython"):
     """split a list of closed issues into non-PR Issues and Pull Requests"""
     pulls = []
     issues = []
     for i in all_issues:
         if is_pull_request(i):
             pull = get_pull_request(project, i['number'], auth=True)
             pulls.append(pull)
         else:
             issues.append(i)
     return issues, pulls
 
 
 def issues_closed_since(period=timedelta(days=365), project="ipython/ipython", pulls=False):
     """Get all issues closed since a particular point in time. period
     can either be a datetime object, or a timedelta object. In the
     latter case, it is used as a time before the present.
     """
 
     which = 'pulls' if pulls else 'issues'
 
     if isinstance(period, timedelta):
         since = round_hour(datetime.utcnow() - period)
     else:
         since = period
     url = "https://api.github.com/repos/%s/%s?state=closed&sort=updated&since=%s&per_page=%i" % (project, which, since.strftime(ISO8601), PER_PAGE)
     allclosed = get_paged_request(url, headers=make_auth_header())
     
     filtered = [ i for i in allclosed if _parse_datetime(i['closed_at']) > since ]
     if pulls:
         filtered = [ i for i in filtered if _parse_datetime(i['merged_at']) > since ]
-        # filter out PRs not against master (backports)
-        filtered = [ i for i in filtered if i['base']['ref'] == 'master' ]
+        # filter out PRs not against main (backports)
+        filtered = [i for i in filtered if i["base"]["ref"] == "main"]
     else:
         filtered = [ i for i in filtered if not is_pull_request(i) ]
     
     return filtered
 
 
 def sorted_by_field(issues, field='closed_at', reverse=False):
     """Return a list of issues sorted by closing date date."""
     return sorted(issues, key = lambda i:i[field], reverse=reverse)
 
 
 def report(issues, show_urls=False):
     """Summary report about a list of issues, printing number and title."""
     if show_urls:
         for i in issues:
             role = 'ghpull' if 'merged_at' in i else 'ghissue'
             print(u'* :%s:`%d`: %s' % (role, i['number'],
                                         i['title'].replace(u'`', u'``')))
     else:
         for i in issues:
             print(u'* %d: %s' % (i['number'], i['title'].replace(u'`', u'``')))
 
 #-----------------------------------------------------------------------------
 # Main script
 #-----------------------------------------------------------------------------
 
 if __name__ == "__main__":
 
     print("DEPRECATE: backport_pr.py is deprecated and it is now recommended"
           "to install `ghpro` from PyPI.", file=sys.stderr)
 
     
     # Whether to add reST urls for all issues in printout.
     show_urls = True
     
     parser = ArgumentParser()
     parser.add_argument('--since-tag', type=str,
         help="The git tag to use for the starting point (typically the last major release)."
     )
     parser.add_argument('--milestone', type=str,
         help="The GitHub milestone to use for filtering issues [optional]."
     )
     parser.add_argument('--days', type=int,
         help="The number of days of data to summarize (use this or --since-tag)."
     )
     parser.add_argument('--project', type=str, default="ipython/ipython",
         help="The project to summarize."
     )
     parser.add_argument('--links', action='store_true', default=False,
         help="Include links to all closed Issues and PRs in the output."
     )
     
     opts = parser.parse_args()
     tag = opts.since_tag
     
     # set `since` from days or git tag
     if opts.days:
         since = datetime.utcnow() - timedelta(days=opts.days)
     else:
         if not tag:
             tag = check_output(['git', 'describe', '--abbrev=0']).strip().decode('utf8')
         cmd = ['git', 'log', '-1', '--format=%ai', tag]
         tagday, tz = check_output(cmd).strip().decode('utf8').rsplit(' ', 1)
         since = datetime.strptime(tagday, "%Y-%m-%d %H:%M:%S")
         h = int(tz[1:3])
         m = int(tz[3:])
         td = timedelta(hours=h, minutes=m)
         if tz[0] == '-':
             since += td
         else:
             since -= td
     
     since = round_hour(since)
     
     milestone = opts.milestone
     project = opts.project
 
     print("fetching GitHub stats since %s (tag: %s, milestone: %s)" % (since, tag, milestone), file=sys.stderr)
     if milestone:
         milestone_id = get_milestone_id(project=project, milestone=milestone,
                 auth=True)
         issues_and_pulls = get_issues_list(project=project,
                 milestone=milestone_id,
                 state='closed',
                 auth=True,
         )
         issues, pulls = split_pulls(issues_and_pulls, project=project)
     else:
         issues = issues_closed_since(since, project=project, pulls=False)
         pulls = issues_closed_since(since, project=project, pulls=True)
     
     # For regular reports, it's nice to show them in reverse chronological order
     issues = sorted_by_field(issues, reverse=True)
     pulls = sorted_by_field(pulls, reverse=True)
     
     n_issues, n_pulls = map(len, (issues, pulls))
     n_total = n_issues + n_pulls
     
     # Print summary report we can directly include into release notes.
     
     print()
     since_day = since.strftime("%Y/%m/%d")
     today = datetime.today().strftime("%Y/%m/%d")
     print("GitHub stats for %s - %s (tag: %s)" % (since_day, today, tag))
     print()
     print("These lists are automatically generated, and may be incomplete or contain duplicates.")
     print()
     
     ncommits = 0
     all_authors = []
     if tag:
         # print git info, in addition to GitHub info:
         since_tag = tag+'..'
         cmd = ['git', 'log', '--oneline', since_tag]
         ncommits += len(check_output(cmd).splitlines())
         
         author_cmd = ['git', 'log', '--use-mailmap', "--format=* %aN", since_tag]
         all_authors.extend(check_output(author_cmd).decode('utf-8', 'replace').splitlines())
     
     pr_authors = []
     for pr in pulls:
         pr_authors.extend(get_authors(pr))
     ncommits = len(pr_authors) + ncommits - len(pulls)
     author_cmd = ['git', 'check-mailmap'] + pr_authors
     with_email = check_output(author_cmd).decode('utf-8', 'replace').splitlines()
     all_authors.extend([ u'* ' + a.split(' <')[0] for a in with_email ])
     unique_authors = sorted(set(all_authors), key=lambda s: s.lower())
 
     print("We closed %d issues and merged %d pull requests." % (n_issues, n_pulls))
     if milestone:
         print("The full list can be seen `on GitHub <https://github.com/{project}/issues?q=milestone%3A{milestone}>`__".format(project=project,milestone=milestone)
         )
     
     print()
     print("The following %i authors contributed %i commits." % (len(unique_authors), ncommits))
     print()
     print('\n'.join(unique_authors))
     
     if opts.links:
         print()
         print("GitHub issues and pull requests:")
         print()
         print('Pull Requests (%d):\n' % n_pulls)
         report(pulls, show_urls)
         print()
         print('Issues (%d):\n' % n_issues)
         report(issues, show_urls)

tools/release_helper.sh

@@ -1,251 +1,251 b''
 # Simple tool to help for release
 # when releasing with bash, simple source it to get asked questions. 
 
 # misc check before starting
 
 python -c 'import keyring'
 python -c 'import twine'
 python -c 'import sphinx'
 python -c 'import sphinx_rtd_theme'
 python -c 'import pytest'
 
 
 BLACK=$(tput setaf 1)
 RED=$(tput setaf 1)
 GREEN=$(tput setaf 2)
 YELLOW=$(tput setaf 3)
 BLUE=$(tput setaf 4)
 MAGENTA=$(tput setaf 5)
 CYAN=$(tput setaf 6)
 WHITE=$(tput setaf 7)
 NOR=$(tput sgr0)
 
 
 echo "Will use $BLUE'$EDITOR'$NOR to edit files when necessary"
 echo -n "PREV_RELEASE (X.y.z) [$PREV_RELEASE]: "
 read input
 PREV_RELEASE=${input:-$PREV_RELEASE}
 echo -n "MILESTONE (X.y) [$MILESTONE]: "
 read input
 MILESTONE=${input:-$MILESTONE}
 echo -n "VERSION (X.y.z) [$VERSION]:"
 read input
 VERSION=${input:-$VERSION}
-echo -n "BRANCH (master|X.y) [$BRANCH]:"
+echo -n "BRANCH (main|X.y) [$BRANCH]:"
 read input
 BRANCH=${input:-$BRANCH}
 
 ask_section(){
     echo
     echo $BLUE"$1"$NOR
     echo -n $GREEN"Press Enter to continue, S to skip: "$NOR
     if [ "$ZSH_NAME" = "zsh" ] ; then
         read -k1 value
         value=${value%$'\n'}
     else
        read -n1 value
     fi
     if [ -z "$value" ] || [ $value = 'y' ]; then
         return 0
     fi
     return 1
 }
 
 
 maybe_edit(){
     echo
     echo $BLUE"$1"$NOR 
     echo -n $GREEN"Press ${BLUE}e$GREEN to Edit ${BLUE}$1$GREEN, any other keys to skip: "$NOR
     if [ "$ZSH_NAME" = "zsh" ] ; then
         read -k1 value
         value=${value%$'\n'}
     else
        read -n1 value
     fi
 
     echo 
     if [ $value = 'e' ]  ; then
         $=EDITOR $1
     fi
 }
 
 
 
 echo 
 if ask_section "Updating what's new with information from docs/source/whatsnew/pr"
 then
     python tools/update_whatsnew.py
 
     echo
     echo $BLUE"please move the contents of "docs/source/whatsnew/development.rst" to version-X.rst"$NOR
     echo $GREEN"Press enter to continue"$NOR
     read
 fi
 
 if ask_section "Gen Stats, and authors"
 then
 
     echo 
     echo $BLUE"here are all the authors that contributed to this release:"$NOR
     git log --format="%aN <%aE>" $PREV_RELEASE... | sort -u -f
 
     echo 
     echo $BLUE"If you see any duplicates cancel (Ctrl-C), then edit .mailmap."
     echo $GREEN"Press enter to continue:"$NOR
     read
 
     echo $BLUE"generating stats"$NOR
     python tools/github_stats.py --milestone $MILESTONE > stats.rst
 
     echo $BLUE"stats.rst files generated."$NOR
     echo $GREEN"Please merge it with the right file (github-stats-X.rst) and commit."$NOR
     echo $GREEN"press enter to continue."$NOR
     read
 
 fi
 
 if ask_section "Generate API difference (using frapuccino)"
 then
     echo $BLUE"Checking out $PREV_RELEASE"$NOR
     git checkout $PREV_RELEASE
     sleep 1
     echo $BLUE"Saving API to file $PREV_RELEASE"$NOR
     frappuccino IPython IPython.kernel IPython.lib IPython.qt IPython.lib.kernel IPython.html IPython.frontend IPython.external --save IPython-$PREV_RELEASE.json
     echo $BLUE"coming back to $BRANCH"$NOR
     git checkout $BRANCH
     sleep 1
     echo $BLUE"comparing ..."$NOR
     frappuccino IPython IPython.kernel IPython.lib --compare IPython-$PREV_RELEASE.json
     echo $GREEN"Use the above guideline to write an API changelog ..."$NOR
     echo $GREEN"Press any keys to continue"$NOR
     read
 fi
 
 echo "Cleaning repository"
 git clean -xfdi
 
 echo $GREEN"please update version number in ${RED}IPython/core/release.py${NOR} , Do not commit yet – we'll do it later."$NOR
 echo $GREEN"I tried ${RED}sed -i bkp -e '/Uncomment/s/^# //g' IPython/core/release.py${NOR}"
 sed -i bkp -e '/Uncomment/s/^# //g' IPython/core/release.py
 rm IPython/core/release.pybkp
 git diff | cat
 maybe_edit IPython/core/release.py
 
 echo $GREEN"Press enter to continue"$NOR
 read
 
 if ask_section "Build the documentation ?"
 then
     make html -C docs
     echo 
     echo $GREEN"Check the docs, press enter to continue"$NOR
     read
 
 fi
 
 if ask_section "Should we commit, tag, push... etc ? "
 then
    echo
    echo $BLUE"Let's commit : git commit -am \"release $VERSION\" -S"
    echo $GREEN"Press enter to commit"$NOR
    read
    git commit -am "release $VERSION" -S
    
    echo
    echo $BLUE"git push origin \$BRANCH ($BRANCH)?"$NOR
    echo $GREEN"Make sure you can push"$NOR
    echo $GREEN"Press enter to continue"$NOR
    read
    git push origin $BRANCH
    
    echo
    echo "Let's tag : git tag -am \"release $VERSION\" \"$VERSION\" -s"
    echo $GREEN"Press enter to tag commit"$NOR
    read
    git tag -am "release $VERSION" "$VERSION" -s
    
    echo
    echo $BLUE"And push the tag: git push origin \$VERSION ?"$NOR
    echo $GREEN"Press enter to continue"$NOR
    read
    git push origin $VERSION
    
    
    echo $GREEN"please update version number and back to .dev in ${RED}IPython/core/release.py"
    echo $GREEN"I tried ${RED}sed -i bkp -e '/Uncomment/s/^/# /g' IPython/core/release.py${NOR}"
    sed -i bkp -e '/Uncomment/s/^/# /g' IPython/core/release.py
    rm IPython/core/release.pybkp
    git diff | cat
    echo $GREEN"Please bump ${RED}the minor version number${NOR}"
    maybe_edit IPython/core/release.py
    echo ${BLUE}"Do not commit yet – we'll do it later."$NOR
 
    
    echo $GREEN"Press enter to continue"$NOR
    read
    
    echo
    echo "Let's commit : "$BLUE"git commit -am \"back to dev\""$NOR
    echo $GREEN"Press enter to commit"$NOR
    read
    git commit -am "back to dev"
 
    echo
    echo $BLUE"git push origin \$BRANCH ($BRANCH)?"$NOR
    echo $GREEN"Press enter to continue"$NOR
    read
    git push origin $BRANCH
 
    
    echo
    echo $BLUE"let's : git checkout $VERSION"$NOR
    echo $GREEN"Press enter to continue"$NOR
    read
    git checkout $VERSION
 fi
 
 if ask_section "Should we build and release ?"
 then
     
     echo $BLUE"going to set SOURCE_DATE_EPOCH"$NOR
     echo $BLUE'export SOURCE_DATE_EPOCH=$(git show -s --format=%ct HEAD)'$NOR
     echo $GREEN"Press enter to continue"$NOR
     read
 
     export SOURCE_DATE_EPOCH=$(git show -s --format=%ct HEAD)
 
     echo $BLUE"SOURCE_DATE_EPOCH set to $SOURCE_DATE_EPOCH"$NOR
     echo $GREEN"Press enter to continue"$NOR
     read
 
 
 
     echo
     echo $BLUE"Attempting to build package..."$NOR
 
     tools/release
 
 
     echo $RED'$ shasum -a 256 dist/*'
     shasum -a 256 dist/*
     echo $NOR
 
     echo $BLUE"We are going to rebuild, node the hash above, and compare them to the rebuild"$NOR
     echo $GREEN"Press enter to continue"$NOR
     read
 
     echo
     echo $BLUE"Attempting to build package..."$NOR
 
     tools/release
 
     echo $RED"Check the shasum for SOURCE_DATE_EPOCH=$SOURCE_DATE_EPOCH"
     echo $RED'$ shasum -a 256 dist/*'
     shasum -a 256 dist/*
     echo $NOR
 
     if ask_section "upload packages ?"
     then 
        tools/release upload
     fi
 fi

tools/retar.py

@@ -1,85 +1,87 b''
 """
 Un-targz and retargz a targz file to ensure reproducible build. 
 
 usage: 
 
     $ export SOURCE_DATE_EPOCH=$(date +%s)
+    # or
+    $ export SOURCE_DATE_EPOCH=$(git show -s --format=%ct HEAD)
     ...
     $ python retar.py <tarfile.gz>
 
 The process of creating an sdist can be non-reproducible:
   - directory created during the process get a mtime of the creation date; 
   - gziping files embed the timestamp of zip creation. 
 
 This will untar-retar; ensuring that all mtime > SOURCE_DATE_EPOCH will be set
 equal to SOURCE_DATE_EPOCH.
 
 """
 
 import tarfile
 import sys
 import os
 import gzip
 import io
 
 from pathlib import Path
 
 if len(sys.argv) > 2:
     raise ValueError("Too many arguments")
 
 
 timestamp = int(os.environ["SOURCE_DATE_EPOCH"])
 
 path = Path(sys.argv[1])
 old_buf = io.BytesIO()
 with open(path, "rb") as f:
     old_buf.write(f.read())
 old_buf.seek(0)
 if path.name.endswith("gz"):
     r_mode = "r:gz"
 if path.name.endswith("bz2"):
     r_mode = "r:bz2"
 if path.name.endswith("xz"):
     raise ValueError("XZ is deprecated but it's written nowhere")
 old = tarfile.open(fileobj=old_buf, mode=r_mode)
 
 buf = io.BytesIO()
 new = tarfile.open(fileobj=buf, mode="w", format=tarfile.GNU_FORMAT)
 for i, m in enumerate(old):
     data = None
     # mutation does not work, copy
     if m.name.endswith('.DS_Store'):
         continue
     m2 = tarfile.TarInfo(m.name)
     m2.mtime = min(timestamp, m.mtime)
     m2.pax_headers["mtime"] = m2.mtime
     m2.size = m.size
     m2.type = m.type
     m2.linkname = m.linkname
     m2.mode = m.mode
     if m.isdir():
         new.addfile(m2)
     else:
         data = old.extractfile(m)
         new.addfile(m2, data)
 new.close()
 old.close()
 
 buf.seek(0)
 
 if r_mode == "r:gz":
     with open(path, "wb") as f:
         with gzip.GzipFile("", "wb", fileobj=f, mtime=timestamp) as gzf:
             gzf.write(buf.read())
 elif r_mode == "r:bz2":
     import bz2
 
     with bz2.open(path, "wb") as f:
         f.write(buf.read())
 
 else:
     assert False
 
 # checks the archive is valid.
 archive = tarfile.open(path)
 names = archive.getnames()