Merge branch 'ipython:master' into master

SECURITY.md

@@ -0,0 +1,6 b''
+# Security Policy
+
+## Reporting a Vulnerability
+
+All IPython and Jupyter security are handled via [e-mail]. 
+You can find more informations on the Jupyter website. https://jupyter.org/security

.github/workflows/test.yml

@@ -2,6 +2,10 b' name: Run tests'
 
 on:
   push:
+    branches:
+      - main
+      - master
+      - '*.x'
   pull_request:
   # Run weekly on Monday at 1:23 UTC
   schedule:

IPython/core/display.py

@@ -83,7 +83,7 b' def _display_mimetype(mimetype, objs, raw=False, metadata=None):'
     if raw:
         # turn list of pngdata into list of { 'image/png': pngdata }
         objs = [ {mimetype: obj} for obj in objs ]
-    display(*objs, raw=raw, metadata=metadata, include=[mimetype])
+    display_functions.display(*objs, raw=raw, metadata=metadata, include=[mimetype])
 
 #-----------------------------------------------------------------------------
 # Main functions
@@ -517,10 +517,10 b' class ProgressBar(DisplayObject):'
             self.html_width, self.total, self.progress)
 
     def display(self):
-        display(self, display_id=self._display_id)
+        display_functions.display(self, display_id=self._display_id)
 
     def update(self):
-        display(self, display_id=self._display_id, update=True)
+        display_functions.display(self, display_id=self._display_id, update=True)
 
     @property
     def progress(self):
@@ -694,7 +694,7 b' class GeoJSON(JSON):'
         metadata = {
             'application/geo+json': self.metadata
         }
-        display(bundle, metadata=metadata, raw=True)
+        display_functions.display(bundle, metadata=metadata, raw=True)
 
 class Javascript(TextDisplayObject):
 

IPython/core/inputtransformer2.py

@@ -507,9 +507,12 b' def make_tokens_by_line(lines:List[str]):'
 
     #   reexported from token on 3.7+
     NEWLINE, NL = tokenize.NEWLINE, tokenize.NL  # type: ignore
-    tokens_by_line:List[List[Any]] = [[]]
-    if len(lines) > 1 and not lines[0].endswith(('\n', '\r', '\r\n', '\x0b', '\x0c')):
-        warnings.warn("`make_tokens_by_line` received a list of lines which do not have lineending markers ('\\n', '\\r', '\\r\\n', '\\x0b', '\\x0c'), behavior will be unspecified")
+    tokens_by_line: List[List[Any]] = [[]]
+    if len(lines) > 1 and not lines[0].endswith(("\n", "\r", "\r\n", "\x0b", "\x0c")):
+        warnings.warn(
+            "`make_tokens_by_line` received a list of lines which do not have lineending markers ('\\n', '\\r', '\\r\\n', '\\x0b', '\\x0c'), behavior will be unspecified",
+            stacklevel=2,
+        )
     parenlev = 0
     try:
         for token in tokenize.generate_tokens(iter(lines).__next__):
@@ -782,9 +785,6 b' class MaybeAsyncCompile(Compile):'
         super().__init__()
         self.flags |= extra_flags
 
-    def __call__(self, *args, **kwds):
-        return compile(*args, **kwds)
-
 
 class MaybeAsyncCommandCompiler(CommandCompiler):
     def __init__(self, extra_flags=0):

IPython/core/magic_arguments.py

@@ -37,6 +37,38 b' arguments::'
           -o OPTION, --option OPTION
                                 An optional argument.
 
+Here is an elaborated example that uses default parameters in `argument` and calls the `args` in the cell magic::
+
+    from IPython.core.magic import register_cell_magic
+    from IPython.core.magic_arguments import (argument, magic_arguments,
+                                            parse_argstring)
+
+
+    @magic_arguments()
+    @argument(
+        "--option",
+        "-o",
+        help=("Add an option here"),
+    )
+    @argument(
+        "--style",
+        "-s",
+        default="foo",
+        help=("Add some style arguments"),
+    )
+    @register_cell_magic
+    def my_cell_magic(line, cell):
+        args = parse_argstring(my_cell_magic, line)
+        print(f"{args.option=}")
+        print(f"{args.style=}")
+        print(f"{cell=}")
+
+In a jupyter notebook, this cell magic can be executed like this::
+
+    %%my_cell_magic -o Hello
+    print("bar")
+    i = 42
+
 Inheritance diagram:
 
 .. inheritance-diagram:: IPython.core.magic_arguments

IPython/core/tests/test_inputtransformer2.py

@@ -4,6 +4,7 b' Line-based transformers are the simpler ones; token-based transformers are'
 more complex. See test_inputtransformer2_line for tests for line-based
 transformations.
 """
+import platform
 import string
 import sys
 from textwrap import dedent
@@ -291,6 +292,7 b' def test_check_complete_param(code, expected, number):'
     assert cc(code) == (expected, number)
 
 
[e-mail](platform.python_implementation() == "PyPy", reason="fail on pypy")
 @pytest.mark.xfail(
     reason="Bug in python 3.9.8 – bpo 45738",
     condition=sys.version_info in [(3, 9, 8, "final", 0), (3, 11, 0, "alpha", 2)],
@@ -319,7 +321,16 b' def test_check_complete():'
     assert cc("def f():\n  x=0\n  \\\n  ") == ("incomplete", 2)
 
 
-def test_check_complete_II():
[e-mail](platform.python_implementation() == "PyPy", reason="fail on pypy")
[e-mail](
+    "value, expected",
+    [
+        ('''def foo():\n    """''', ("incomplete", 4)),
+        ("""async with example:\n    pass""", ("incomplete", 4)),
+        ("""async with example:\n    pass\n    """, ("complete", None)),
+    ],
+)
+def test_check_complete_II(value, expected):
     """
     Test that multiple line strings are properly handled.
 
@@ -327,25 +338,31 b' def test_check_complete_II():'
 
     """
     cc = ipt2.TransformerManager().check_complete
-    assert cc('''def foo():\n    """''') == ("incomplete", 4)
-
-
-def test_check_complete_invalidates_sunken_brackets():
+    assert cc(value) == expected
+
+
[e-mail](
+    "value, expected",
+    [
+        (")", ("invalid", None)),
+        ("]", ("invalid", None)),
+        ("}", ("invalid", None)),
+        (")(", ("invalid", None)),
+        ("][", ("invalid", None)),
+        ("}{", ("invalid", None)),
+        ("]()(", ("invalid", None)),
+        ("())(", ("invalid", None)),
+        (")[](", ("invalid", None)),
+        ("()](", ("invalid", None)),
+    ],
+)
+def test_check_complete_invalidates_sunken_brackets(value, expected):
     """
     Test that a single line with more closing brackets than the opening ones is
     interpreted as invalid
     """
     cc = ipt2.TransformerManager().check_complete
-    assert cc(")") == ("invalid", None)
-    assert cc("]") == ("invalid", None)
-    assert cc("}") == ("invalid", None)
-    assert cc(")(") == ("invalid", None)
-    assert cc("][") == ("invalid", None)
-    assert cc("}{") == ("invalid", None)
-    assert cc("]()(") == ("invalid", None)
-    assert cc("())(") == ("invalid", None)
-    assert cc(")[](") == ("invalid", None)
-    assert cc("()](") == ("invalid", None)
+    assert cc(value) == expected
 
 
 def test_null_cleanup_transformer():

IPython/utils/tests/test_path.py

@@ -406,13 +406,18 b' class TestShellGlob(unittest.TestCase):'
             self.check_match(patterns, matches)
 
 
-# TODO : pytest.mark.parametrise once nose is gone.
-def test_unescape_glob():
-    assert path.unescape_glob(r"\*\[\!\]\?") == "*[!]?"
-    assert path.unescape_glob(r"\\*") == r"\*"
-    assert path.unescape_glob(r"\\\*") == r"\*"
-    assert path.unescape_glob(r"\\a") == r"\a"
-    assert path.unescape_glob(r"\a") == r"\a"
[e-mail](
+    "globstr, unescaped_globstr",
+    [
+        (r"\*\[\!\]\?", "*[!]?"),
+        (r"\\*", r"\*"),
+        (r"\\\*", r"\*"),
+        (r"\\a", r"\a"),
+        (r"\a", r"\a"),
+    ],
+)
+def test_unescape_glob(globstr, unescaped_globstr):
+    assert path.unescape_glob(globstr) == unescaped_globstr
 
 
 @onlyif_unicode_paths

IPython/utils/tests/test_process.py

@@ -1,4 +1,3 b''
-# encoding: utf-8
 """
 Tests for platutils.py
 """
@@ -56,35 +55,37 b' def test_find_cmd_fail():'
     pytest.raises(FindCmdError, find_cmd, "asdfasdf")
 
 
-# TODO: move to pytest.mark.parametrize once nose gone
 @dec.skip_win32
-def test_arg_split():
[e-mail](
+    "argstr, argv",
+    [
+        ("hi", ["hi"]),
+        ("hello there", ["hello", "there"]),
+        # \u01ce == \N{LATIN SMALL LETTER A WITH CARON}
+        # Do not use \N because the tests crash with syntax error in
+        # some cases, for example windows python2.6.
+        ("h\u01cello", ["h\u01cello"]),
+        ('something "with quotes"', ["something", '"with quotes"']),
+    ],
+)
+def test_arg_split(argstr, argv):
     """Ensure that argument lines are correctly split like in a shell."""
-    tests = [['hi', ['hi']],
-             [u'hi', [u'hi']],
-             ['hello there', ['hello', 'there']],
-             # \u01ce == \N{LATIN SMALL LETTER A WITH CARON}
-             # Do not use \N because the tests crash with syntax error in
-             # some cases, for example windows python2.6.
-             [u'h\u01cello', [u'h\u01cello']],
-             ['something "with quotes"', ['something', '"with quotes"']],
-             ]
-    for argstr, argv in tests:
-        assert arg_split(argstr) == argv
-
-
-# TODO: move to pytest.mark.parametrize once nose gone
+    assert arg_split(argstr) == argv
+
+
 @dec.skip_if_not_win32
-def test_arg_split_win32():
[e-mail](
+    "argstr,argv",
+    [
+        ("hi", ["hi"]),
+        ("hello there", ["hello", "there"]),
+        ("h\u01cello", ["h\u01cello"]),
+        ('something "with quotes"', ["something", "with quotes"]),
+    ],
+)
+def test_arg_split_win32(argstr, argv):
     """Ensure that argument lines are correctly split like in a shell."""
-    tests = [['hi', ['hi']],
-             [u'hi', [u'hi']],
-             ['hello there', ['hello', 'there']],
-             [u'h\u01cello', [u'h\u01cello']],
-             ['something "with quotes"', ['something', 'with quotes']],
-             ]
-    for argstr, argv in tests:
-        assert arg_split(argstr) == argv
+    assert arg_split(argstr) == argv
 
 
 class SubProcessTestCase(tt.TempFileMixin):
@@ -98,7 +99,7 b' class SubProcessTestCase(tt.TempFileMixin):'
         self.mktmp('\n'.join(lines))
 
     def test_system(self):
-        status = system('%s "%s"' % (python, self.fname))
+        status = system(f'{python} "{self.fname}"')
         self.assertEqual(status, 0)
 
     def test_system_quotes(self):
@@ -145,11 +146,11 b' class SubProcessTestCase(tt.TempFileMixin):'
 
         status = self.assert_interrupts(command)
         self.assertNotEqual(
-            status, 0, "The process wasn't interrupted. Status: %s" % (status,)
+            status, 0, f"The process wasn't interrupted. Status: {status}"
         )
 
     def test_getoutput(self):
-        out = getoutput('%s "%s"' % (python, self.fname))
+        out = getoutput(f'{python} "{self.fname}"')
         # we can't rely on the order the line buffered streams are flushed
         try:
             self.assertEqual(out, 'on stderron stdout')
@@ -169,7 +170,7 b' class SubProcessTestCase(tt.TempFileMixin):'
         self.assertEqual(out.strip(), '1')
 
     def test_getoutput_error(self):
-        out, err = getoutputerror('%s "%s"' % (python, self.fname))
+        out, err = getoutputerror(f'{python} "{self.fname}"')
         self.assertEqual(out, 'on stdout')
         self.assertEqual(err, 'on stderr')
 
@@ -179,7 +180,7 b' class SubProcessTestCase(tt.TempFileMixin):'
         self.assertEqual(out, '')
         self.assertEqual(err, '')
         self.assertEqual(code, 1)
-        out, err, code = get_output_error_code('%s "%s"' % (python, self.fname))
+        out, err, code = get_output_error_code(f'{python} "{self.fname}"')
         self.assertEqual(out, 'on stdout')
         self.assertEqual(err, 'on stderr')
         self.assertEqual(code, 0)

IPython/utils/tests/test_text.py

@@ -80,24 +80,22 b' def test_columnize_random():'
             )
 
 
-# TODO: pytest mark.parametrize once nose removed.
-def test_columnize_medium():
[e-mail]("row_first", [True, False])
+def test_columnize_medium(row_first):
     """Test with inputs than shouldn't be wider than 80"""
     size = 40
     items = [l*size for l in 'abc']
-    for row_first in [True, False]:
-        out = text.columnize(items, row_first=row_first, displaywidth=80)
-        assert out == "\n".join(items + [""]), "row_first={0}".format(row_first)
+    out = text.columnize(items, row_first=row_first, displaywidth=80)
+    assert out == "\n".join(items + [""]), "row_first={0}".format(row_first)
 
 
-# TODO: pytest mark.parametrize once nose removed.
-def test_columnize_long():
[e-mail]("row_first", [True, False])
+def test_columnize_long(row_first):
     """Test columnize with inputs longer than the display window"""
     size = 11
     items = [l*size for l in 'abc']
-    for row_first in [True, False]:
-        out = text.columnize(items, row_first=row_first, displaywidth=size - 1)
-        assert out == "\n".join(items + [""]), "row_first={0}".format(row_first)
+    out = text.columnize(items, row_first=row_first, displaywidth=size - 1)
+    assert out == "\n".join(items + [""]), "row_first={0}".format(row_first)
 
 
 def eval_formatter_check(f):

MANIFEST.in

@@ -14,6 +14,7 b' recursive-exclude tools *'
 exclude tools
 exclude CONTRIBUTING.md
 exclude .editorconfig
+exclude SECURITY.md
 
 graft scripts
 

docs/source/development/config.rst

@@ -81,8 +81,8 b' profile with:'
     $ ipython locate profile foo
     /home/you/.ipython/profile_foo
 
-These map to the utility functions: :func:`IPython.utils.path.get_ipython_dir`
-and :func:`IPython.utils.path.locate_profile` respectively.
+These map to the utility functions: :func:`IPython.paths.get_ipython_dir`
+and :func:`IPython.paths.locate_profile` respectively.
 
 
 .. _profiles_dev:

docs/source/whatsnew/version8.rst

@@ -48,7 +48,7 b' or have been turned into explicit errors for better error messages.'
 I will use this occasion to add the following requests to anyone emitting a
 deprecation warning:
 
- - Please use at least ``stacklevel=2`` so that the warning is emitted into the
+ - Please add at least ``stacklevel=2`` so that the warning is emitted into the
    caller context, and not the callee one.
  - Please add **since which version** something is deprecated.
 

setup.cfg

@@ -15,6 +15,7 b' keywords = Interactive, Interpreter, Shell, Embedding'
 platforms = Linux, Mac OSX, Windows
 classifiers =
     Framework :: IPython
+    Framework :: Jupyter
     Intended Audience :: Developers
     Intended Audience :: Science/Research
     License :: OSI Approved :: BSD License
@@ -23,26 +24,69 b' classifiers ='
     Programming Language :: Python :: 3 :: Only
     Topic :: System :: Shells
 
-
 [options]
 packages = find:
 python_requires = >=3.8
 zip_safe = False
 install_requires =
-    setuptools>=18.5
-    jedi>=0.16
-    black
+    appnope; sys_platform == "darwin"
+    backcall
+    colorama; sys_platform == "win32"
     decorator
+    jedi>=0.16
+    matplotlib-inline
+    pexpect>4.3; sys_platform != "win32"
     pickleshare
-    traitlets>=5
     prompt_toolkit>=2.0.0,<3.1.0,!=3.0.0,!=3.0.1
     pygments>=2.4.0
-    backcall
+    setuptools>=18.5
     stack_data
-    matplotlib-inline
-    pexpect>4.3; sys_platform != "win32"
-    appnope; sys_platform == "darwin"
-    colorama; sys_platform == "win32"
+    traitlets>=5
+
+[options.extras_require]
+black =
+    black
+doc =
+    Sphinx>=1.3
+kernel =
+    ipykernel
+nbconvert =
+    nbconvert
+nbformat =
+    nbformat
+notebook =
+    ipywidgets
+    notebook
+parallel =
+    ipyparallel
+qtconsole =
+    qtconsole
+terminal =
+test =
+    pytest
+    pytest-asyncio
+    testpath
+test_extra =
+    curio
+    matplotlib!=3.2.0
+    nbformat
+    numpy>=1.19
+    pandas
+    pytest
+    testpath
+    trio
+all =
+    %(black)s
+    %(doc)s
+    %(kernel)s
+    %(nbconvert)s
+    %(nbformat)s
+    %(notebook)s
+    %(parallel)s
+    %(qtconsole)s
+    %(terminal)s
+    %(test_extra)s
+    %(test)s
 
 [options.packages.find]
 exclude =
@@ -64,7 +108,7 b' pygments.lexers ='
     ipython3 = IPython.lib.lexers:IPython3Lexer
 
 [velin]
-ignore_patterns = 
+ignore_patterns =
    IPython/core/tests
    IPython/testing
 

setup.py

@@ -137,46 +137,6 b" setup_args['cmdclass'] = {"
     'unsymlink': unsymlink,
 }
 
-
-#---------------------------------------------------------------------------
-# Handle scripts, dependencies, and setuptools specific things
-#---------------------------------------------------------------------------
-
-# setuptools requirements
-
-extras_require = dict(
-    parallel=["ipyparallel"],
-    qtconsole=["qtconsole"],
-    doc=["Sphinx>=1.3"],
-    test=[
-        "pytest",
-        "pytest-asyncio",
-        "testpath",
-        "pygments>=2.4.0",
-    ],
-    test_extra=[
-        "pytest",
-        "testpath",
-        "curio",
-        "matplotlib!=3.2.0",
-        "nbformat",
-        "numpy>=1.19",
-        "pandas",
-        "pygments>=2.4.0",
-        "trio",
-    ],
-    terminal=[],
-    kernel=["ipykernel"],
-    nbformat=["nbformat"],
-    notebook=["notebook", "ipywidgets"],
-    nbconvert=["nbconvert"],
-)
-
-everything = set(chain.from_iterable(extras_require.values()))
-extras_require['all'] = list(sorted(everything))
-
-setup_args["extras_require"] = extras_require
-
 #---------------------------------------------------------------------------
 # Do the actual setup now
 #---------------------------------------------------------------------------