Merge branch 'master' into dev

IPython/core/debugger.py

@@ -290,7 +290,7 b' class Pdb(OldPdb):'
 
     def hidden_frames(self, stack):
         """
-        Given an index in the stack return wether it should be skipped.
+        Given an index in the stack return whether it should be skipped.
 
         This is used in up/down and where to skip frames.
         """
@@ -713,7 +713,9 b' class Pdb(OldPdb):'
                     break
             else:
                 # if no break occured.
-                self.error("all frames above hidden")
+                self.error(
+                    "all frames above hidden, use `skip_hidden False` to get get into those."
+                )
                 return
 
             Colors = self.color_scheme_table.active_colors
@@ -756,7 +758,9 b' class Pdb(OldPdb):'
                 if counter >= count:
                     break
             else:
-                self.error("all frames bellow hidden")
+                self.error(
+                    "all frames bellow hidden, use `skip_hidden False` to get get into those."
+                )
                 return
 
             Colors = self.color_scheme_table.active_colors

IPython/core/display.py

@@ -1086,7 +1086,7 b' class Video(DisplayObject):'
         if url is None and isinstance(data, str) and data.startswith(('http:', 'https:')):
             url = data
             data = None
-        elif os.path.exists(data):
+        elif data is not None and os.path.exists(data):
             filename = data
             data = None
 

IPython/core/interactiveshell.py

@@ -757,6 +757,7 b' class InteractiveShell(SingletonConfigurable):'
         self.meta = Struct()
 
         # Temporary files used for various purposes.  Deleted at exit.
+        # The files here are stored with Path from Pathlib
         self.tempfiles = []
         self.tempdirs = []
 
@@ -3595,16 +3596,17 b' class InteractiveShell(SingletonConfigurable):'
           - data(None): if data is given, it gets written out to the temp file
             immediately, and the file is closed again."""
 
-        dirname = tempfile.mkdtemp(prefix=prefix)
-        self.tempdirs.append(dirname)
+        dir_path = Path(tempfile.mkdtemp(prefix=prefix))
+        self.tempdirs.append(dir_path)
 
-        handle, filename = tempfile.mkstemp('.py', prefix, dir=dirname)
+        handle, filename = tempfile.mkstemp(".py", prefix, dir=str(dir_path))
         os.close(handle)  # On Windows, there can only be one open handle on a file
-        self.tempfiles.append(filename)
+
+        file_path = Path(filename)
+        self.tempfiles.append(file_path)
 
         if data:
-            with open(filename, 'w') as tmp_file:
-                tmp_file.write(data)
+            file_path.write_text(data)
         return filename
 
     @undoc
@@ -3761,14 +3763,14 b' class InteractiveShell(SingletonConfigurable):'
         # Cleanup all tempfiles and folders left around
         for tfile in self.tempfiles:
             try:
-                os.unlink(tfile)
-            except OSError:
+                tfile.unlink()
+            except FileNotFoundError:
                 pass
 
         for tdir in self.tempdirs:
             try:
-                os.rmdir(tdir)
-            except OSError:
+                tdir.rmdir()
+            except FileNotFoundError:
                 pass
 
         # Clear all user namespaces to release all references cleanly.

IPython/core/magics/execution.py

@@ -40,6 +40,7 b' from IPython.utils.timing import clock, clock2'
 from warnings import warn
 from logging import error
 from io import StringIO
+from pathlib import Path
 
 if sys.version_info > (3,8):
     from ast import Module
@@ -362,8 +363,7 b' class ExecutionMagics(Magics):'
             print('\n*** Profile stats marshalled to file',\
                   repr(dump_file)+'.',sys_exit)
         if text_file:
-            with open(text_file, 'w') as pfile:
-                pfile.write(output)
+            Path(text_file).write_text(output)
             print('\n*** Profile printout saved to text file',\
                   repr(text_file)+'.',sys_exit)
 
@@ -724,7 +724,7 b' class ExecutionMagics(Magics):'
         sys.argv = [filename] + args  # put in the proper filename
 
         if 'n' in opts:
-            name = os.path.splitext(os.path.basename(filename))[0]
+            name = Path(filename).stem
         else:
             name = '__main__'
 

IPython/core/magics/packaging.py

@@ -8,37 +8,38 b''
 #  The full license is in the file COPYING.txt, distributed with this software.
 #-----------------------------------------------------------------------------
 
-import os
 import re
 import shlex
 import sys
 
+from pathlib import Path
 from IPython.core.magic import Magics, magics_class, line_magic
 
 
 def _is_conda_environment():
     """Return True if the current Python executable is in a conda env"""
     # TODO: does this need to change on windows?
-    conda_history = os.path.join(sys.prefix, 'conda-meta', 'history')
-    return os.path.exists(conda_history)
+    return Path(sys.prefix, "conda-meta", "history").exists()
 
 
 def _get_conda_executable():
     """Find the path to the conda executable"""
     # Check if there is a conda executable in the same directory as the Python executable.
     # This is the case within conda's root environment.
-    conda = os.path.join(os.path.dirname(sys.executable), 'conda')
-    if os.path.isfile(conda):
-        return conda
+    conda = Path(sys.executable).parent / "conda"
+    if conda.isfile():
+        return str(conda)
 
     # Otherwise, attempt to extract the executable from conda history.
     # This applies in any conda environment.
-    R = re.compile(r"^#\s*cmd:\s*(?P<command>.*conda)\s[create|install]")
-    with open(os.path.join(sys.prefix, 'conda-meta', 'history')) as f:
-        for line in f:
-            match = R.match(line)
-            if match:
-                return match.groupdict()['command']
+    history = Path(sys.prefix, "conda-meta", "history").read_text()
+    match = re.search(
+        r"^#\s*cmd:\s*(?P<command>.*conda)\s[create|install]",
+        history,
+        flags=re.MULTILINE,
+    )
+    if match:
+        return match.groupdict()["command"]
     
     # Fallback: assume conda is available on the system path.
     return "conda"

IPython/core/tests/test_async_helpers.py

@@ -1,5 +1,5 b''
 """
-Test for async helpers. 
+Test for async helpers.
 
 Should only trigger on python 3.5+ or will have syntax errors.
 """
@@ -9,6 +9,13 b' from textwrap import dedent, indent'
 from unittest import TestCase
 from IPython.testing.decorators import skip_without
 import sys
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from IPython import get_ipython
+
+    ip = get_ipython()
+
 
 iprc = lambda x: ip.run_cell(dedent(x)).raise_error()
 iprc_nr = lambda x: ip.run_cell(dedent(x))
@@ -275,7 +282,7 b' class AsyncTest(TestCase):'
         await sleep(0.1)
         """
         )
-    
+
     if sys.version_info < (3,9):
         # new pgen parser in 3.9 does not raise MemoryError on too many nested
         # parens anymore

IPython/core/tests/test_display.py

@@ -138,7 +138,9 b' def _get_inline_config():'
     from ipykernel.pylab.config import InlineBackend
     return InlineBackend.instance()
 
[e-mail]_without('matplotlib')
+
[e-mail]_without("ipykernel")
[e-mail]_without("matplotlib")
 def test_set_matplotlib_close():
     cfg = _get_inline_config()
     cfg.close_figures = False
@@ -173,7 +175,9 b' def test_set_matplotlib_formats():'
             else:
                 nt.assert_not_in(Figure, f)
 
[e-mail]_without('matplotlib')
+
[e-mail]_without("ipykernel")
[e-mail]_without("matplotlib")
 def test_set_matplotlib_formats_kwargs():
     from matplotlib.figure import Figure
     ip = get_ipython()

IPython/core/tests/test_magic.py

@@ -14,6 +14,7 b' from unittest import TestCase'
 from unittest import mock
 from importlib import invalidate_caches
 from io import StringIO
+from pathlib import Path
 
 import nose.tools as nt
 
@@ -831,8 +832,7 b' def test_file():'
             'line1',
             'line2',
         ]))
-        with open(fname) as f:
-            s = f.read()
+        s = Path(fname).read_text()
         nt.assert_in('line1\n', s)
         nt.assert_in('line2', s)
 
@@ -846,8 +846,7 b' def test_file_single_quote():'
             'line1',
             'line2',
         ]))
-        with open(fname) as f:
-            s = f.read()
+        s = Path(fname).read_text()
         nt.assert_in('line1\n', s)
         nt.assert_in('line2', s)
 
@@ -861,8 +860,7 b' def test_file_double_quote():'
             'line1',
             'line2',
         ]))
-        with open(fname) as f:
-            s = f.read()
+        s = Path(fname).read_text()
         nt.assert_in('line1\n', s)
         nt.assert_in('line2', s)
 
@@ -876,8 +874,7 b' def test_file_var_expand():'
             'line1',
             'line2',
         ]))
-        with open(fname) as f:
-            s = f.read()
+        s = Path(fname).read_text()
         nt.assert_in('line1\n', s)
         nt.assert_in('line2', s)
 
@@ -908,8 +905,7 b' def test_file_amend():'
             'line3',
             'line4',
         ]))
-        with open(fname) as f:
-            s = f.read()
+        s = Path(fname).read_text()
         nt.assert_in('line1\n', s)
         nt.assert_in('line3\n', s)
 
@@ -922,8 +918,7 b' def test_file_spaces():'
             'line1',
             'line2',
         ]))
-        with open(fname) as f:
-            s = f.read()
+        s = Path(fname).read_text()
         nt.assert_in('line1\n', s)
         nt.assert_in('line2', s)
     
@@ -1063,15 +1058,13 b' def test_save():'
     with TemporaryDirectory() as tmpdir:
         file = os.path.join(tmpdir, "testsave.py")
         ip.run_line_magic("save", "%s 1-10" % file)
-        with open(file) as f:
-            content = f.read()
-            nt.assert_equal(content.count(cmds[0]), 1)
-            nt.assert_in('coding: utf-8', content)
+        content = Path(file).read_text()
+        nt.assert_equal(content.count(cmds[0]), 1)
+        nt.assert_in("coding: utf-8", content)
         ip.run_line_magic("save", "-a %s 1-10" % file)
-        with open(file) as f:
-            content = f.read()
-            nt.assert_equal(content.count(cmds[0]), 2)
-            nt.assert_in('coding: utf-8', content)
+        content = Path(file).read_text()
+        nt.assert_equal(content.count(cmds[0]), 2)
+        nt.assert_in("coding: utf-8", content)
 
 
 def test_store():
@@ -1231,8 +1224,7 b' def test_run_module_from_import_hook():'
     "Test that a module can be loaded via an import hook"
     with TemporaryDirectory() as tmpdir:
         fullpath = os.path.join(tmpdir, 'my_tmp.py')
-        with open(fullpath, 'w') as f:
-            f.write(TEST_MODULE)
+        Path(fullpath).write_text(TEST_MODULE)
 
         class MyTempImporter(object):
             def __init__(self):
@@ -1248,8 +1240,7 b' def test_run_module_from_import_hook():'
                 return imp.load_source('my_tmp', fullpath)
 
             def get_code(self, fullname):
-                with open(fullpath, 'r') as f:
-                    return compile(f.read(), 'foo', 'exec')
+                return compile(Path(fullpath).read_text(), "foo", "exec")
 
             def is_package(self, __):
                 return False

IPython/core/ultratb.py

@@ -699,9 +699,10 b' class VerboseTB(TBTools):'
 
         frames = []
         skipped = 0
-        for r in records:
+        lastrecord = len(records) - 1
+        for i, r in enumerate(records):
             if not isinstance(r, stack_data.RepeatedFrames) and self.skip_hidden:
-                if r.frame.f_locals.get("__tracebackhide__", 0):
+                if r.frame.f_locals.get("__tracebackhide__", 0) and i != lastrecord:
                     skipped += 1
                     continue
             if skipped:

IPython/lib/tests/test_deepreload.py

@@ -4,7 +4,7 b''
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
-import os
+from pathlib import Path
 
 import nose.tools as nt
 
@@ -12,20 +12,22 b' from IPython.utils.syspathcontext import prepended_to_syspath'
 from IPython.utils.tempdir import TemporaryDirectory
 from IPython.lib.deepreload import reload as dreload
 
+
 def test_deepreload():
     "Test that dreload does deep reloads and skips excluded modules."
     with TemporaryDirectory() as tmpdir:
         with prepended_to_syspath(tmpdir):
-            with open(os.path.join(tmpdir, 'A.py'), 'w') as f:
+            tmpdirpath = Path(tmpdir)
+            with open(tmpdirpath / "A.py", "w") as f:
                 f.write("class Object(object):\n    pass\n")
-            with open(os.path.join(tmpdir, 'B.py'), 'w') as f:
+            with open(tmpdirpath / "B.py", "w") as f:
                 f.write("import A\n")
             import A
             import B
 
             # Test that A is not reloaded.
             obj = A.Object()
-            dreload(B, exclude=['A'])
+            dreload(B, exclude=["A"])
             nt.assert_true(isinstance(obj, A.Object))
 
             # Test that A is reloaded.

docs/README.rst

@@ -16,7 +16,7 b' Requirements'
 
 The documentation must be built using Python 3.
 
-In additions to :ref:`devinstall`,
+In addition to :ref:`devinstall`,
 the following tools are needed to build the documentation:
 
  - sphinx
@@ -40,7 +40,7 b' html``.'
 
 ``make html_noapi`` - same as above, but without running the auto-generated API
 docs. When you are working on the narrative documentation, the most time
-consuming portion  of the build process is the processing and rending of the
+consuming portion  of the build process is the processing and rendering of the
 API documentation. This build target skips that.
 
 ``make pdf`` will compile a pdf from the documentation.
@@ -53,7 +53,7 b' previous docs build.'
 To remove the previous docs build you can use ``make clean``.
 You can also combine ``clean`` with other `make` commands;
 for example,
-``make clean html`` will do a complete rebuild of the docs or `make clean pdf` will do a complete build of the pdf.
+``make clean html`` will do a complete rebuild of the docs or ``make clean pdf`` will do a complete build of the pdf.
 
 
 Continuous Integration

docs/autogen_config.py

@@ -2,7 +2,7 b''
 
 from os.path import join, dirname, abspath
 import inspect
-
+from pathlib import Path
 from IPython.terminal.ipapp import TerminalIPythonApp
 from ipykernel.kernelapp import IPKernelApp
 from traitlets import Undefined
@@ -118,8 +118,7 b' def write_doc(name, title, app, preamble=None):'
 
 if __name__ == '__main__':
     # Touch this file for the make target
-    with open(generated, 'w'):
-        pass
+    Path(generated).write_text("")
 
     write_doc('terminal', 'Terminal IPython options', TerminalIPythonApp())
     write_doc('kernel', 'IPython kernel options', IPKernelApp(),

docs/autogen_magics.py

@@ -1,3 +1,4 b''
+
 from pathlib import Path
 from IPython.core.alias import Alias
 from IPython.core.interactiveshell import InteractiveShell
@@ -61,7 +62,7 b' for name, func in sorted(magics["cell"].items(), key=sortkey):'
                    format_docstring(func),
                    ""])
 
+
 src_path = Path(__file__).parent
 dest = src_path.joinpath("source", "interactive", "magics-generated.txt")
-with open(dest, "w") as f:
-    f.write("\n".join(output))
+dest.write_text("\n".join(output))

docs/source/sphinxext.rst

@@ -10,12 +10,15 b' IPython Sphinx Directive'
    The IPython Sphinx Directive is in 'beta' and currently under
    active development. Improvements to the code or documentation are welcome!
 
-The ipython directive is a stateful ipython shell for embedding in
-sphinx documents.  It knows about standard ipython prompts, and
-extracts the input and output lines.  These prompts will be renumbered
-starting at ``1``.  The inputs will be fed to an embedded ipython
-interpreter and the outputs from that interpreter will be inserted as
-well.  For example, code blocks like the following::
+.. |rst| replace:: reStructured text
+
+The :rst:dir:`ipython` directive is a stateful shell that can be used
+in |rst| files.
+
+It knows about standard ipython prompts, and extracts the input and output
+lines.  These prompts will be renumbered starting at ``1``.  The inputs will be
+fed to an embedded ipython interpreter and the outputs from that interpreter
+will be inserted as well.  For example, code blocks like the following::
 
   .. ipython::
 
@@ -42,6 +45,48 b' will be rendered as'
    document that generates the rendered output.
 
 
+Directive and options
+=====================
+
+The IPython directive takes a number of options detailed here.
+
+.. rst:directive:: ipython
+
+   Create an IPython directive.
+
+   .. rst:directive:option:: doctest
+
+      Run a doctest on IPython code blocks in rst.
+
+   .. rst:directive:option:: python
+
+      Used to indicate that the relevant code block does not have IPython prompts.
+
+   .. rst:directive:option:: okexcept
+
+      Allow the code block to raise an exception.
+
+   .. rst:directive:option:: okwarning
+
+      Allow the code block to emit an warning.
+
+   .. rst:directive:option:: suppress
+
+      Silence any warnings or expected errors.
+
+   .. rst:directive:option:: verbatim
+
+      A noop that allows for any text to be syntax highlighted as valid IPython code.
+
+   .. rst:directive:option:: savefig: OUTFILE [IMAGE_OPTIONS]
+
+      Save output from matplotlib to *outfile*.
+
+It's important to note that all of these options can be used for the entire
+directive block or they can decorate individual lines of code as explained
+in :ref:`pseudo-decorators`.
+
+
 Persisting the Python session across IPython directive blocks
 =============================================================
 
@@ -393,6 +438,8 b' Pretty much anything you can do with the ipython code, you can do with'
 with a simple python script. Obviously, though it doesn't make sense
 to use the doctest option.
 
+.. _pseudo-decorators:
+
 Pseudo-Decorators
 =================
 

tools/update_whatsnew.py

@@ -7,26 +7,24 b' whatsnew/development.rst (chronologically ordered), and deletes the snippets.'
 
 import io
 import sys
-from glob import glob
-from os.path import dirname, basename, abspath, join as pjoin
+from pathlib import Path
 from subprocess import check_call, check_output
 
-repo_root = dirname(dirname(abspath(__file__)))
-whatsnew_dir = pjoin(repo_root, 'docs', 'source', 'whatsnew')
-pr_dir = pjoin(whatsnew_dir, 'pr')
-target = pjoin(whatsnew_dir, 'development.rst')
+repo_root = Path(__file__).resolve().parent.parent
+whatsnew_dir = repo_root / "docs" / "source" / "whatsnew"
+pr_dir = whatsnew_dir / "pr"
+target = whatsnew_dir / "development.rst"
 
 FEATURE_MARK = ".. DO NOT EDIT THIS LINE BEFORE RELEASE. FEATURE INSERTION POINT."
 INCOMPAT_MARK = ".. DO NOT EDIT THIS LINE BEFORE RELEASE. INCOMPAT INSERTION POINT."
 
 # 1. Collect the whatsnew snippet files ---------------------------------------
 
-files = set(glob(pjoin(pr_dir, '*.rst')))
+files = set(pr_dir.glob("*.rst"))
 # Ignore explanatory and example files
-files.difference_update({pjoin(pr_dir, f) for f in {
-                         'incompat-switching-to-perl.rst',
-                         'antigravity-feature.rst'}
-                         })
+files.difference_update(
+    {pr_dir / f for f in {"incompat-switching-to-perl.rst", "antigravity-feature.rst"}}
+)
 
 if not files:
     print("No automatic update available for what's new")
@@ -34,30 +32,31 b' if not files:'
 
 
 def getmtime(f):
-    return check_output(['git', 'log', '-1', '--format="%ai"', '--', f])
+    return check_output(["git", "log", "-1", '--format="%ai"', "--", f])
+
 
 files = sorted(files, key=getmtime)
 
 features, incompats = [], []
 for path in files:
-    with io.open(path, encoding='utf-8') as f:
+    with io.open(path, encoding="utf-8") as f:
         try:
             content = f.read().rstrip()
         except Exception as e:
             raise Exception('Error reading "{}"'.format(f)) from e
 
-    if basename(path).startswith('incompat-'):
+    if path.name.startswith("incompat-"):
         incompats.append(content)
     else:
         features.append(content)
 
 # Put the insertion markers back on the end, so they're ready for next time.
-feature_block = '\n\n'.join(features + [FEATURE_MARK])
-incompat_block = '\n\n'.join(incompats + [INCOMPAT_MARK])
+feature_block = "\n\n".join(features + [FEATURE_MARK])
+incompat_block = "\n\n".join(incompats + [INCOMPAT_MARK])
 
 # 2. Update the target file ---------------------------------------------------
 
-with io.open(target, encoding='utf-8') as f:
+with io.open(target, encoding="utf-8") as f:
     content = f.read()
 
 assert content.count(FEATURE_MARK) == 1
@@ -67,16 +66,16 b' content = content.replace(FEATURE_MARK, feature_block)'
 content = content.replace(INCOMPAT_MARK, incompat_block)
 
 # Clean trailing whitespace
-content = '\n'.join(l.rstrip() for l in content.splitlines())
+content = "\n".join(l.rstrip() for l in content.splitlines())
 
-with io.open(target, 'w', encoding='utf-8') as f:
+with io.open(target, "w", encoding="utf-8") as f:
     f.write(content)
 
 # 3. Stage the changes in git -------------------------------------------------
 
 for file in files:
-    check_call(['git', 'rm', file])
+    check_call(["git", "rm", file])
 
-check_call(['git', 'add', target])
+check_call(["git", "add", target])
 
 print("Merged what's new changes. Check the diff and commit the change.")