Merge pull request #13412 from bnavigator/backport-inspect

Backport Python 3.10 fixes

.github/workflows/test.yml

@@ -15,7 +15,7 b' jobs:'
     strategy:
       matrix:
         os: [ubuntu-latest]
-        python-version: ["3.7", "3.8", "3.9"]
+        python-version: ["3.7", "3.8", "3.9", "3.10"]
         # Test all on ubuntu, test ends on macos
         include:
           - os: macos-latest
@@ -39,6 +39,7 b' jobs:'
     - name: Check manifest
       run: check-manifest
     - name: iptest
+      if: matrix.python-version != '3.10'
       run: |
         cd /tmp && iptest --coverage xml && cd -
         cp /tmp/ipy_coverage.xml ./

IPython/core/oinspect.py

@@ -182,11 +182,12 b" def getsource(obj, oname='') -> Union[str,None]:"
         except TypeError:
             # The object itself provided no meaningful source, try looking for
             # its class definition instead.
-            if hasattr(obj, '__class__'):
-                try:
-                    src = inspect.getsource(obj.__class__)
-                except TypeError:
-                    return None
+            try:
+                src = inspect.getsource(obj.__class__)
+            except (OSError, TypeError):
+                return None
+        except OSError:
+            return None
 
         return src
 
@@ -308,14 +309,14 b' def find_file(obj) -> str:'
     except TypeError:
         # For an instance, the file that matters is where its class was
         # declared.
-        if hasattr(obj, '__class__'):
-            try:
-                fname = inspect.getabsfile(obj.__class__)
-            except TypeError:
-                # Can happen for builtins
-                pass
-    except:
+        try:
+            fname = inspect.getabsfile(obj.__class__)
+        except (OSError, TypeError):
+            # Can happen for builtins
+            pass
+    except OSError:
         pass
+
     return cast_unicode(fname)
 
 
@@ -338,15 +339,14 b' def find_source_lines(obj):'
     obj = _get_wrapped(obj)
 
     try:
+        lineno = inspect.getsourcelines(obj)[1]
+    except TypeError:
+        # For instances, try the class object like getsource() does
         try:
-            lineno = inspect.getsourcelines(obj)[1]
-        except TypeError:
-            # For instances, try the class object like getsource() does
-            if hasattr(obj, '__class__'):
-                lineno = inspect.getsourcelines(obj.__class__)[1]
-            else:
-                lineno = None
-    except:
+            lineno = inspect.getsourcelines(obj.__class__)[1]
+        except (OSError, TypeError):
+            return None
+    except OSError:
         return None
 
     return lineno

IPython/core/tests/test_completer.py

@@ -12,6 +12,7 b' import unittest'
 from contextlib import contextmanager
 
 import nose.tools as nt
+import pytest
 
 from traitlets.config.loader import Config
 from IPython import get_ipython
@@ -29,6 +30,15 b' from IPython.core.completer import ('
 )
 from nose.tools import assert_in, assert_not_in
 
+if sys.version_info >= (3, 10):
+    import jedi
+    from pkg_resources import parse_version
+
+    # Requires https://github.com/davidhalter/jedi/pull/1795
+    jedi_issue = parse_version(jedi.__version__) <= parse_version("0.18.0")
+else:
+    jedi_issue = False
+
 # -----------------------------------------------------------------------------
 # Test functions
 # -----------------------------------------------------------------------------
@@ -381,6 +391,8 b' class TestCompleter(unittest.TestCase):'
                 matches = c.all_completions("TestCl")
                 assert matches == ['TestClass'], jedi_status
                 matches = c.all_completions("TestClass.")
+                if jedi_status and jedi_issue:
+                    continue
                 assert len(matches) > 2, jedi_status
                 matches = c.all_completions("TestClass.a")
                 assert matches == ['TestClass.a', 'TestClass.a1'], jedi_status
@@ -435,6 +447,7 b' class TestCompleter(unittest.TestCase):'
             "encoding" in c.signature
         ), "Signature of function was not found by completer"
 
+    @pytest.mark.xfail(jedi_issue, reason="Known failure on jedi<=0.18.0")
     def test_deduplicate_completions(self):
         """
         Test that completions are correctly deduplicated (even if ranges are not the same)

IPython/core/tests/test_magic_arguments.py

@@ -7,6 +7,7 b''
 #-----------------------------------------------------------------------------
 
 import argparse
+import sys
 from nose.tools import assert_equal
 
 from IPython.core.magic_arguments import (argument, argument_group, kwds,
@@ -74,7 +75,12 b' def foo(self, args):'
 
 
 def test_magic_arguments():
-    assert_equal(magic_foo1.__doc__, '::\n\n  %foo1 [-f FOO]\n\n A docstring.\n\noptional arguments:\n  -f FOO, --foo FOO  an argument\n')
+    # “optional arguments” was replaced with “options” in argparse help
+    # https://docs.python.org/3/whatsnew/3.10.html#argparse
+    # https://bugs.python.org/issue9694
+    options = "optional arguments" if sys.version_info < (3, 10) else "options"
+
+    assert_equal(magic_foo1.__doc__, f"::\n\n  %foo1 [-f FOO]\n\n A docstring.\n\n{options}:\n  -f FOO, --foo FOO  an argument\n")
     assert_equal(getattr(magic_foo1, 'argcmd_name', None), None)
     assert_equal(real_name(magic_foo1), 'foo1')
     assert_equal(magic_foo1(None, ''), argparse.Namespace(foo=None))
@@ -86,32 +92,32 b' def test_magic_arguments():'
     assert_equal(magic_foo2(None, ''), argparse.Namespace())
     assert hasattr(magic_foo2, 'has_arguments')
 
-    assert_equal(magic_foo3.__doc__, '::\n\n  %foo3 [-f FOO] [-b BAR] [-z BAZ]\n\n A docstring.\n\noptional arguments:\n  -f FOO, --foo FOO  an argument\n\nGroup:\n  -b BAR, --bar BAR  a grouped argument\n\nSecond Group:\n  -z BAZ, --baz BAZ  another grouped argument\n')
+    assert_equal(magic_foo3.__doc__, f"::\n\n  %foo3 [-f FOO] [-b BAR] [-z BAZ]\n\n A docstring.\n\n{options}:\n  -f FOO, --foo FOO  an argument\n\nGroup:\n  -b BAR, --bar BAR  a grouped argument\n\nSecond Group:\n  -z BAZ, --baz BAZ  another grouped argument\n")
     assert_equal(getattr(magic_foo3, 'argcmd_name', None), None)
     assert_equal(real_name(magic_foo3), 'foo3')
     assert_equal(magic_foo3(None, ''),
                        argparse.Namespace(bar=None, baz=None, foo=None))
     assert hasattr(magic_foo3, 'has_arguments')
 
-    assert_equal(magic_foo4.__doc__, '::\n\n  %foo4 [-f FOO]\n\n A docstring.\n\noptional arguments:\n  -f FOO, --foo FOO  an argument\n')
+    assert_equal(magic_foo4.__doc__, f"::\n\n  %foo4 [-f FOO]\n\n A docstring.\n\n{options}:\n  -f FOO, --foo FOO  an argument\n")
     assert_equal(getattr(magic_foo4, 'argcmd_name', None), None)
     assert_equal(real_name(magic_foo4), 'foo4')
     assert_equal(magic_foo4(None, ''), argparse.Namespace())
     assert hasattr(magic_foo4, 'has_arguments')
 
-    assert_equal(magic_foo5.__doc__, '::\n\n  %frobnicate [-f FOO]\n\n A docstring.\n\noptional arguments:\n  -f FOO, --foo FOO  an argument\n')
+    assert_equal(magic_foo5.__doc__, f"::\n\n  %frobnicate [-f FOO]\n\n A docstring.\n\n{options}:\n  -f FOO, --foo FOO  an argument\n")
     assert_equal(getattr(magic_foo5, 'argcmd_name', None), 'frobnicate')
     assert_equal(real_name(magic_foo5), 'frobnicate')
     assert_equal(magic_foo5(None, ''), argparse.Namespace(foo=None))
     assert hasattr(magic_foo5, 'has_arguments')
 
-    assert_equal(magic_magic_foo.__doc__, '::\n\n  %magic_foo [-f FOO]\n\n A docstring.\n\noptional arguments:\n  -f FOO, --foo FOO  an argument\n')
+    assert_equal(magic_magic_foo.__doc__, f"::\n\n  %magic_foo [-f FOO]\n\n A docstring.\n\n{options}:\n  -f FOO, --foo FOO  an argument\n")
     assert_equal(getattr(magic_magic_foo, 'argcmd_name', None), None)
     assert_equal(real_name(magic_magic_foo), 'magic_foo')
     assert_equal(magic_magic_foo(None, ''), argparse.Namespace(foo=None))
     assert hasattr(magic_magic_foo, 'has_arguments')
 
-    assert_equal(foo.__doc__, '::\n\n  %foo [-f FOO]\n\n A docstring.\n\noptional arguments:\n  -f FOO, --foo FOO  an argument\n')
+    assert_equal(foo.__doc__, f"::\n\n  %foo [-f FOO]\n\n A docstring.\n\n{options}:\n  -f FOO, --foo FOO  an argument\n")
     assert_equal(getattr(foo, 'argcmd_name', None), None)
     assert_equal(real_name(foo), 'foo')
     assert_equal(foo(None, ''), argparse.Namespace(foo=None))

IPython/core/tests/test_oinspect.py

@@ -6,10 +6,11 b''
 
 
 from inspect import signature, Signature, Parameter
+import inspect
 import os
+import pytest
 import re
-
-import nose.tools as nt
+import sys
 
 from .. import oinspect
 
@@ -30,6 +31,10 b' def setup_module():'
     inspector = oinspect.Inspector()
 
 
+class SourceModuleMainTest:
+    __module__ = "__main__"
+
+
 #-----------------------------------------------------------------------------
 # Local utilities
 #-----------------------------------------------------------------------------
@@ -38,15 +43,28 b' def setup_module():'
 # defined, if any code is inserted above, the following line will need to be
 # updated.  Do NOT insert any whitespace between the next line and the function
 # definition below.
-THIS_LINE_NUMBER = 41  # Put here the actual number of this line
+THIS_LINE_NUMBER = 46  # Put here the actual number of this line
+
+
+def test_find_source_lines():
+    assert oinspect.find_source_lines(test_find_source_lines) == THIS_LINE_NUMBER + 3
+    assert oinspect.find_source_lines(type) is None
+    assert oinspect.find_source_lines(SourceModuleMainTest) is None
+    assert oinspect.find_source_lines(SourceModuleMainTest()) is None
+
 
-from unittest import TestCase
+def test_getsource():
+    assert oinspect.getsource(type) is None
+    assert oinspect.getsource(SourceModuleMainTest) is None
+    assert oinspect.getsource(SourceModuleMainTest()) is None
 
-class Test(TestCase):
 
-    def test_find_source_lines(self):
-        self.assertEqual(oinspect.find_source_lines(Test.test_find_source_lines),
-                    THIS_LINE_NUMBER+6)
+def test_inspect_getfile_raises_exception():
+    """Check oinspect.find_file/getsource/find_source_lines expectations"""
+    with pytest.raises(TypeError):
+        inspect.getfile(type)
+    with pytest.raises(OSError if sys.version_info >= (3, 10) else TypeError):
+        inspect.getfile(SourceModuleMainTest)
 
 
 # A couple of utilities to ensure these tests work the same from a source or a
@@ -56,11 +74,14 b' def pyfile(fname):'
 
 
 def match_pyfiles(f1, f2):
-    nt.assert_equal(pyfile(f1), pyfile(f2))
+    assert pyfile(f1) == pyfile(f2)
 
 
 def test_find_file():
     match_pyfiles(oinspect.find_file(test_find_file), os.path.abspath(__file__))
+    assert oinspect.find_file(type) is None
+    assert oinspect.find_file(SourceModuleMainTest) is None
+    assert oinspect.find_file(SourceModuleMainTest()) is None
 
 
 def test_find_file_decorated1():
@@ -74,9 +95,9 b' def test_find_file_decorated1():'
     @noop1
     def f(x):
         "My docstring"
-    
+
     match_pyfiles(oinspect.find_file(f), os.path.abspath(__file__))
-    nt.assert_equal(f.__doc__, "My docstring")
+    assert f.__doc__ == "My docstring"
 
 
 def test_find_file_decorated2():
@@ -90,14 +111,14 b' def test_find_file_decorated2():'
     @noop2
     def f(x):
         "My docstring 2"
-    
+
     match_pyfiles(oinspect.find_file(f), os.path.abspath(__file__))
-    nt.assert_equal(f.__doc__, "My docstring 2")
-    
+    assert f.__doc__ == "My docstring 2"
+
 
 def test_find_file_magic():
     run = ip.find_line_magic('run')
-    nt.assert_not_equal(oinspect.find_file(run), None)
+    assert oinspect.find_file(run) is not None
 
 
 # A few generic objects we can then inspect in the tests below
@@ -167,41 +188,46 b' class SerialLiar(object):'
 
 def test_info():
     "Check that Inspector.info fills out various fields as expected."
-    i = inspector.info(Call, oname='Call')
-    nt.assert_equal(i['type_name'], 'type')
-    expted_class = str(type(type))  # <class 'type'> (Python 3) or <type 'type'>
-    nt.assert_equal(i['base_class'], expted_class)
-    nt.assert_regex(i['string_form'], "<class 'IPython.core.tests.test_oinspect.Call'( at 0x[0-9a-f]{1,9})?>")
+    i = inspector.info(Call, oname="Call")
+    assert i["type_name"] == "type"
+    expected_class = str(type(type))  # <class 'type'> (Python 3) or <type 'type'>
+    assert i["base_class"] == expected_class
+    assert re.search(
+        "<class 'IPython.core.tests.test_oinspect.Call'( at 0x[0-9a-f]{1,9})?>",
+        i["string_form"],
+    )
     fname = __file__
     if fname.endswith(".pyc"):
         fname = fname[:-1]
     # case-insensitive comparison needed on some filesystems
     # e.g. Windows:
-    nt.assert_equal(i['file'].lower(), compress_user(fname).lower())
-    nt.assert_equal(i['definition'], None)
-    nt.assert_equal(i['docstring'], Call.__doc__)
-    nt.assert_equal(i['source'], None)
-    nt.assert_true(i['isclass'])
-    nt.assert_equal(i['init_definition'], "Call(x, y=1)")
-    nt.assert_equal(i['init_docstring'], Call.__init__.__doc__)
+    assert i["file"].lower() == compress_user(fname).lower()
+    assert i["definition"] == None
+    assert i["docstring"] == Call.__doc__
+    assert i["source"] == None
+    assert i["isclass"] is True
+    assert i["init_definition"] == "Call(x, y=1)"
+    assert i["init_docstring"] == Call.__init__.__doc__
 
     i = inspector.info(Call, detail_level=1)
-    nt.assert_not_equal(i['source'], None)
-    nt.assert_equal(i['docstring'], None)
+    assert i["source"] is not None
+    assert i["docstring"] == None
 
     c = Call(1)
     c.__doc__ = "Modified instance docstring"
     i = inspector.info(c)
-    nt.assert_equal(i['type_name'], 'Call')
-    nt.assert_equal(i['docstring'], "Modified instance docstring")
-    nt.assert_equal(i['class_docstring'], Call.__doc__)
-    nt.assert_equal(i['init_docstring'], Call.__init__.__doc__)
-    nt.assert_equal(i['call_docstring'], Call.__call__.__doc__)
+    assert i["type_name"] == "Call"
+    assert i["docstring"] == "Modified instance docstring"
+    assert i["class_docstring"] == Call.__doc__
+    assert i["init_docstring"] == Call.__init__.__doc__
+    assert i["call_docstring"] == Call.__call__.__doc__
+
 
 def test_class_signature():
-    info = inspector.info(HasSignature, 'HasSignature')
-    nt.assert_equal(info['init_definition'], "HasSignature(test)")
-    nt.assert_equal(info['init_docstring'], HasSignature.__init__.__doc__)
+    info = inspector.info(HasSignature, "HasSignature")
+    assert info["init_definition"] == "HasSignature(test)"
+    assert info["init_docstring"] == HasSignature.__init__.__doc__
+
 
 def test_info_awkward():
     # Just test that this doesn't throw an error.
@@ -216,7 +242,7 b' def test_info_serialliar():'
 
     # Nested attribute access should be cut off at 100 levels deep to avoid
     # infinite loops: https://github.com/ipython/ipython/issues/9122
-    nt.assert_less(fib_tracker[0], 9000)
+    assert fib_tracker[0] < 9000
 
 def support_function_one(x, y=2, *a, **kw):
     """A simple function."""
@@ -225,14 +251,16 b' def test_calldef_none():'
     # We should ignore __call__ for all of these.
     for obj in [support_function_one, SimpleClass().method, any, str.upper]:
         i = inspector.info(obj)
-        nt.assert_is(i['call_def'], None)
+        assert i["call_def"] is None
+
 
 def f_kwarg(pos, *, kwonly):
     pass
 
 def test_definition_kwonlyargs():
-    i = inspector.info(f_kwarg, oname='f_kwarg')  # analysis:ignore
-    nt.assert_equal(i['definition'], "f_kwarg(pos, *, kwonly)")
+    i = inspector.info(f_kwarg, oname="f_kwarg")  # analysis:ignore
+    assert i["definition"] == "f_kwarg(pos, *, kwonly)"
+
 
 def test_getdoc():
     class A(object):
@@ -243,34 +271,33 b' def test_getdoc():'
         """standard docstring"""
         def getdoc(self):
             return "custom docstring"
-    
+
     class C(object):
         """standard docstring"""
         def getdoc(self):
             return None
-    
+
     a = A()
     b = B()
     c = C()
-    
-    nt.assert_equal(oinspect.getdoc(a), "standard docstring")
-    nt.assert_equal(oinspect.getdoc(b), "custom docstring")
-    nt.assert_equal(oinspect.getdoc(c), "standard docstring")
+
+    assert oinspect.getdoc(a) == "standard docstring"
+    assert oinspect.getdoc(b) == "custom docstring"
+    assert oinspect.getdoc(c) == "standard docstring"
 
 
 def test_empty_property_has_no_source():
     i = inspector.info(property(), detail_level=1)
-    nt.assert_is(i['source'], None)
+    assert i["source"] is None
 
 
 def test_property_sources():
-    import posixpath 
     # A simple adder whose source and signature stays
     # the same across Python distributions
     def simple_add(a, b):
         "Adds two numbers"
         return a + b
-    
+
     class A(object):
         @property
         def foo(self):
@@ -278,18 +305,18 b' def test_property_sources():'
 
         foo = foo.setter(lambda self, v: setattr(self, 'bar', v))
 
-        dname = property(posixpath.dirname)
-        adder = property(simple_add) 
+        dname = property(oinspect.getdoc)
+        adder = property(simple_add)
 
     i = inspector.info(A.foo, detail_level=1)
-    nt.assert_in('def foo(self):', i['source'])
-    nt.assert_in('lambda self, v:', i['source'])
+    assert "def foo(self):" in i["source"]
+    assert "lambda self, v:" in i["source"]
 
     i = inspector.info(A.dname, detail_level=1)
-    nt.assert_in('def dirname(p)', i['source'])
-    
+    assert "def getdoc(obj)" in i["source"]
+
     i = inspector.info(A.adder, detail_level=1)
-    nt.assert_in('def simple_add(a, b)', i['source'])
+    assert "def simple_add(a, b)" in i["source"]
 
 
 def test_property_docstring_is_in_info_for_detail_level_0():
@@ -299,15 +326,17 b' def test_property_docstring_is_in_info_for_detail_level_0():'
             """This is `foobar` property."""
             pass
 
-    ip.user_ns['a_obj'] = A()
-    nt.assert_equal(
-        'This is `foobar` property.',
-        ip.object_inspect('a_obj.foobar', detail_level=0)['docstring'])
+    ip.user_ns["a_obj"] = A()
+    assert (
+        "This is `foobar` property."
+        == ip.object_inspect("a_obj.foobar", detail_level=0)["docstring"]
+    )
 
-    ip.user_ns['a_cls'] = A
-    nt.assert_equal(
-        'This is `foobar` property.',
-        ip.object_inspect('a_cls.foobar', detail_level=0)['docstring'])
+    ip.user_ns["a_cls"] = A
+    assert (
+        "This is `foobar` property."
+        == ip.object_inspect("a_cls.foobar", detail_level=0)["docstring"]
+    )
 
 
 def test_pdef():
@@ -359,11 +388,11 b' def test_pinfo_docstring_if_detail_and_no_source():'
                   def bar(self):
                       """ This is a docstring for Foo.bar """
                       pass
-              ''' 
-    
+              '''
+
     ip.run_cell(obj_def)
     ip.run_cell('foo = Foo()')
-    
+
     with AssertNotPrints("Source:"):
         with AssertPrints('Docstring:'):
             ip._inspect('pinfo', 'foo', detail_level=0)
@@ -388,14 +417,14 b' def test_pinfo_magic():'
 def test_init_colors():
     # ensure colors are not present in signature info
     info = inspector.info(HasSignature)
-    init_def = info['init_definition']
-    nt.assert_not_in('[0m', init_def)
+    init_def = info["init_definition"]
+    assert "[0m" not in init_def
 
 
 def test_builtin_init():
     info = inspector.info(list)
     init_def = info['init_definition']
-    nt.assert_is_not_none(init_def)
+    assert init_def is not None
 
 
 def test_render_signature_short():
@@ -404,7 +433,7 b' def test_render_signature_short():'
         signature(short_fun),
         short_fun.__name__,
     )
-    nt.assert_equal(sig, 'short_fun(a=1)')
+    assert sig == "short_fun(a=1)"
 
 
 def test_render_signature_long():
@@ -420,7 +449,7 b' def test_render_signature_long():'
         signature(long_function),
         long_function.__name__,
     )
-    nt.assert_in(sig, [
+    assert sig in [
         # Python >=3.9
         '''\
 long_function(
@@ -444,4 +473,4 b' long_function('
     let_us_make_sure_this_is_looong:Union[str, NoneType]=None,
 ) -> bool\
 ''',
-    ])
+    ]
  No newline at end of file

IPython/lib/tests/test_pretty.py

@@ -9,6 +9,7 b' from collections import Counter, defaultdict, deque, OrderedDict'
 import os
 import types
 import string
+import sys
 import unittest
 
 import nose.tools as nt
@@ -118,12 +119,15 b' def test_sets():'
         yield nt.assert_equal, got_output, expected_output
 
 
-@skip_without('xxlimited')
+@skip_without("xxlimited" if sys.version_info < (3, 10) else "xxlimited_35")
 def test_pprint_heap_allocated_type():
     """
     Test that pprint works for heap allocated types.
     """
-    import xxlimited
+    if sys.version_info < (3, 10):
+        import xxlimited
+    else:
+        import xxlimited_35 as xxlimited
     output = pretty.pretty(xxlimited.Null)
     nt.assert_equal(output, 'xxlimited.Null')