py3: conditionalize test-demandimport.py for Python 3

The Python 3 lazy importer uses the LazyLoader that is part of
importlib.

On Python 3 and later, LazyLoader is implemented using a custom module
type that defines a __getattribute__ which triggers module loading.

Furthermore, there are additional differences as well. For example,
it appears that Python 3 will return an existing sys.modules
entry instead of constructing a new module object.

This commit adds additional test coverage for lazy importing
behavior to cover the differences between Python 2 and 3. This
reveals that the test and some lazy import functionality is kinda
busted. For example, the test assumes "contextlib" will be lazy.
But in reality an import before it has already imported contextlib!

There's definitely room to improve the behavior of the demand
importer code, both for Python 2 and 3. But at least the test
passes on Python 3 now.

Differential Revision: https://phab.mercurial-scm.org/D5796

tests/test-demandimport.py

@@ -1,125 +1,224 b''
 from __future__ import absolute_import, print_function
 
 from mercurial import demandimport
 demandimport.enable()
 
 import os
 import subprocess
 import sys
+import types
+
+# Don't import pycompat because it has too many side-effects.
+ispy3 = sys.version_info[0] >= 3
 
 # Only run if demandimport is allowed
 if subprocess.call(['python', '%s/hghave' % os.environ['TESTDIR'],
                     'demandimport']):
     sys.exit(80)
 
 # We rely on assert, which gets optimized out.
 if sys.flags.optimize:
     sys.exit(80)
 
+if ispy3:
+    from importlib.util import _LazyModule
+
+    try:
+        from importlib.util import _Module as moduletype
+    except ImportError:
+        moduletype = types.ModuleType
+else:
+    moduletype = types.ModuleType
+
 if os.name != 'nt':
     try:
         import distutils.msvc9compiler
         print('distutils.msvc9compiler needs to be an immediate '
               'importerror on non-windows platforms')
         distutils.msvc9compiler
     except ImportError:
         pass
 
 import re
 
 rsub = re.sub
 def f(obj):
     l = repr(obj)
     l = rsub("0x[0-9a-fA-F]+", "0x?", l)
     l = rsub("from '.*'", "from '?'", l)
     l = rsub("'<[a-z]*>'", "'<whatever>'", l)
     return l
 
 demandimport.disable()
 os.environ['HGDEMANDIMPORT'] = 'disable'
 # this enable call should not actually enable demandimport!
 demandimport.enable()
 from mercurial import node
 
 # We use assert instead of a unittest test case because having imports inside
 # functions changes behavior of the demand importer.
-assert f(node) == "<module 'mercurial.node' from '?'>", f(node)
+if ispy3:
+    assert not isinstance(node, _LazyModule)
+else:
+    assert f(node) == "<module 'mercurial.node' from '?'>", f(node)
 
 # now enable it for real
 del os.environ['HGDEMANDIMPORT']
 demandimport.enable()
 
 # Test access to special attributes through demandmod proxy
+assert 'mercurial.error' not in sys.modules
 from mercurial import error as errorproxy
-assert f(errorproxy) == "<unloaded module 'error'>", f(errorproxy)
+
+if ispy3:
+    # unsure why this isn't lazy.
+    assert not isinstance(f, _LazyModule)
+    assert f(errorproxy) == "<module 'mercurial.error' from '?'>", f(errorproxy)
+else:
+    assert f(errorproxy) == "<unloaded module 'error'>", f(errorproxy)
+
 doc = ' '.join(errorproxy.__doc__.split()[:3])
 assert doc == 'Mercurial exceptions. This', doc
 assert errorproxy.__name__ == 'mercurial.error', errorproxy.__name__
 
 # __name__ must be accessible via __dict__ so the relative imports can be
 # resolved
 name = errorproxy.__dict__['__name__']
 assert name == 'mercurial.error', name
 
-assert f(errorproxy) == "<proxied module 'error'>", f(errorproxy)
+if ispy3:
+    assert not isinstance(errorproxy, _LazyModule)
+    assert f(errorproxy) == "<module 'mercurial.error' from '?'>", f(errorproxy)
+else:
+    assert f(errorproxy) == "<proxied module 'error'>", f(errorproxy)
 
 import os
 
-assert f(os) == "<unloaded module 'os'>", f(os)
+if ispy3:
+    assert not isinstance(os, _LazyModule)
+    assert f(os) == "<module 'os' from '?'>", f(os)
+else:
+    assert f(os) == "<unloaded module 'os'>", f(os)
+
 assert f(os.system) == '<built-in function system>', f(os.system)
 assert f(os) == "<module 'os' from '?'>", f(os)
 
+assert 'mercurial.utils.procutil' not in sys.modules
 from mercurial.utils import procutil
 
-assert f(procutil) == "<unloaded module 'procutil'>", f(procutil)
+if ispy3:
+    assert isinstance(procutil, _LazyModule)
+    assert f(procutil) == "<module 'mercurial.utils.procutil' from '?'>", f(
+        procutil
+    )
+else:
+    assert f(procutil) == "<unloaded module 'procutil'>", f(procutil)
+
 assert f(procutil.system) == '<function system at 0x?>', f(procutil.system)
+assert procutil.__class__ == moduletype, procutil.__class__
 assert f(procutil) == "<module 'mercurial.utils.procutil' from '?'>", f(
     procutil
 )
 assert f(procutil.system) == '<function system at 0x?>', f(procutil.system)
 
+assert 'mercurial.hgweb' not in sys.modules
 from mercurial import hgweb
-assert f(hgweb) == "<unloaded module 'hgweb'>", f(hgweb)
-assert f(hgweb.hgweb_mod) == "<unloaded module 'hgweb_mod'>", f(hgweb.hgweb_mod)
+
+if ispy3:
+    assert not isinstance(hgweb, _LazyModule)
+    assert f(hgweb) == "<module 'mercurial.hgweb' from '?'>", f(hgweb)
+    assert isinstance(hgweb.hgweb_mod, _LazyModule)
+    assert (
+        f(hgweb.hgweb_mod) == "<module 'mercurial.hgweb.hgweb_mod' from '?'>"
+    ), f(hgweb.hgweb_mod)
+else:
+    assert f(hgweb) == "<unloaded module 'hgweb'>", f(hgweb)
+    assert f(hgweb.hgweb_mod) == "<unloaded module 'hgweb_mod'>", f(
+        hgweb.hgweb_mod
+    )
+
 assert f(hgweb) == "<module 'mercurial.hgweb' from '?'>", f(hgweb)
 
 import re as fred
-assert f(fred) == "<unloaded module 're'>", f(fred)
+
+if ispy3:
+    assert not isinstance(fred, _LazyModule)
+    assert f(fred) == "<module 're' from '?'>"
+else:
+    assert f(fred) == "<unloaded module 're'>", f(fred)
 
 import re as remod
-assert f(remod) == "<unloaded module 're'>", f(remod)
+
+if ispy3:
+    assert not isinstance(remod, _LazyModule)
+    assert f(remod) == "<module 're' from '?'>"
+else:
+    assert f(remod) == "<unloaded module 're'>", f(remod)
 
 import sys as re
-assert f(re) == "<unloaded module 'sys'>", f(re)
+
+if ispy3:
+    assert not isinstance(re, _LazyModule)
+    assert f(re) == "<module 'sys' (built-in)>"
+else:
+    assert f(re) == "<unloaded module 'sys'>", f(re)
 
-assert f(fred) == "<unloaded module 're'>", f(fred)
+if ispy3:
+    assert not isinstance(fred, _LazyModule)
+    assert f(fred) == "<module 're' from '?'>", f(fred)
+else:
+    assert f(fred) == "<unloaded module 're'>", f(fred)
+
 assert f(fred.sub) == '<function sub at 0x?>', f(fred.sub)
-assert f(fred) == "<proxied module 're'>", f(fred)
+
+if ispy3:
+    assert not isinstance(fred, _LazyModule)
+    assert f(fred) == "<module 're' from '?'>", f(fred)
+else:
+    assert f(fred) == "<proxied module 're'>", f(fred)
 
 remod.escape  # use remod
 assert f(remod) == "<module 're' from '?'>", f(remod)
 
-assert f(re) == "<unloaded module 'sys'>", f(re)
-assert f(re.stderr) == "<open file '<whatever>', mode 'w' at 0x?>", f(re.stderr)
-assert f(re) == "<proxied module 'sys'>", f(re)
+if ispy3:
+    assert not isinstance(re, _LazyModule)
+    assert f(re) == "<module 'sys' (built-in)>"
+    assert f(type(re.stderr)) == "<class '_io.TextIOWrapper'>", f(
+        type(re.stderr)
+    )
+    assert f(re) == "<module 'sys' (built-in)>"
+else:
+    assert f(re) == "<unloaded module 'sys'>", f(re)
+    assert f(re.stderr) == "<open file '<whatever>', mode 'w' at 0x?>", f(
+        re.stderr
+    )
+    assert f(re) == "<proxied module 'sys'>", f(re)
 
 import contextlib
-assert f(contextlib) == "<unloaded module 'contextlib'>", f(contextlib)
+
+if ispy3:
+    assert not isinstance(contextlib, _LazyModule)
+    assert f(contextlib) == "<module 'contextlib' from '?'>"
+else:
+    assert f(contextlib) == "<unloaded module 'contextlib'>", f(contextlib)
+
 try:
     from contextlib import unknownattr
 
     assert False, (
         'no demandmod should be created for attribute of non-package '
         'module:\ncontextlib.unknownattr = %s' % f(unknownattr)
     )
 except ImportError as inst:
-    assert rsub(r"'", '', str(inst)) == 'cannot import name unknownattr'
+    assert rsub(r"'", '', str(inst)).startswith(
+        'cannot import name unknownattr'
+    )
 
 from mercurial import util
 
 # Unlike the import statement, __import__() function should not raise
 # ImportError even if fromlist has an unknown item
 # (see Python/import.c:import_module_level() and ensure_fromlist())
 contextlibimp = __import__('contextlib', globals(), locals(), ['unknownattr'])
 assert f(contextlibimp) == "<module 'contextlib' from '?'>", f(contextlibimp)
 assert not util.safehasattr(contextlibimp, 'unknownattr')