demandimport: ensure lazyloaderex sets loader attributes (issue6725)

Adds test capturing missed expectation.

hgdemandimport/demandimportpy3.py

@@ -1,172 +1,176
 # demandimportpy3 - global demand-loading of modules for Mercurial
 #
 # Copyright 2017 Facebook Inc.
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 """Lazy loading for Python 3.6 and above.
 
 This uses the new importlib finder/loader functionality available in Python 3.5
 and up. The code reuses most of the mechanics implemented inside importlib.util,
 but with a few additions:
 
 * Allow excluding certain modules from lazy imports.
 * Expose an interface that's substantially the same as demandimport for
   Python 2.
 
 This also has some limitations compared to the Python 2 implementation:
 
 * Much of the logic is per-package, not per-module, so any packages loaded
   before demandimport is enabled will not be lazily imported in the future. In
   practice, we only expect builtins to be loaded before demandimport is
   enabled.
 """
 
 import contextlib
 import importlib.util
 import sys
 
 from . import tracing
 
 _deactivated = False
 
 
 class _lazyloaderex(importlib.util.LazyLoader):
     """This is a LazyLoader except it also follows the _deactivated global and
     the ignore list.
     """
 
     def exec_module(self, module):
         """Make the module load lazily."""
         with tracing.log('demandimport %s', module):
             if _deactivated or module.__name__ in ignores:
+                # Reset the loader on the module as super() does (issue6725)
+                module.__spec__.loader = self.loader
+                module.__loader__ = self.loader
+
                 self.loader.exec_module(module)
             else:
                 super().exec_module(module)
 
 
 class LazyFinder:
     """A wrapper around a ``MetaPathFinder`` that makes loaders lazy.
 
     ``sys.meta_path`` finders have their ``find_spec()`` called to locate a
     module. This returns a ``ModuleSpec`` if found or ``None``. The
     ``ModuleSpec`` has a ``loader`` attribute, which is called to actually
     load a module.
 
     Our class wraps an existing finder and overloads its ``find_spec()`` to
     replace the ``loader`` with our lazy loader proxy.
 
     We have to use __getattribute__ to proxy the instance because some meta
     path finders don't support monkeypatching.
     """
 
     __slots__ = ("_finder",)
 
     def __init__(self, finder):
         object.__setattr__(self, "_finder", finder)
 
     def __repr__(self):
         return "<LazyFinder for %r>" % object.__getattribute__(self, "_finder")
 
     # __bool__ is canonical Python 3. But check-code insists on __nonzero__ being
     # defined via `def`.
     def __nonzero__(self):
         return bool(object.__getattribute__(self, "_finder"))
 
     __bool__ = __nonzero__
 
     def __getattribute__(self, name):
         if name in ("_finder", "find_spec"):
             return object.__getattribute__(self, name)
 
         return getattr(object.__getattribute__(self, "_finder"), name)
 
     def __delattr__(self, name):
         return delattr(object.__getattribute__(self, "_finder"))
 
     def __setattr__(self, name, value):
         return setattr(object.__getattribute__(self, "_finder"), name, value)
 
     def find_spec(self, fullname, path, target=None):
         finder = object.__getattribute__(self, "_finder")
         try:
             find_spec = finder.find_spec
         except AttributeError:
             loader = finder.find_module(fullname, path)
             if loader is None:
                 spec = None
             else:
                 spec = importlib.util.spec_from_loader(fullname, loader)
         else:
             spec = find_spec(fullname, path, target)
 
         # Lazy loader requires exec_module().
         if (
             spec is not None
             and spec.loader is not None
             and getattr(spec.loader, "exec_module", None)
         ):
             spec.loader = _lazyloaderex(spec.loader)
 
         return spec
 
 
 ignores = set()
 
 
 def init(ignoreset):
     global ignores
     ignores = ignoreset
 
 
 def isenabled():
     return not _deactivated and any(
         isinstance(finder, LazyFinder) for finder in sys.meta_path
     )
 
 
 def disable():
     new_finders = []
     for finder in sys.meta_path:
         new_finders.append(
             finder._finder if isinstance(finder, LazyFinder) else finder
         )
     sys.meta_path[:] = new_finders
 
 
 def enable():
     new_finders = []
     for finder in sys.meta_path:
         new_finders.append(
             LazyFinder(finder) if not isinstance(finder, LazyFinder) else finder
         )
     sys.meta_path[:] = new_finders
 
 
 @contextlib.contextmanager
 def deactivated():
     # This implementation is a bit different from Python 2's. Python 3
     # maintains a per-package finder cache in sys.path_importer_cache (see
     # PEP 302). This means that we can't just call disable + enable.
     # If we do that, in situations like:
     #
     #   demandimport.enable()
     #   ...
     #   from foo.bar import mod1
     #   with demandimport.deactivated():
     #       from foo.bar import mod2
     #
     # mod2 will be imported lazily. (The converse also holds -- whatever finder
     # first gets cached will be used.)
     #
     # Instead, have a global flag the LazyLoader can use.
     global _deactivated
     demandenabled = isenabled()
     if demandenabled:
         _deactivated = True
     try:
         yield
     finally:
         if demandenabled:
             _deactivated = False

tests/test-demandimport.py

@@ -1,236 +1,244
 from mercurial import demandimport
 
 demandimport.enable()
 
 import os
 import subprocess
 import sys
 import types
 
 # Don't import pycompat because it has too many side-effects.
 ispy3 = sys.version_info[0] >= 3
 
 # Only run if demandimport is allowed
 if subprocess.call(
     [os.environ['PYTHON'], '%s/hghave' % os.environ['TESTDIR'], 'demandimport']
 ):
     sys.exit(80)
 
 # We rely on assert, which gets optimized out.
 if sys.flags.optimize:
     sys.exit(80)
 
 # The demand importer doesn't work on Python 3.5.
 if sys.version_info[0:2] == (3, 5):
     sys.exit(80)
 
 if ispy3:
     from importlib.util import _LazyModule
 
     try:
         from importlib.util import _Module as moduletype
     except ImportError:
         moduletype = types.ModuleType
 else:
     moduletype = types.ModuleType
 
 if os.name != 'nt':
     try:
         import distutils.msvc9compiler
 
         print(
             'distutils.msvc9compiler needs to be an immediate '
             'importerror on non-windows platforms'
         )
         distutils.msvc9compiler
     except ImportError:
         pass
 
 import re
 
 rsub = re.sub
 
 
 def f(obj):
     l = repr(obj)
     l = rsub("0x[0-9a-fA-F]+", "0x?", l)
     l = rsub("from '.*'", "from '?'", l)
     l = rsub("'<[a-z]*>'", "'<whatever>'", l)
     return l
 
 
 demandimport.disable()
 os.environ['HGDEMANDIMPORT'] = 'disable'
 # this enable call should not actually enable demandimport!
 demandimport.enable()
 from mercurial import node
 
 # We use assert instead of a unittest test case because having imports inside
 # functions changes behavior of the demand importer.
 if ispy3:
     assert not isinstance(node, _LazyModule)
 else:
     assert f(node) == "<module 'mercurial.node' from '?'>", f(node)
 
 # now enable it for real
 del os.environ['HGDEMANDIMPORT']
 demandimport.enable()
 
 # Test access to special attributes through demandmod proxy
 assert 'mercurial.error' not in sys.modules
 from mercurial import error as errorproxy
 
 if ispy3:
     # unsure why this isn't lazy.
     assert not isinstance(f, _LazyModule)
     assert f(errorproxy) == "<module 'mercurial.error' from '?'>", f(errorproxy)
 else:
     assert f(errorproxy) == "<unloaded module 'error'>", f(errorproxy)
 
 doc = ' '.join(errorproxy.__doc__.split()[:3])
 assert doc == 'Mercurial exceptions. This', doc
 assert errorproxy.__name__ == 'mercurial.error', errorproxy.__name__
 
 # __name__ must be accessible via __dict__ so the relative imports can be
 # resolved
 name = errorproxy.__dict__['__name__']
 assert name == 'mercurial.error', name
 
 if ispy3:
     assert not isinstance(errorproxy, _LazyModule)
     assert f(errorproxy) == "<module 'mercurial.error' from '?'>", f(errorproxy)
 else:
     assert f(errorproxy) == "<proxied module 'error'>", f(errorproxy)
 
 import os
 
 if ispy3:
     assert not isinstance(os, _LazyModule)
     assert f(os) == "<module 'os' from '?'>", f(os)
 else:
     assert f(os) == "<unloaded module 'os'>", f(os)
 
 assert f(os.system) == '<built-in function system>', f(os.system)
 assert f(os) == "<module 'os' from '?'>", f(os)
 
 assert 'mercurial.utils.procutil' not in sys.modules
 from mercurial.utils import procutil
 
 if ispy3:
     assert isinstance(procutil, _LazyModule)
     assert f(procutil) == "<module 'mercurial.utils.procutil' from '?'>", f(
         procutil
     )
 else:
     assert f(procutil) == "<unloaded module 'procutil'>", f(procutil)
 
 assert f(procutil.system) == '<function system at 0x?>', f(procutil.system)
 assert procutil.__class__ == moduletype, procutil.__class__
 assert f(procutil) == "<module 'mercurial.utils.procutil' from '?'>", f(
     procutil
 )
 assert f(procutil.system) == '<function system at 0x?>', f(procutil.system)
 
 assert 'mercurial.hgweb' not in sys.modules
 from mercurial import hgweb
 
 if ispy3:
     assert isinstance(hgweb, _LazyModule)
     assert f(hgweb) == "<module 'mercurial.hgweb' from '?'>", f(hgweb)
     assert isinstance(hgweb.hgweb_mod, _LazyModule)
     assert (
         f(hgweb.hgweb_mod) == "<module 'mercurial.hgweb.hgweb_mod' from '?'>"
     ), f(hgweb.hgweb_mod)
 else:
     assert f(hgweb) == "<unloaded module 'hgweb'>", f(hgweb)
     assert f(hgweb.hgweb_mod) == "<unloaded module 'hgweb_mod'>", f(
         hgweb.hgweb_mod
     )
 
 assert f(hgweb) == "<module 'mercurial.hgweb' from '?'>", f(hgweb)
 
 import re as fred
 
 if ispy3:
     assert not isinstance(fred, _LazyModule)
     assert f(fred) == "<module 're' from '?'>"
 else:
     assert f(fred) == "<unloaded module 're'>", f(fred)
 
 import re as remod
 
 if ispy3:
     assert not isinstance(remod, _LazyModule)
     assert f(remod) == "<module 're' from '?'>"
 else:
     assert f(remod) == "<unloaded module 're'>", f(remod)
 
 import sys as re
 
 if ispy3:
     assert not isinstance(re, _LazyModule)
     assert f(re) == "<module 'sys' (built-in)>"
 else:
     assert f(re) == "<unloaded module 'sys'>", f(re)
 
 if ispy3:
     assert not isinstance(fred, _LazyModule)
     assert f(fred) == "<module 're' from '?'>", f(fred)
 else:
     assert f(fred) == "<unloaded module 're'>", f(fred)
 
 assert f(fred.sub) == '<function sub at 0x?>', f(fred.sub)
 
 if ispy3:
     assert not isinstance(fred, _LazyModule)
     assert f(fred) == "<module 're' from '?'>", f(fred)
 else:
     assert f(fred) == "<proxied module 're'>", f(fred)
 
 remod.escape  # use remod
 assert f(remod) == "<module 're' from '?'>", f(remod)
 
 if ispy3:
     assert not isinstance(re, _LazyModule)
     assert f(re) == "<module 'sys' (built-in)>"
     assert f(type(re.stderr)) == "<class '_io.TextIOWrapper'>", f(
         type(re.stderr)
     )
     assert f(re) == "<module 'sys' (built-in)>"
 else:
     assert f(re) == "<unloaded module 'sys'>", f(re)
     assert f(re.stderr) == "<open file '<whatever>', mode 'w' at 0x?>", f(
         re.stderr
     )
     assert f(re) == "<proxied module 'sys'>", f(re)
 
 assert 'telnetlib' not in sys.modules
 import telnetlib
 
 if ispy3:
     assert isinstance(telnetlib, _LazyModule)
     assert f(telnetlib) == "<module 'telnetlib' from '?'>"
 else:
     assert f(telnetlib) == "<unloaded module 'telnetlib'>", f(telnetlib)
 
 try:
     from telnetlib import unknownattr
 
     assert False, (
         'no demandmod should be created for attribute of non-package '
         'module:\ntelnetlib.unknownattr = %s' % f(unknownattr)
     )
 except ImportError as inst:
     assert rsub(r"'", '', str(inst)).startswith(
         'cannot import name unknownattr'
     )
 
 from mercurial import util
 
 # Unlike the import statement, __import__() function should not raise
 # ImportError even if fromlist has an unknown item
 # (see Python/import.c:import_module_level() and ensure_fromlist())
 assert 'ftplib' not in sys.modules
 zipfileimp = __import__('ftplib', globals(), locals(), ['unknownattr'])
 assert f(zipfileimp) == "<module 'ftplib' from '?'>", f(zipfileimp)
 assert not util.safehasattr(zipfileimp, 'unknownattr')
+
+
+# test deactivation for issue6725
+del sys.modules['telnetlib']
+with demandimport.deactivated():
+    import telnetlib
+assert telnetlib.__loader__ == telnetlib.__spec__.loader
+assert telnetlib.__loader__.get_resource_reader