demandimport: avoid infinite recursion at actual module importing (issue5304)

Before this patch, importing C module on Windows environment causes
infinite recursion call, if py2exe is used with -b2 option.

At importing C module "a.b", extra hooking by zipextimporter of py2exe
causes:

  0. assumption before accessing "b" of "a":

     - built-in module object is created for "a",
       (= "a" is actually imported)
     - _demandmod is created for "a.b" as a proxy object, and
       (= "a.b" is not yet imported)
     - an attribute "b" of "a" is initialized by the latter

  1. invocation of __import__ via _hgextimport() in _demandmod._load()
     for "a.b" implies _demandimport() for "a.b"

     This is unintentional, because _demandmod might be returned by
     _hgextimport() instead of built-in module object.

  2. _demandimport() at (1) is invoked with not context of "a", but
     context of zipextimporter

     Just after invocation of _hgextimport() in _demandimport(), an
     attribute "b" of the built-in module object for "a" is still
     bound to the proxy object for "a.b", because context of "a" isn't
     updated by actual importing "a.b". even though the built-in
     module object for "a.b" already appears in sys.modules.

     Therefore, chainmodules() returns _demandmod for "a.b", which is
     gotten from the attribute "b" of "a".

  3. processfromitem() on "a.b" causes _demandmod._load() for "a.b"
     again

     _demandimport() takes context of "a" in this case.

     Therefore, attributes below are bound to built-in module object
     for "a.b", as expected:

     - "b" of built-in module object for "a"
     - _module of _demandmod for "a.b"

  4. but _demandimport() invoked at (1) returns _demandmod object

     because _demandimport() just returns the object returned by
     chainmodules() at (3) above.

  5. then, _demandmod._load() causes infinite recursion call

     _demandimport() returns _demandmod for "a.b", and it is "self" at
     _demandmod._load().

To avoid infinite recursion at actual module importing, this patch
uses self._module, if _hgextimport() returns _demandmod itself. If
_demandmod._module isn't yet bound at this point, execution should be
aborted, because actual importing failed.

In this patch, _demandmod._module is examined not on _demandimport()
side, but on _demandmod._load() side, because:

  - the former has some exit points
  - only the latter uses _hgextimport(), except for _demandimport()

BTW, this issue occurs only in the code path for non .py/.pyc files in
zipextimporter (strictly speaking, in _memimporter) of py2exe.

Even if zipextimporter is enabled, .py/.pyc files are handled by
zipimporter, and it doesn't imply unintentional _demandimport() at
invocation of __import__ via _hgextimport().

mercurial/demandimport.py

@@ -1,291 +1,308 b''
 # demandimport.py - global demand-loading of modules for Mercurial
 #
 # Copyright 2006, 2007 Matt Mackall <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 '''
 demandimport - automatic demandloading of modules
 
 To enable this module, do:
 
   import demandimport; demandimport.enable()
 
 Imports of the following forms will be demand-loaded:
 
   import a, b.c
   import a.b as c
   from a import b,c # a will be loaded immediately
 
 These imports will not be delayed:
 
   from a import *
   b = __import__(a)
 '''
 
 from __future__ import absolute_import
 
 import contextlib
 import os
 import sys
 
 # __builtin__ in Python 2, builtins in Python 3.
 try:
     import __builtin__ as builtins
 except ImportError:
     import builtins
 
 contextmanager = contextlib.contextmanager
 
 _origimport = __import__
 
 nothing = object()
 
 # Python 3 doesn't have relative imports nor level -1.
 level = -1
 if sys.version_info[0] >= 3:
     level = 0
 _import = _origimport
 
 def _hgextimport(importfunc, name, globals, *args, **kwargs):
     try:
         return importfunc(name, globals, *args, **kwargs)
     except ImportError:
         if not globals:
             raise
         # extensions are loaded with "hgext_" prefix
         hgextname = 'hgext_%s' % name
         nameroot = hgextname.split('.', 1)[0]
         contextroot = globals.get('__name__', '').split('.', 1)[0]
         if nameroot != contextroot:
             raise
         # retry to import with "hgext_" prefix
         return importfunc(hgextname, globals, *args, **kwargs)
 
 class _demandmod(object):
     """module demand-loader and proxy"""
     def __init__(self, name, globals, locals, level=level):
         if '.' in name:
             head, rest = name.split('.', 1)
             after = [rest]
         else:
             head = name
             after = []
         object.__setattr__(self, "_data",
                            (head, globals, locals, after, level, set()))
         object.__setattr__(self, "_module", None)
     def _extend(self, name):
         """add to the list of submodules to load"""
         self._data[3].append(name)
 
     def _addref(self, name):
         """Record that the named module ``name`` imports this module.
 
         References to this proxy class having the name of this module will be
         replaced at module load time. We assume the symbol inside the importing
         module is identical to the "head" name of this module. We don't
         actually know if "as X" syntax is being used to change the symbol name
         because this information isn't exposed to __import__.
         """
         self._data[5].add(name)
 
     def _load(self):
         if not self._module:
             head, globals, locals, after, level, modrefs = self._data
             mod = _hgextimport(_import, head, globals, locals, None, level)
+            if mod is self:
+                # In this case, _hgextimport() above should imply
+                # _demandimport(). Otherwise, _hgextimport() never
+                # returns _demandmod. This isn't intentional behavior,
+                # in fact. (see also issue5304 for detail)
+                #
+                # If self._module is already bound at this point, self
+                # should be already _load()-ed while _hgextimport().
+                # Otherwise, there is no way to import actual module
+                # as expected, because (re-)invoking _hgextimport()
+                # should cause same result.
+                # This is reason why _load() returns without any more
+                # setup but assumes self to be already bound.
+                mod = self._module
+                assert mod and mod is not self, "%s, %s" % (self, mod)
+                return
+
             # load submodules
             def subload(mod, p):
                 h, t = p, None
                 if '.' in p:
                     h, t = p.split('.', 1)
                 if getattr(mod, h, nothing) is nothing:
                     setattr(mod, h, _demandmod(p, mod.__dict__, mod.__dict__))
                 elif t:
                     subload(getattr(mod, h), t)
 
             for x in after:
                 subload(mod, x)
 
             # Replace references to this proxy instance with the actual module.
             if locals and locals.get(head) == self:
                 locals[head] = mod
 
             for modname in modrefs:
                 modref = sys.modules.get(modname, None)
                 if modref and getattr(modref, head, None) == self:
                     setattr(modref, head, mod)
 
             object.__setattr__(self, "_module", mod)
 
     def __repr__(self):
         if self._module:
             return "<proxied module '%s'>" % self._data[0]
         return "<unloaded module '%s'>" % self._data[0]
     def __call__(self, *args, **kwargs):
         raise TypeError("%s object is not callable" % repr(self))
     def __getattribute__(self, attr):
         if attr in ('_data', '_extend', '_load', '_module', '_addref'):
             return object.__getattribute__(self, attr)
         self._load()
         return getattr(self._module, attr)
     def __setattr__(self, attr, val):
         self._load()
         setattr(self._module, attr, val)
 
 _pypy = '__pypy__' in sys.builtin_module_names
 
 def _demandimport(name, globals=None, locals=None, fromlist=None, level=level):
     if not locals or name in ignore or fromlist == ('*',):
         # these cases we can't really delay
         return _hgextimport(_import, name, globals, locals, fromlist, level)
     elif not fromlist:
         # import a [as b]
         if '.' in name: # a.b
             base, rest = name.split('.', 1)
             # email.__init__ loading email.mime
             if globals and globals.get('__name__', None) == base:
                 return _import(name, globals, locals, fromlist, level)
             # if a is already demand-loaded, add b to its submodule list
             if base in locals:
                 if isinstance(locals[base], _demandmod):
                     locals[base]._extend(rest)
                 return locals[base]
         return _demandmod(name, globals, locals, level)
     else:
         # There is a fromlist.
         # from a import b,c,d
         # from . import b,c,d
         # from .a import b,c,d
 
         # level == -1: relative and absolute attempted (Python 2 only).
         # level >= 0: absolute only (Python 2 w/ absolute_import and Python 3).
         # The modern Mercurial convention is to use absolute_import everywhere,
         # so modern Mercurial code will have level >= 0.
 
         # The name of the module the import statement is located in.
         globalname = globals.get('__name__')
 
         def processfromitem(mod, attr):
             """Process an imported symbol in the import statement.
 
             If the symbol doesn't exist in the parent module, it must be a
             module. We set missing modules up as _demandmod instances.
             """
             symbol = getattr(mod, attr, nothing)
             if symbol is nothing:
                 mn = '%s.%s' % (mod.__name__, attr)
                 if mn in ignore:
                     importfunc = _origimport
                 else:
                     importfunc = _demandmod
                 symbol = importfunc(attr, mod.__dict__, locals, level=1)
                 setattr(mod, attr, symbol)
 
             # Record the importing module references this symbol so we can
             # replace the symbol with the actual module instance at load
             # time.
             if globalname and isinstance(symbol, _demandmod):
                 symbol._addref(globalname)
 
         def chainmodules(rootmod, modname):
             # recurse down the module chain, and return the leaf module
             mod = rootmod
             for comp in modname.split('.')[1:]:
                 if getattr(mod, comp, nothing) is nothing:
                     setattr(mod, comp,
                             _demandmod(comp, mod.__dict__, mod.__dict__))
                 mod = getattr(mod, comp)
             return mod
 
         if level >= 0:
             if name:
                 # "from a import b" or "from .a import b" style
                 rootmod = _hgextimport(_origimport, name, globals, locals,
                                        level=level)
                 mod = chainmodules(rootmod, name)
             elif _pypy:
                 # PyPy's __import__ throws an exception if invoked
                 # with an empty name and no fromlist.  Recreate the
                 # desired behaviour by hand.
                 mn = globalname
                 mod = sys.modules[mn]
                 if getattr(mod, '__path__', nothing) is nothing:
                     mn = mn.rsplit('.', 1)[0]
                     mod = sys.modules[mn]
                 if level > 1:
                     mn = mn.rsplit('.', level - 1)[0]
                     mod = sys.modules[mn]
             else:
                 mod = _hgextimport(_origimport, name, globals, locals,
                                    level=level)
 
             for x in fromlist:
                 processfromitem(mod, x)
 
             return mod
 
         # But, we still need to support lazy loading of standard library and 3rd
         # party modules. So handle level == -1.
         mod = _hgextimport(_origimport, name, globals, locals)
         mod = chainmodules(mod, name)
 
         for x in fromlist:
             processfromitem(mod, x)
 
         return mod
 
 ignore = [
     '__future__',
     '_hashlib',
     # ImportError during pkg_resources/__init__.py:fixup_namespace_package
     '_imp',
     '_xmlplus',
     'fcntl',
     'win32com.gen_py',
     '_winreg', # 2.7 mimetypes needs immediate ImportError
     'pythoncom',
     # imported by tarfile, not available under Windows
     'pwd',
     'grp',
     # imported by profile, itself imported by hotshot.stats,
     # not available under Windows
     'resource',
     # this trips up many extension authors
     'gtk',
     # setuptools' pkg_resources.py expects "from __main__ import x" to
     # raise ImportError if x not defined
     '__main__',
     '_ssl', # conditional imports in the stdlib, issue1964
     '_sre', # issue4920
     'rfc822',
     'mimetools',
     'sqlalchemy.events', # has import-time side effects (issue5085)
     # setuptools 8 expects this module to explode early when not on windows
     'distutils.msvc9compiler'
     ]
 
 def isenabled():
     return builtins.__import__ == _demandimport
 
 def enable():
     "enable global demand-loading of modules"
     if os.environ.get('HGDEMANDIMPORT') != 'disable':
         builtins.__import__ = _demandimport
 
 def disable():
     "disable global demand-loading of modules"
     builtins.__import__ = _origimport
 
 @contextmanager
 def deactivated():
     "context manager for disabling demandimport in 'with' blocks"
     demandenabled = isenabled()
     if demandenabled:
         disable()
 
     try:
         yield
     finally:
         if demandenabled:
             enable()