##// END OF EJS Templates
demandimport: strictly check missing locals argument...
Yuya Nishihara -
r32365:b2b56052 default
parent child Browse files
Show More
@@ -1,332 +1,332 b''
1 # demandimport.py - global demand-loading of modules for Mercurial
1 # demandimport.py - global demand-loading of modules for Mercurial
2 #
2 #
3 # Copyright 2006, 2007 Matt Mackall <mpm@selenic.com>
3 # Copyright 2006, 2007 Matt Mackall <mpm@selenic.com>
4 #
4 #
5 # This software may be used and distributed according to the terms of the
5 # This software may be used and distributed according to the terms of the
6 # GNU General Public License version 2 or any later version.
6 # GNU General Public License version 2 or any later version.
7
7
8 '''
8 '''
9 demandimport - automatic demandloading of modules
9 demandimport - automatic demandloading of modules
10
10
11 To enable this module, do:
11 To enable this module, do:
12
12
13 import demandimport; demandimport.enable()
13 import demandimport; demandimport.enable()
14
14
15 Imports of the following forms will be demand-loaded:
15 Imports of the following forms will be demand-loaded:
16
16
17 import a, b.c
17 import a, b.c
18 import a.b as c
18 import a.b as c
19 from a import b,c # a will be loaded immediately
19 from a import b,c # a will be loaded immediately
20
20
21 These imports will not be delayed:
21 These imports will not be delayed:
22
22
23 from a import *
23 from a import *
24 b = __import__(a)
24 b = __import__(a)
25 '''
25 '''
26
26
27 from __future__ import absolute_import
27 from __future__ import absolute_import
28
28
29 import contextlib
29 import contextlib
30 import os
30 import os
31 import sys
31 import sys
32
32
33 # __builtin__ in Python 2, builtins in Python 3.
33 # __builtin__ in Python 2, builtins in Python 3.
34 try:
34 try:
35 import __builtin__ as builtins
35 import __builtin__ as builtins
36 except ImportError:
36 except ImportError:
37 import builtins
37 import builtins
38
38
39 contextmanager = contextlib.contextmanager
39 contextmanager = contextlib.contextmanager
40
40
41 _origimport = __import__
41 _origimport = __import__
42
42
43 nothing = object()
43 nothing = object()
44
44
45 # Python 3 doesn't have relative imports nor level -1.
45 # Python 3 doesn't have relative imports nor level -1.
46 level = -1
46 level = -1
47 if sys.version_info[0] >= 3:
47 if sys.version_info[0] >= 3:
48 level = 0
48 level = 0
49 _import = _origimport
49 _import = _origimport
50
50
51 def _hgextimport(importfunc, name, globals, *args, **kwargs):
51 def _hgextimport(importfunc, name, globals, *args, **kwargs):
52 try:
52 try:
53 return importfunc(name, globals, *args, **kwargs)
53 return importfunc(name, globals, *args, **kwargs)
54 except ImportError:
54 except ImportError:
55 if not globals:
55 if not globals:
56 raise
56 raise
57 # extensions are loaded with "hgext_" prefix
57 # extensions are loaded with "hgext_" prefix
58 hgextname = 'hgext_%s' % name
58 hgextname = 'hgext_%s' % name
59 nameroot = hgextname.split('.', 1)[0]
59 nameroot = hgextname.split('.', 1)[0]
60 contextroot = globals.get('__name__', '').split('.', 1)[0]
60 contextroot = globals.get('__name__', '').split('.', 1)[0]
61 if nameroot != contextroot:
61 if nameroot != contextroot:
62 raise
62 raise
63 # retry to import with "hgext_" prefix
63 # retry to import with "hgext_" prefix
64 return importfunc(hgextname, globals, *args, **kwargs)
64 return importfunc(hgextname, globals, *args, **kwargs)
65
65
66 class _demandmod(object):
66 class _demandmod(object):
67 """module demand-loader and proxy
67 """module demand-loader and proxy
68
68
69 Specify 1 as 'level' argument at construction, to import module
69 Specify 1 as 'level' argument at construction, to import module
70 relatively.
70 relatively.
71 """
71 """
72 def __init__(self, name, globals, locals, level):
72 def __init__(self, name, globals, locals, level):
73 if '.' in name:
73 if '.' in name:
74 head, rest = name.split('.', 1)
74 head, rest = name.split('.', 1)
75 after = [rest]
75 after = [rest]
76 else:
76 else:
77 head = name
77 head = name
78 after = []
78 after = []
79 object.__setattr__(self, r"_data",
79 object.__setattr__(self, r"_data",
80 (head, globals, locals, after, level, set()))
80 (head, globals, locals, after, level, set()))
81 object.__setattr__(self, r"_module", None)
81 object.__setattr__(self, r"_module", None)
82 def _extend(self, name):
82 def _extend(self, name):
83 """add to the list of submodules to load"""
83 """add to the list of submodules to load"""
84 self._data[3].append(name)
84 self._data[3].append(name)
85
85
86 def _addref(self, name):
86 def _addref(self, name):
87 """Record that the named module ``name`` imports this module.
87 """Record that the named module ``name`` imports this module.
88
88
89 References to this proxy class having the name of this module will be
89 References to this proxy class having the name of this module will be
90 replaced at module load time. We assume the symbol inside the importing
90 replaced at module load time. We assume the symbol inside the importing
91 module is identical to the "head" name of this module. We don't
91 module is identical to the "head" name of this module. We don't
92 actually know if "as X" syntax is being used to change the symbol name
92 actually know if "as X" syntax is being used to change the symbol name
93 because this information isn't exposed to __import__.
93 because this information isn't exposed to __import__.
94 """
94 """
95 self._data[5].add(name)
95 self._data[5].add(name)
96
96
97 def _load(self):
97 def _load(self):
98 if not self._module:
98 if not self._module:
99 head, globals, locals, after, level, modrefs = self._data
99 head, globals, locals, after, level, modrefs = self._data
100 mod = _hgextimport(_import, head, globals, locals, None, level)
100 mod = _hgextimport(_import, head, globals, locals, None, level)
101 if mod is self:
101 if mod is self:
102 # In this case, _hgextimport() above should imply
102 # In this case, _hgextimport() above should imply
103 # _demandimport(). Otherwise, _hgextimport() never
103 # _demandimport(). Otherwise, _hgextimport() never
104 # returns _demandmod. This isn't intentional behavior,
104 # returns _demandmod. This isn't intentional behavior,
105 # in fact. (see also issue5304 for detail)
105 # in fact. (see also issue5304 for detail)
106 #
106 #
107 # If self._module is already bound at this point, self
107 # If self._module is already bound at this point, self
108 # should be already _load()-ed while _hgextimport().
108 # should be already _load()-ed while _hgextimport().
109 # Otherwise, there is no way to import actual module
109 # Otherwise, there is no way to import actual module
110 # as expected, because (re-)invoking _hgextimport()
110 # as expected, because (re-)invoking _hgextimport()
111 # should cause same result.
111 # should cause same result.
112 # This is reason why _load() returns without any more
112 # This is reason why _load() returns without any more
113 # setup but assumes self to be already bound.
113 # setup but assumes self to be already bound.
114 mod = self._module
114 mod = self._module
115 assert mod and mod is not self, "%s, %s" % (self, mod)
115 assert mod and mod is not self, "%s, %s" % (self, mod)
116 return
116 return
117
117
118 # load submodules
118 # load submodules
119 def subload(mod, p):
119 def subload(mod, p):
120 h, t = p, None
120 h, t = p, None
121 if '.' in p:
121 if '.' in p:
122 h, t = p.split('.', 1)
122 h, t = p.split('.', 1)
123 if getattr(mod, h, nothing) is nothing:
123 if getattr(mod, h, nothing) is nothing:
124 setattr(mod, h, _demandmod(p, mod.__dict__, mod.__dict__,
124 setattr(mod, h, _demandmod(p, mod.__dict__, mod.__dict__,
125 level=1))
125 level=1))
126 elif t:
126 elif t:
127 subload(getattr(mod, h), t)
127 subload(getattr(mod, h), t)
128
128
129 for x in after:
129 for x in after:
130 subload(mod, x)
130 subload(mod, x)
131
131
132 # Replace references to this proxy instance with the actual module.
132 # Replace references to this proxy instance with the actual module.
133 if locals and locals.get(head) == self:
133 if locals and locals.get(head) == self:
134 locals[head] = mod
134 locals[head] = mod
135
135
136 for modname in modrefs:
136 for modname in modrefs:
137 modref = sys.modules.get(modname, None)
137 modref = sys.modules.get(modname, None)
138 if modref and getattr(modref, head, None) == self:
138 if modref and getattr(modref, head, None) == self:
139 setattr(modref, head, mod)
139 setattr(modref, head, mod)
140
140
141 object.__setattr__(self, r"_module", mod)
141 object.__setattr__(self, r"_module", mod)
142
142
143 def __repr__(self):
143 def __repr__(self):
144 if self._module:
144 if self._module:
145 return "<proxied module '%s'>" % self._data[0]
145 return "<proxied module '%s'>" % self._data[0]
146 return "<unloaded module '%s'>" % self._data[0]
146 return "<unloaded module '%s'>" % self._data[0]
147 def __call__(self, *args, **kwargs):
147 def __call__(self, *args, **kwargs):
148 raise TypeError("%s object is not callable" % repr(self))
148 raise TypeError("%s object is not callable" % repr(self))
149 def __getattribute__(self, attr):
149 def __getattribute__(self, attr):
150 if attr in ('_data', '_extend', '_load', '_module', '_addref'):
150 if attr in ('_data', '_extend', '_load', '_module', '_addref'):
151 return object.__getattribute__(self, attr)
151 return object.__getattribute__(self, attr)
152 self._load()
152 self._load()
153 return getattr(self._module, attr)
153 return getattr(self._module, attr)
154 def __setattr__(self, attr, val):
154 def __setattr__(self, attr, val):
155 self._load()
155 self._load()
156 setattr(self._module, attr, val)
156 setattr(self._module, attr, val)
157
157
158 _pypy = '__pypy__' in sys.builtin_module_names
158 _pypy = '__pypy__' in sys.builtin_module_names
159
159
160 def _demandimport(name, globals=None, locals=None, fromlist=None, level=level):
160 def _demandimport(name, globals=None, locals=None, fromlist=None, level=level):
161 if not locals or name in ignore or fromlist == ('*',):
161 if locals is None or name in ignore or fromlist == ('*',):
162 # these cases we can't really delay
162 # these cases we can't really delay
163 return _hgextimport(_import, name, globals, locals, fromlist, level)
163 return _hgextimport(_import, name, globals, locals, fromlist, level)
164 elif not fromlist:
164 elif not fromlist:
165 # import a [as b]
165 # import a [as b]
166 if '.' in name: # a.b
166 if '.' in name: # a.b
167 base, rest = name.split('.', 1)
167 base, rest = name.split('.', 1)
168 # email.__init__ loading email.mime
168 # email.__init__ loading email.mime
169 if globals and globals.get('__name__', None) == base:
169 if globals and globals.get('__name__', None) == base:
170 return _import(name, globals, locals, fromlist, level)
170 return _import(name, globals, locals, fromlist, level)
171 # if a is already demand-loaded, add b to its submodule list
171 # if a is already demand-loaded, add b to its submodule list
172 if base in locals:
172 if base in locals:
173 if isinstance(locals[base], _demandmod):
173 if isinstance(locals[base], _demandmod):
174 locals[base]._extend(rest)
174 locals[base]._extend(rest)
175 return locals[base]
175 return locals[base]
176 return _demandmod(name, globals, locals, level)
176 return _demandmod(name, globals, locals, level)
177 else:
177 else:
178 # There is a fromlist.
178 # There is a fromlist.
179 # from a import b,c,d
179 # from a import b,c,d
180 # from . import b,c,d
180 # from . import b,c,d
181 # from .a import b,c,d
181 # from .a import b,c,d
182
182
183 # level == -1: relative and absolute attempted (Python 2 only).
183 # level == -1: relative and absolute attempted (Python 2 only).
184 # level >= 0: absolute only (Python 2 w/ absolute_import and Python 3).
184 # level >= 0: absolute only (Python 2 w/ absolute_import and Python 3).
185 # The modern Mercurial convention is to use absolute_import everywhere,
185 # The modern Mercurial convention is to use absolute_import everywhere,
186 # so modern Mercurial code will have level >= 0.
186 # so modern Mercurial code will have level >= 0.
187
187
188 # The name of the module the import statement is located in.
188 # The name of the module the import statement is located in.
189 globalname = globals.get('__name__')
189 globalname = globals.get('__name__')
190
190
191 def processfromitem(mod, attr):
191 def processfromitem(mod, attr):
192 """Process an imported symbol in the import statement.
192 """Process an imported symbol in the import statement.
193
193
194 If the symbol doesn't exist in the parent module, and if the
194 If the symbol doesn't exist in the parent module, and if the
195 parent module is a package, it must be a module. We set missing
195 parent module is a package, it must be a module. We set missing
196 modules up as _demandmod instances.
196 modules up as _demandmod instances.
197 """
197 """
198 symbol = getattr(mod, attr, nothing)
198 symbol = getattr(mod, attr, nothing)
199 nonpkg = getattr(mod, '__path__', nothing) is nothing
199 nonpkg = getattr(mod, '__path__', nothing) is nothing
200 if symbol is nothing:
200 if symbol is nothing:
201 if nonpkg:
201 if nonpkg:
202 # do not try relative import, which would raise ValueError,
202 # do not try relative import, which would raise ValueError,
203 # and leave unknown attribute as the default __import__()
203 # and leave unknown attribute as the default __import__()
204 # would do. the missing attribute will be detected later
204 # would do. the missing attribute will be detected later
205 # while processing the import statement.
205 # while processing the import statement.
206 return
206 return
207 mn = '%s.%s' % (mod.__name__, attr)
207 mn = '%s.%s' % (mod.__name__, attr)
208 if mn in ignore:
208 if mn in ignore:
209 importfunc = _origimport
209 importfunc = _origimport
210 else:
210 else:
211 importfunc = _demandmod
211 importfunc = _demandmod
212 symbol = importfunc(attr, mod.__dict__, locals, level=1)
212 symbol = importfunc(attr, mod.__dict__, locals, level=1)
213 setattr(mod, attr, symbol)
213 setattr(mod, attr, symbol)
214
214
215 # Record the importing module references this symbol so we can
215 # Record the importing module references this symbol so we can
216 # replace the symbol with the actual module instance at load
216 # replace the symbol with the actual module instance at load
217 # time.
217 # time.
218 if globalname and isinstance(symbol, _demandmod):
218 if globalname and isinstance(symbol, _demandmod):
219 symbol._addref(globalname)
219 symbol._addref(globalname)
220
220
221 def chainmodules(rootmod, modname):
221 def chainmodules(rootmod, modname):
222 # recurse down the module chain, and return the leaf module
222 # recurse down the module chain, and return the leaf module
223 mod = rootmod
223 mod = rootmod
224 for comp in modname.split('.')[1:]:
224 for comp in modname.split('.')[1:]:
225 if getattr(mod, comp, nothing) is nothing:
225 if getattr(mod, comp, nothing) is nothing:
226 setattr(mod, comp, _demandmod(comp, mod.__dict__,
226 setattr(mod, comp, _demandmod(comp, mod.__dict__,
227 mod.__dict__, level=1))
227 mod.__dict__, level=1))
228 mod = getattr(mod, comp)
228 mod = getattr(mod, comp)
229 return mod
229 return mod
230
230
231 if level >= 0:
231 if level >= 0:
232 if name:
232 if name:
233 # "from a import b" or "from .a import b" style
233 # "from a import b" or "from .a import b" style
234 rootmod = _hgextimport(_origimport, name, globals, locals,
234 rootmod = _hgextimport(_origimport, name, globals, locals,
235 level=level)
235 level=level)
236 mod = chainmodules(rootmod, name)
236 mod = chainmodules(rootmod, name)
237 elif _pypy:
237 elif _pypy:
238 # PyPy's __import__ throws an exception if invoked
238 # PyPy's __import__ throws an exception if invoked
239 # with an empty name and no fromlist. Recreate the
239 # with an empty name and no fromlist. Recreate the
240 # desired behaviour by hand.
240 # desired behaviour by hand.
241 mn = globalname
241 mn = globalname
242 mod = sys.modules[mn]
242 mod = sys.modules[mn]
243 if getattr(mod, '__path__', nothing) is nothing:
243 if getattr(mod, '__path__', nothing) is nothing:
244 mn = mn.rsplit('.', 1)[0]
244 mn = mn.rsplit('.', 1)[0]
245 mod = sys.modules[mn]
245 mod = sys.modules[mn]
246 if level > 1:
246 if level > 1:
247 mn = mn.rsplit('.', level - 1)[0]
247 mn = mn.rsplit('.', level - 1)[0]
248 mod = sys.modules[mn]
248 mod = sys.modules[mn]
249 else:
249 else:
250 mod = _hgextimport(_origimport, name, globals, locals,
250 mod = _hgextimport(_origimport, name, globals, locals,
251 level=level)
251 level=level)
252
252
253 for x in fromlist:
253 for x in fromlist:
254 processfromitem(mod, x)
254 processfromitem(mod, x)
255
255
256 return mod
256 return mod
257
257
258 # But, we still need to support lazy loading of standard library and 3rd
258 # But, we still need to support lazy loading of standard library and 3rd
259 # party modules. So handle level == -1.
259 # party modules. So handle level == -1.
260 mod = _hgextimport(_origimport, name, globals, locals)
260 mod = _hgextimport(_origimport, name, globals, locals)
261 mod = chainmodules(mod, name)
261 mod = chainmodules(mod, name)
262
262
263 for x in fromlist:
263 for x in fromlist:
264 processfromitem(mod, x)
264 processfromitem(mod, x)
265
265
266 return mod
266 return mod
267
267
268 ignore = [
268 ignore = [
269 '__future__',
269 '__future__',
270 '_hashlib',
270 '_hashlib',
271 # ImportError during pkg_resources/__init__.py:fixup_namespace_package
271 # ImportError during pkg_resources/__init__.py:fixup_namespace_package
272 '_imp',
272 '_imp',
273 '_xmlplus',
273 '_xmlplus',
274 'fcntl',
274 'fcntl',
275 'nt', # pathlib2 tests the existence of built-in 'nt' module
275 'nt', # pathlib2 tests the existence of built-in 'nt' module
276 'win32com.gen_py',
276 'win32com.gen_py',
277 'win32com.shell', # 'appdirs' tries to import win32com.shell
277 'win32com.shell', # 'appdirs' tries to import win32com.shell
278 '_winreg', # 2.7 mimetypes needs immediate ImportError
278 '_winreg', # 2.7 mimetypes needs immediate ImportError
279 'pythoncom',
279 'pythoncom',
280 # imported by tarfile, not available under Windows
280 # imported by tarfile, not available under Windows
281 'pwd',
281 'pwd',
282 'grp',
282 'grp',
283 # imported by profile, itself imported by hotshot.stats,
283 # imported by profile, itself imported by hotshot.stats,
284 # not available under Windows
284 # not available under Windows
285 'resource',
285 'resource',
286 # this trips up many extension authors
286 # this trips up many extension authors
287 'gtk',
287 'gtk',
288 # setuptools' pkg_resources.py expects "from __main__ import x" to
288 # setuptools' pkg_resources.py expects "from __main__ import x" to
289 # raise ImportError if x not defined
289 # raise ImportError if x not defined
290 '__main__',
290 '__main__',
291 '_ssl', # conditional imports in the stdlib, issue1964
291 '_ssl', # conditional imports in the stdlib, issue1964
292 '_sre', # issue4920
292 '_sre', # issue4920
293 'rfc822',
293 'rfc822',
294 'mimetools',
294 'mimetools',
295 'sqlalchemy.events', # has import-time side effects (issue5085)
295 'sqlalchemy.events', # has import-time side effects (issue5085)
296 # setuptools 8 expects this module to explode early when not on windows
296 # setuptools 8 expects this module to explode early when not on windows
297 'distutils.msvc9compiler',
297 'distutils.msvc9compiler',
298 '__builtin__',
298 '__builtin__',
299 'builtins',
299 'builtins',
300 'urwid.command_map', # for pudb
300 'urwid.command_map', # for pudb
301 ]
301 ]
302
302
303 if _pypy:
303 if _pypy:
304 ignore.extend([
304 ignore.extend([
305 # _ctypes.pointer is shadowed by "from ... import pointer" (PyPy 5)
305 # _ctypes.pointer is shadowed by "from ... import pointer" (PyPy 5)
306 '_ctypes.pointer',
306 '_ctypes.pointer',
307 ])
307 ])
308
308
309 def isenabled():
309 def isenabled():
310 return builtins.__import__ == _demandimport
310 return builtins.__import__ == _demandimport
311
311
312 def enable():
312 def enable():
313 "enable global demand-loading of modules"
313 "enable global demand-loading of modules"
314 if os.environ.get('HGDEMANDIMPORT') != 'disable':
314 if os.environ.get('HGDEMANDIMPORT') != 'disable':
315 builtins.__import__ = _demandimport
315 builtins.__import__ = _demandimport
316
316
317 def disable():
317 def disable():
318 "disable global demand-loading of modules"
318 "disable global demand-loading of modules"
319 builtins.__import__ = _origimport
319 builtins.__import__ = _origimport
320
320
321 @contextmanager
321 @contextmanager
322 def deactivated():
322 def deactivated():
323 "context manager for disabling demandimport in 'with' blocks"
323 "context manager for disabling demandimport in 'with' blocks"
324 demandenabled = isenabled()
324 demandenabled = isenabled()
325 if demandenabled:
325 if demandenabled:
326 disable()
326 disable()
327
327
328 try:
328 try:
329 yield
329 yield
330 finally:
330 finally:
331 if demandenabled:
331 if demandenabled:
332 enable()
332 enable()
General Comments 0
You need to be logged in to leave comments. Login now