##// END OF EJS Templates
Ensure modules for autoreload have a __loader__ attribute.
Thomas Kluyver -
Show More
@@ -1,536 +1,537
1 1 """IPython extension to reload modules before executing user code.
2 2
3 3 ``autoreload`` reloads modules automatically before entering the execution of
4 4 code typed at the IPython prompt.
5 5
6 6 This makes for example the following workflow possible:
7 7
8 8 .. sourcecode:: ipython
9 9
10 10 In [1]: %load_ext autoreload
11 11
12 12 In [2]: %autoreload 2
13 13
14 14 In [3]: from foo import some_function
15 15
16 16 In [4]: some_function()
17 17 Out[4]: 42
18 18
19 19 In [5]: # open foo.py in an editor and change some_function to return 43
20 20
21 21 In [6]: some_function()
22 22 Out[6]: 43
23 23
24 24 The module was reloaded without reloading it explicitly, and the object
25 25 imported with ``from foo import ...`` was also updated.
26 26
27 27 Usage
28 28 =====
29 29
30 30 The following magic commands are provided:
31 31
32 32 ``%autoreload``
33 33
34 34 Reload all modules (except those excluded by ``%aimport``)
35 35 automatically now.
36 36
37 37 ``%autoreload 0``
38 38
39 39 Disable automatic reloading.
40 40
41 41 ``%autoreload 1``
42 42
43 43 Reload all modules imported with ``%aimport`` every time before
44 44 executing the Python code typed.
45 45
46 46 ``%autoreload 2``
47 47
48 48 Reload all modules (except those excluded by ``%aimport``) every
49 49 time before executing the Python code typed.
50 50
51 51 ``%aimport``
52 52
53 53 List modules which are to be automatically imported or not to be imported.
54 54
55 55 ``%aimport foo``
56 56
57 57 Import module 'foo' and mark it to be autoreloaded for ``%autoreload 1``
58 58
59 59 ``%aimport -foo``
60 60
61 61 Mark module 'foo' to not be autoreloaded.
62 62
63 63 Caveats
64 64 =======
65 65
66 66 Reloading Python modules in a reliable way is in general difficult,
67 67 and unexpected things may occur. ``%autoreload`` tries to work around
68 68 common pitfalls by replacing function code objects and parts of
69 69 classes previously in the module with new versions. This makes the
70 70 following things to work:
71 71
72 72 - Functions and classes imported via 'from xxx import foo' are upgraded
73 73 to new versions when 'xxx' is reloaded.
74 74
75 75 - Methods and properties of classes are upgraded on reload, so that
76 76 calling 'c.foo()' on an object 'c' created before the reload causes
77 77 the new code for 'foo' to be executed.
78 78
79 79 Some of the known remaining caveats are:
80 80
81 81 - Replacing code objects does not always succeed: changing a @property
82 82 in a class to an ordinary method or a method to a member variable
83 83 can cause problems (but in old objects only).
84 84
85 85 - Functions that are removed (eg. via monkey-patching) from a module
86 86 before it is reloaded are not upgraded.
87 87
88 88 - C extension modules cannot be reloaded, and so cannot be autoreloaded.
89 89 """
90 90
91 91 skip_doctest = True
92 92
93 93 #-----------------------------------------------------------------------------
94 94 # Copyright (C) 2000 Thomas Heller
95 95 # Copyright (C) 2008 Pauli Virtanen <pav@iki.fi>
96 96 # Copyright (C) 2012 The IPython Development Team
97 97 #
98 98 # Distributed under the terms of the BSD License. The full license is in
99 99 # the file COPYING, distributed as part of this software.
100 100 #-----------------------------------------------------------------------------
101 101 #
102 102 # This IPython module is written by Pauli Virtanen, based on the autoreload
103 103 # code by Thomas Heller.
104 104
105 105 #-----------------------------------------------------------------------------
106 106 # Imports
107 107 #-----------------------------------------------------------------------------
108 108 import atexit
109 109 import imp
110 110 import inspect
111 111 import os
112 112 import sys
113 113 import threading
114 114 import time
115 115 import traceback
116 116 import types
117 117 import weakref
118 118
119 119 try:
120 120 # Reload is not defined by default in Python3.
121 121 reload
122 122 except NameError:
123 123 from imp import reload
124 124
125 125 from IPython.utils import pyfile
126 126 from IPython.utils.py3compat import PY3
127 127
128 128 #------------------------------------------------------------------------------
129 129 # Autoreload functionality
130 130 #------------------------------------------------------------------------------
131 131
132 132 def _get_compiled_ext():
133 133 """Official way to get the extension of compiled files (.pyc or .pyo)"""
134 134 for ext, mode, typ in imp.get_suffixes():
135 135 if typ == imp.PY_COMPILED:
136 136 return ext
137 137
138 138
139 139 PY_COMPILED_EXT = _get_compiled_ext()
140 140
141 141
142 142 class ModuleReloader(object):
143 143 enabled = False
144 144 """Whether this reloader is enabled"""
145 145
146 146 failed = {}
147 147 """Modules that failed to reload: {module: mtime-on-failed-reload, ...}"""
148 148
149 149 modules = {}
150 150 """Modules specially marked as autoreloadable."""
151 151
152 152 skip_modules = {}
153 153 """Modules specially marked as not autoreloadable."""
154 154
155 155 check_all = True
156 156 """Autoreload all modules, not just those listed in 'modules'"""
157 157
158 158 old_objects = {}
159 159 """(module-name, name) -> weakref, for replacing old code objects"""
160 160
161 161 def mark_module_skipped(self, module_name):
162 162 """Skip reloading the named module in the future"""
163 163 try:
164 164 del self.modules[module_name]
165 165 except KeyError:
166 166 pass
167 167 self.skip_modules[module_name] = True
168 168
169 169 def mark_module_reloadable(self, module_name):
170 170 """Reload the named module in the future (if it is imported)"""
171 171 try:
172 172 del self.skip_modules[module_name]
173 173 except KeyError:
174 174 pass
175 175 self.modules[module_name] = True
176 176
177 177 def aimport_module(self, module_name):
178 178 """Import a module, and mark it reloadable
179 179
180 180 Returns
181 181 -------
182 182 top_module : module
183 183 The imported module if it is top-level, or the top-level
184 184 top_name : module
185 185 Name of top_module
186 186
187 187 """
188 188 self.mark_module_reloadable(module_name)
189 189
190 190 __import__(module_name)
191 191 top_name = module_name.split('.')[0]
192 192 top_module = sys.modules[top_name]
193 193 return top_module, top_name
194 194
195 195 def check(self, check_all=False):
196 196 """Check whether some modules need to be reloaded."""
197 197
198 198 if not self.enabled and not check_all:
199 199 return
200 200
201 201 if check_all or self.check_all:
202 202 modules = sys.modules.keys()
203 203 else:
204 204 modules = self.modules.keys()
205 205
206 206 for modname in modules:
207 207 m = sys.modules.get(modname, None)
208 208
209 209 if modname in self.skip_modules:
210 210 continue
211 211
212 212 if not hasattr(m, '__file__'):
213 213 continue
214 214
215 215 if m.__name__ == '__main__':
216 216 # we cannot reload(__main__)
217 217 continue
218 218
219 219 filename = m.__file__
220 220 path, ext = os.path.splitext(filename)
221 221
222 222 if ext.lower() == '.py':
223 223 ext = PY_COMPILED_EXT
224 224 pyc_filename = pyfile.cache_from_source(filename)
225 225 py_filename = filename
226 226 else:
227 227 pyc_filename = filename
228 228 try:
229 229 py_filename = pyfile.source_from_cache(filename)
230 230 except ValueError:
231 231 continue
232 232
233 233 try:
234 234 pymtime = os.stat(py_filename).st_mtime
235 235 if pymtime <= os.stat(pyc_filename).st_mtime:
236 236 continue
237 237 if self.failed.get(py_filename, None) == pymtime:
238 238 continue
239 239 except OSError:
240 240 continue
241 241
242 242 try:
243 243 superreload(m, reload, self.old_objects)
244 244 if py_filename in self.failed:
245 245 del self.failed[py_filename]
246 246 except:
247 247 print >> sys.stderr, "[autoreload of %s failed: %s]" % (
248 248 modname, traceback.format_exc(1))
249 249 self.failed[py_filename] = pymtime
250 250
251 251 #------------------------------------------------------------------------------
252 252 # superreload
253 253 #------------------------------------------------------------------------------
254 254
255 255 if PY3:
256 256 func_attrs = ['__code__', '__defaults__', '__doc__',
257 257 '__closure__', '__globals__', '__dict__']
258 258 else:
259 259 func_attrs = ['func_code', 'func_defaults', 'func_doc',
260 260 'func_closure', 'func_globals', 'func_dict']
261 261
262 262
263 263 def update_function(old, new):
264 264 """Upgrade the code object of a function"""
265 265 for name in func_attrs:
266 266 try:
267 267 setattr(old, name, getattr(new, name))
268 268 except (AttributeError, TypeError):
269 269 pass
270 270
271 271
272 272 def update_class(old, new):
273 273 """Replace stuff in the __dict__ of a class, and upgrade
274 274 method code objects"""
275 275 for key in old.__dict__.keys():
276 276 old_obj = getattr(old, key)
277 277
278 278 try:
279 279 new_obj = getattr(new, key)
280 280 except AttributeError:
281 281 # obsolete attribute: remove it
282 282 try:
283 283 delattr(old, key)
284 284 except (AttributeError, TypeError):
285 285 pass
286 286 continue
287 287
288 288 if update_generic(old_obj, new_obj): continue
289 289
290 290 try:
291 291 setattr(old, key, getattr(new, key))
292 292 except (AttributeError, TypeError):
293 293 pass # skip non-writable attributes
294 294
295 295
296 296 def update_property(old, new):
297 297 """Replace get/set/del functions of a property"""
298 298 update_generic(old.fdel, new.fdel)
299 299 update_generic(old.fget, new.fget)
300 300 update_generic(old.fset, new.fset)
301 301
302 302
303 303 def isinstance2(a, b, typ):
304 304 return isinstance(a, typ) and isinstance(b, typ)
305 305
306 306
307 307 UPDATE_RULES = [
308 308 (lambda a, b: isinstance2(a, b, type),
309 309 update_class),
310 310 (lambda a, b: isinstance2(a, b, types.FunctionType),
311 311 update_function),
312 312 (lambda a, b: isinstance2(a, b, property),
313 313 update_property),
314 314 ]
315 315
316 316
317 317 if PY3:
318 318 UPDATE_RULES.extend([(lambda a, b: isinstance2(a, b, types.MethodType),
319 319 lambda a, b: update_function(a.__func__, b.__func__)),
320 320 ])
321 321 else:
322 322 UPDATE_RULES.extend([(lambda a, b: isinstance2(a, b, types.ClassType),
323 323 update_class),
324 324 (lambda a, b: isinstance2(a, b, types.MethodType),
325 325 lambda a, b: update_function(a.im_func, b.im_func)),
326 326 ])
327 327
328 328
329 329 def update_generic(a, b):
330 330 for type_check, update in UPDATE_RULES:
331 331 if type_check(a, b):
332 332 update(a, b)
333 333 return True
334 334 return False
335 335
336 336
337 337 class StrongRef(object):
338 338 def __init__(self, obj):
339 339 self.obj = obj
340 340 def __call__(self):
341 341 return self.obj
342 342
343 343
344 344 def superreload(module, reload=reload, old_objects={}):
345 345 """Enhanced version of the builtin reload function.
346 346
347 347 superreload remembers objects previously in the module, and
348 348
349 349 - upgrades the class dictionary of every old class in the module
350 350 - upgrades the code object of every old function and method
351 351 - clears the module's namespace before reloading
352 352
353 353 """
354 354
355 355 # collect old objects in the module
356 356 for name, obj in module.__dict__.items():
357 357 if not hasattr(obj, '__module__') or obj.__module__ != module.__name__:
358 358 continue
359 359 key = (module.__name__, name)
360 360 try:
361 361 old_objects.setdefault(key, []).append(weakref.ref(obj))
362 362 except TypeError:
363 363 # weakref doesn't work for all types;
364 364 # create strong references for 'important' cases
365 365 if not PY3 and isinstance(obj, types.ClassType):
366 366 old_objects.setdefault(key, []).append(StrongRef(obj))
367 367
368 368 # reload module
369 369 try:
370 370 # clear namespace first from old cruft
371 371 old_dict = module.__dict__.copy()
372 372 old_name = module.__name__
373 373 module.__dict__.clear()
374 374 module.__dict__['__name__'] = old_name
375 module.__dict__['__loader__'] = old_dict['__loader__']
375 376 except (TypeError, AttributeError, KeyError):
376 377 pass
377 378
378 379 try:
379 380 module = reload(module)
380 381 except:
381 382 # restore module dictionary on failed reload
382 383 module.__dict__.update(old_dict)
383 384 raise
384 385
385 386 # iterate over all objects and update functions & classes
386 387 for name, new_obj in module.__dict__.items():
387 388 key = (module.__name__, name)
388 389 if key not in old_objects: continue
389 390
390 391 new_refs = []
391 392 for old_ref in old_objects[key]:
392 393 old_obj = old_ref()
393 394 if old_obj is None: continue
394 395 new_refs.append(old_ref)
395 396 update_generic(old_obj, new_obj)
396 397
397 398 if new_refs:
398 399 old_objects[key] = new_refs
399 400 else:
400 401 del old_objects[key]
401 402
402 403 return module
403 404
404 405 #------------------------------------------------------------------------------
405 406 # IPython connectivity
406 407 #------------------------------------------------------------------------------
407 408
408 409 from IPython.core.hooks import TryNext
409 410 from IPython.core.magic import Magics, magics_class, line_magic
410 411 from IPython.core.plugin import Plugin
411 412
412 413 @magics_class
413 414 class AutoreloadMagics(Magics):
414 415 def __init__(self, *a, **kw):
415 416 super(AutoreloadMagics, self).__init__(*a, **kw)
416 417 self._reloader = ModuleReloader()
417 418 self._reloader.check_all = False
418 419
419 420 @line_magic
420 421 def autoreload(self, parameter_s=''):
421 422 r"""%autoreload => Reload modules automatically
422 423
423 424 %autoreload
424 425 Reload all modules (except those excluded by %aimport) automatically
425 426 now.
426 427
427 428 %autoreload 0
428 429 Disable automatic reloading.
429 430
430 431 %autoreload 1
431 432 Reload all modules imported with %aimport every time before executing
432 433 the Python code typed.
433 434
434 435 %autoreload 2
435 436 Reload all modules (except those excluded by %aimport) every time
436 437 before executing the Python code typed.
437 438
438 439 Reloading Python modules in a reliable way is in general
439 440 difficult, and unexpected things may occur. %autoreload tries to
440 441 work around common pitfalls by replacing function code objects and
441 442 parts of classes previously in the module with new versions. This
442 443 makes the following things to work:
443 444
444 445 - Functions and classes imported via 'from xxx import foo' are upgraded
445 446 to new versions when 'xxx' is reloaded.
446 447
447 448 - Methods and properties of classes are upgraded on reload, so that
448 449 calling 'c.foo()' on an object 'c' created before the reload causes
449 450 the new code for 'foo' to be executed.
450 451
451 452 Some of the known remaining caveats are:
452 453
453 454 - Replacing code objects does not always succeed: changing a @property
454 455 in a class to an ordinary method or a method to a member variable
455 456 can cause problems (but in old objects only).
456 457
457 458 - Functions that are removed (eg. via monkey-patching) from a module
458 459 before it is reloaded are not upgraded.
459 460
460 461 - C extension modules cannot be reloaded, and so cannot be
461 462 autoreloaded.
462 463
463 464 """
464 465 if parameter_s == '':
465 466 self._reloader.check(True)
466 467 elif parameter_s == '0':
467 468 self._reloader.enabled = False
468 469 elif parameter_s == '1':
469 470 self._reloader.check_all = False
470 471 self._reloader.enabled = True
471 472 elif parameter_s == '2':
472 473 self._reloader.check_all = True
473 474 self._reloader.enabled = True
474 475
475 476 @line_magic
476 477 def aimport(self, parameter_s='', stream=None):
477 478 """%aimport => Import modules for automatic reloading.
478 479
479 480 %aimport
480 481 List modules to automatically import and not to import.
481 482
482 483 %aimport foo
483 484 Import module 'foo' and mark it to be autoreloaded for %autoreload 1
484 485
485 486 %aimport -foo
486 487 Mark module 'foo' to not be autoreloaded for %autoreload 1
487 488 """
488 489 modname = parameter_s
489 490 if not modname:
490 491 to_reload = self._reloader.modules.keys()
491 492 to_reload.sort()
492 493 to_skip = self._reloader.skip_modules.keys()
493 494 to_skip.sort()
494 495 if stream is None:
495 496 stream = sys.stdout
496 497 if self._reloader.check_all:
497 498 stream.write("Modules to reload:\nall-except-skipped\n")
498 499 else:
499 500 stream.write("Modules to reload:\n%s\n" % ' '.join(to_reload))
500 501 stream.write("\nModules to skip:\n%s\n" % ' '.join(to_skip))
501 502 elif modname.startswith('-'):
502 503 modname = modname[1:]
503 504 self._reloader.mark_module_skipped(modname)
504 505 else:
505 506 top_module, top_name = self._reloader.aimport_module(modname)
506 507
507 508 # Inject module to user namespace
508 509 self.shell.push({top_name: top_module})
509 510
510 511 def pre_run_code_hook(self, ip):
511 512 if not self._reloader.enabled:
512 513 raise TryNext
513 514 try:
514 515 self._reloader.check()
515 516 except:
516 517 pass
517 518
518 519
519 520 class AutoreloadPlugin(Plugin):
520 521 def __init__(self, shell=None, config=None):
521 522 super(AutoreloadPlugin, self).__init__(shell=shell, config=config)
522 523 self.auto_magics = AutoreloadMagics(shell)
523 524 shell.register_magics(self.auto_magics)
524 525 shell.set_hook('pre_run_code_hook', self.auto_magics.pre_run_code_hook)
525 526
526 527
527 528 _loaded = False
528 529
529 530
530 531 def load_ipython_extension(ip):
531 532 """Load the extension in IPython."""
532 533 global _loaded
533 534 if not _loaded:
534 535 plugin = AutoreloadPlugin(shell=ip, config=ip.config)
535 536 ip.plugin_manager.register_plugin('autoreload', plugin)
536 537 _loaded = True
General Comments 0
You need to be logged in to leave comments. Login now