##// END OF EJS Templates
Switch module separator to comas to be consistent.
Matthias Bussonnier -
Show More
@@ -1,544 +1,544 b''
1 1 """IPython extension to reload modules before executing user code.
2 2
3 3 ``autoreload`` reloads modules automatically before entering the execution of
4 4 code typed at the IPython prompt.
5 5
6 6 This makes for example the following workflow possible:
7 7
8 8 .. sourcecode:: ipython
9 9
10 10 In [1]: %load_ext autoreload
11 11
12 12 In [2]: %autoreload 2
13 13
14 14 In [3]: from foo import some_function
15 15
16 16 In [4]: some_function()
17 17 Out[4]: 42
18 18
19 19 In [5]: # open foo.py in an editor and change some_function to return 43
20 20
21 21 In [6]: some_function()
22 22 Out[6]: 43
23 23
24 24 The module was reloaded without reloading it explicitly, and the object
25 25 imported with ``from foo import ...`` was also updated.
26 26
27 27 Usage
28 28 =====
29 29
30 30 The following magic commands are provided:
31 31
32 32 ``%autoreload``
33 33
34 34 Reload all modules (except those excluded by ``%aimport``)
35 35 automatically now.
36 36
37 37 ``%autoreload 0``
38 38
39 39 Disable automatic reloading.
40 40
41 41 ``%autoreload 1``
42 42
43 43 Reload all modules imported with ``%aimport`` every time before
44 44 executing the Python code typed.
45 45
46 46 ``%autoreload 2``
47 47
48 48 Reload all modules (except those excluded by ``%aimport``) every
49 49 time before executing the Python code typed.
50 50
51 51 ``%aimport``
52 52
53 53 List modules which are to be automatically imported or not to be imported.
54 54
55 55 ``%aimport foo``
56 56
57 57 Import module 'foo' and mark it to be autoreloaded for ``%autoreload 1``
58 58
59 ``%aimport foo bar``
59 ``%aimport foo, bar``
60 60
61 61 Import modules 'foo', 'bar' and mark them to be autoreloaded for ``%autoreload 1``
62 62
63 63 ``%aimport -foo``
64 64
65 65 Mark module 'foo' to not be autoreloaded.
66 66
67 67 Caveats
68 68 =======
69 69
70 70 Reloading Python modules in a reliable way is in general difficult,
71 71 and unexpected things may occur. ``%autoreload`` tries to work around
72 72 common pitfalls by replacing function code objects and parts of
73 73 classes previously in the module with new versions. This makes the
74 74 following things to work:
75 75
76 76 - Functions and classes imported via 'from xxx import foo' are upgraded
77 77 to new versions when 'xxx' is reloaded.
78 78
79 79 - Methods and properties of classes are upgraded on reload, so that
80 80 calling 'c.foo()' on an object 'c' created before the reload causes
81 81 the new code for 'foo' to be executed.
82 82
83 83 Some of the known remaining caveats are:
84 84
85 85 - Replacing code objects does not always succeed: changing a @property
86 86 in a class to an ordinary method or a method to a member variable
87 87 can cause problems (but in old objects only).
88 88
89 89 - Functions that are removed (eg. via monkey-patching) from a module
90 90 before it is reloaded are not upgraded.
91 91
92 92 - C extension modules cannot be reloaded, and so cannot be autoreloaded.
93 93 """
94 94
95 95 skip_doctest = True
96 96
97 97 #-----------------------------------------------------------------------------
98 98 # Copyright (C) 2000 Thomas Heller
99 99 # Copyright (C) 2008 Pauli Virtanen <pav@iki.fi>
100 100 # Copyright (C) 2012 The IPython Development Team
101 101 #
102 102 # Distributed under the terms of the BSD License. The full license is in
103 103 # the file COPYING, distributed as part of this software.
104 104 #-----------------------------------------------------------------------------
105 105 #
106 106 # This IPython module is written by Pauli Virtanen, based on the autoreload
107 107 # code by Thomas Heller.
108 108
109 109 #-----------------------------------------------------------------------------
110 110 # Imports
111 111 #-----------------------------------------------------------------------------
112 112
113 113 import os
114 114 import sys
115 115 import traceback
116 116 import types
117 117 import weakref
118 118 from importlib import import_module
119 119
120 120 try:
121 121 # Reload is not defined by default in Python3.
122 122 reload
123 123 except NameError:
124 124 from imp import reload
125 125
126 126 from IPython.utils import openpy
127 127 from IPython.utils.py3compat import PY3
128 128
129 129 #------------------------------------------------------------------------------
130 130 # Autoreload functionality
131 131 #------------------------------------------------------------------------------
132 132
133 133 class ModuleReloader(object):
134 134 enabled = False
135 135 """Whether this reloader is enabled"""
136 136
137 137 check_all = True
138 138 """Autoreload all modules, not just those listed in 'modules'"""
139 139
140 140 def __init__(self):
141 141 # Modules that failed to reload: {module: mtime-on-failed-reload, ...}
142 142 self.failed = {}
143 143 # Modules specially marked as autoreloadable.
144 144 self.modules = {}
145 145 # Modules specially marked as not autoreloadable.
146 146 self.skip_modules = {}
147 147 # (module-name, name) -> weakref, for replacing old code objects
148 148 self.old_objects = {}
149 149 # Module modification timestamps
150 150 self.modules_mtimes = {}
151 151
152 152 # Cache module modification times
153 153 self.check(check_all=True, do_reload=False)
154 154
155 155 def mark_module_skipped(self, module_name):
156 156 """Skip reloading the named module in the future"""
157 157 try:
158 158 del self.modules[module_name]
159 159 except KeyError:
160 160 pass
161 161 self.skip_modules[module_name] = True
162 162
163 163 def mark_module_reloadable(self, module_name):
164 164 """Reload the named module in the future (if it is imported)"""
165 165 try:
166 166 del self.skip_modules[module_name]
167 167 except KeyError:
168 168 pass
169 169 self.modules[module_name] = True
170 170
171 171 def aimport_module(self, module_name):
172 172 """Import a module, and mark it reloadable
173 173
174 174 Returns
175 175 -------
176 176 top_module : module
177 177 The imported module if it is top-level, or the top-level
178 178 top_name : module
179 179 Name of top_module
180 180
181 181 """
182 182 self.mark_module_reloadable(module_name)
183 183
184 184 import_module(module_name)
185 185 top_name = module_name.split('.')[0]
186 186 top_module = sys.modules[top_name]
187 187 return top_module, top_name
188 188
189 189 def filename_and_mtime(self, module):
190 190 if not hasattr(module, '__file__') or module.__file__ is None:
191 191 return None, None
192 192
193 193 if getattr(module, '__name__', None) == '__main__':
194 194 # we cannot reload(__main__)
195 195 return None, None
196 196
197 197 filename = module.__file__
198 198 path, ext = os.path.splitext(filename)
199 199
200 200 if ext.lower() == '.py':
201 201 py_filename = filename
202 202 else:
203 203 try:
204 204 py_filename = openpy.source_from_cache(filename)
205 205 except ValueError:
206 206 return None, None
207 207
208 208 try:
209 209 pymtime = os.stat(py_filename).st_mtime
210 210 except OSError:
211 211 return None, None
212 212
213 213 return py_filename, pymtime
214 214
215 215 def check(self, check_all=False, do_reload=True):
216 216 """Check whether some modules need to be reloaded."""
217 217
218 218 if not self.enabled and not check_all:
219 219 return
220 220
221 221 if check_all or self.check_all:
222 222 modules = list(sys.modules.keys())
223 223 else:
224 224 modules = list(self.modules.keys())
225 225
226 226 for modname in modules:
227 227 m = sys.modules.get(modname, None)
228 228
229 229 if modname in self.skip_modules:
230 230 continue
231 231
232 232 py_filename, pymtime = self.filename_and_mtime(m)
233 233 if py_filename is None:
234 234 continue
235 235
236 236 try:
237 237 if pymtime <= self.modules_mtimes[modname]:
238 238 continue
239 239 except KeyError:
240 240 self.modules_mtimes[modname] = pymtime
241 241 continue
242 242 else:
243 243 if self.failed.get(py_filename, None) == pymtime:
244 244 continue
245 245
246 246 self.modules_mtimes[modname] = pymtime
247 247
248 248 # If we've reached this point, we should try to reload the module
249 249 if do_reload:
250 250 try:
251 251 superreload(m, reload, self.old_objects)
252 252 if py_filename in self.failed:
253 253 del self.failed[py_filename]
254 254 except:
255 255 print("[autoreload of %s failed: %s]" % (
256 256 modname, traceback.format_exc(1)), file=sys.stderr)
257 257 self.failed[py_filename] = pymtime
258 258
259 259 #------------------------------------------------------------------------------
260 260 # superreload
261 261 #------------------------------------------------------------------------------
262 262
263 263 if PY3:
264 264 func_attrs = ['__code__', '__defaults__', '__doc__',
265 265 '__closure__', '__globals__', '__dict__']
266 266 else:
267 267 func_attrs = ['func_code', 'func_defaults', 'func_doc',
268 268 'func_closure', 'func_globals', 'func_dict']
269 269
270 270
271 271 def update_function(old, new):
272 272 """Upgrade the code object of a function"""
273 273 for name in func_attrs:
274 274 try:
275 275 setattr(old, name, getattr(new, name))
276 276 except (AttributeError, TypeError):
277 277 pass
278 278
279 279
280 280 def update_class(old, new):
281 281 """Replace stuff in the __dict__ of a class, and upgrade
282 282 method code objects"""
283 283 for key in list(old.__dict__.keys()):
284 284 old_obj = getattr(old, key)
285 285
286 286 try:
287 287 new_obj = getattr(new, key)
288 288 except AttributeError:
289 289 # obsolete attribute: remove it
290 290 try:
291 291 delattr(old, key)
292 292 except (AttributeError, TypeError):
293 293 pass
294 294 continue
295 295
296 296 if update_generic(old_obj, new_obj): continue
297 297
298 298 try:
299 299 setattr(old, key, getattr(new, key))
300 300 except (AttributeError, TypeError):
301 301 pass # skip non-writable attributes
302 302
303 303
304 304 def update_property(old, new):
305 305 """Replace get/set/del functions of a property"""
306 306 update_generic(old.fdel, new.fdel)
307 307 update_generic(old.fget, new.fget)
308 308 update_generic(old.fset, new.fset)
309 309
310 310
311 311 def isinstance2(a, b, typ):
312 312 return isinstance(a, typ) and isinstance(b, typ)
313 313
314 314
315 315 UPDATE_RULES = [
316 316 (lambda a, b: isinstance2(a, b, type),
317 317 update_class),
318 318 (lambda a, b: isinstance2(a, b, types.FunctionType),
319 319 update_function),
320 320 (lambda a, b: isinstance2(a, b, property),
321 321 update_property),
322 322 ]
323 323
324 324
325 325 if PY3:
326 326 UPDATE_RULES.extend([(lambda a, b: isinstance2(a, b, types.MethodType),
327 327 lambda a, b: update_function(a.__func__, b.__func__)),
328 328 ])
329 329 else:
330 330 UPDATE_RULES.extend([(lambda a, b: isinstance2(a, b, types.ClassType),
331 331 update_class),
332 332 (lambda a, b: isinstance2(a, b, types.MethodType),
333 333 lambda a, b: update_function(a.__func__, b.__func__)),
334 334 ])
335 335
336 336
337 337 def update_generic(a, b):
338 338 for type_check, update in UPDATE_RULES:
339 339 if type_check(a, b):
340 340 update(a, b)
341 341 return True
342 342 return False
343 343
344 344
345 345 class StrongRef(object):
346 346 def __init__(self, obj):
347 347 self.obj = obj
348 348 def __call__(self):
349 349 return self.obj
350 350
351 351
352 352 def superreload(module, reload=reload, old_objects={}):
353 353 """Enhanced version of the builtin reload function.
354 354
355 355 superreload remembers objects previously in the module, and
356 356
357 357 - upgrades the class dictionary of every old class in the module
358 358 - upgrades the code object of every old function and method
359 359 - clears the module's namespace before reloading
360 360
361 361 """
362 362
363 363 # collect old objects in the module
364 364 for name, obj in list(module.__dict__.items()):
365 365 if not hasattr(obj, '__module__') or obj.__module__ != module.__name__:
366 366 continue
367 367 key = (module.__name__, name)
368 368 try:
369 369 old_objects.setdefault(key, []).append(weakref.ref(obj))
370 370 except TypeError:
371 371 # weakref doesn't work for all types;
372 372 # create strong references for 'important' cases
373 373 if not PY3 and isinstance(obj, types.ClassType):
374 374 old_objects.setdefault(key, []).append(StrongRef(obj))
375 375
376 376 # reload module
377 377 try:
378 378 # clear namespace first from old cruft
379 379 old_dict = module.__dict__.copy()
380 380 old_name = module.__name__
381 381 module.__dict__.clear()
382 382 module.__dict__['__name__'] = old_name
383 383 module.__dict__['__loader__'] = old_dict['__loader__']
384 384 except (TypeError, AttributeError, KeyError):
385 385 pass
386 386
387 387 try:
388 388 module = reload(module)
389 389 except:
390 390 # restore module dictionary on failed reload
391 391 module.__dict__.update(old_dict)
392 392 raise
393 393
394 394 # iterate over all objects and update functions & classes
395 395 for name, new_obj in list(module.__dict__.items()):
396 396 key = (module.__name__, name)
397 397 if key not in old_objects: continue
398 398
399 399 new_refs = []
400 400 for old_ref in old_objects[key]:
401 401 old_obj = old_ref()
402 402 if old_obj is None: continue
403 403 new_refs.append(old_ref)
404 404 update_generic(old_obj, new_obj)
405 405
406 406 if new_refs:
407 407 old_objects[key] = new_refs
408 408 else:
409 409 del old_objects[key]
410 410
411 411 return module
412 412
413 413 #------------------------------------------------------------------------------
414 414 # IPython connectivity
415 415 #------------------------------------------------------------------------------
416 416
417 417 from IPython.core.magic import Magics, magics_class, line_magic
418 418
419 419 @magics_class
420 420 class AutoreloadMagics(Magics):
421 421 def __init__(self, *a, **kw):
422 422 super(AutoreloadMagics, self).__init__(*a, **kw)
423 423 self._reloader = ModuleReloader()
424 424 self._reloader.check_all = False
425 425 self.loaded_modules = set(sys.modules)
426 426
427 427 @line_magic
428 428 def autoreload(self, parameter_s=''):
429 429 r"""%autoreload => Reload modules automatically
430 430
431 431 %autoreload
432 432 Reload all modules (except those excluded by %aimport) automatically
433 433 now.
434 434
435 435 %autoreload 0
436 436 Disable automatic reloading.
437 437
438 438 %autoreload 1
439 439 Reload all modules imported with %aimport every time before executing
440 440 the Python code typed.
441 441
442 442 %autoreload 2
443 443 Reload all modules (except those excluded by %aimport) every time
444 444 before executing the Python code typed.
445 445
446 446 Reloading Python modules in a reliable way is in general
447 447 difficult, and unexpected things may occur. %autoreload tries to
448 448 work around common pitfalls by replacing function code objects and
449 449 parts of classes previously in the module with new versions. This
450 450 makes the following things to work:
451 451
452 452 - Functions and classes imported via 'from xxx import foo' are upgraded
453 453 to new versions when 'xxx' is reloaded.
454 454
455 455 - Methods and properties of classes are upgraded on reload, so that
456 456 calling 'c.foo()' on an object 'c' created before the reload causes
457 457 the new code for 'foo' to be executed.
458 458
459 459 Some of the known remaining caveats are:
460 460
461 461 - Replacing code objects does not always succeed: changing a @property
462 462 in a class to an ordinary method or a method to a member variable
463 463 can cause problems (but in old objects only).
464 464
465 465 - Functions that are removed (eg. via monkey-patching) from a module
466 466 before it is reloaded are not upgraded.
467 467
468 468 - C extension modules cannot be reloaded, and so cannot be
469 469 autoreloaded.
470 470
471 471 """
472 472 if parameter_s == '':
473 473 self._reloader.check(True)
474 474 elif parameter_s == '0':
475 475 self._reloader.enabled = False
476 476 elif parameter_s == '1':
477 477 self._reloader.check_all = False
478 478 self._reloader.enabled = True
479 479 elif parameter_s == '2':
480 480 self._reloader.check_all = True
481 481 self._reloader.enabled = True
482 482
483 483 @line_magic
484 484 def aimport(self, parameter_s='', stream=None):
485 485 """%aimport => Import modules for automatic reloading.
486 486
487 487 %aimport
488 488 List modules to automatically import and not to import.
489 489
490 490 %aimport foo
491 491 Import module 'foo' and mark it to be autoreloaded for %autoreload 1
492 492
493 %aimport foo bar
493 %aimport foo, bar
494 494 Import modules 'foo', 'bar' and mark them to be autoreloaded for %autoreload 1
495 495
496 496 %aimport -foo
497 497 Mark module 'foo' to not be autoreloaded for %autoreload 1
498 498 """
499 499 modname = parameter_s
500 500 if not modname:
501 501 to_reload = sorted(self._reloader.modules.keys())
502 502 to_skip = sorted(self._reloader.skip_modules.keys())
503 503 if stream is None:
504 504 stream = sys.stdout
505 505 if self._reloader.check_all:
506 506 stream.write("Modules to reload:\nall-except-skipped\n")
507 507 else:
508 508 stream.write("Modules to reload:\n%s\n" % ' '.join(to_reload))
509 509 stream.write("\nModules to skip:\n%s\n" % ' '.join(to_skip))
510 510 elif modname.startswith('-'):
511 511 modname = modname[1:]
512 512 self._reloader.mark_module_skipped(modname)
513 513 else:
514 for _module in modname.split():
514 for _module in ([_.strip() for _ in modname.split(',')]):
515 515 top_module, top_name = self._reloader.aimport_module(_module)
516 516
517 517 # Inject module to user namespace
518 518 self.shell.push({top_name: top_module})
519 519
520 520 def pre_run_cell(self):
521 521 if self._reloader.enabled:
522 522 try:
523 523 self._reloader.check()
524 524 except:
525 525 pass
526 526
527 527 def post_execute_hook(self):
528 528 """Cache the modification times of any modules imported in this execution
529 529 """
530 530 newly_loaded_modules = set(sys.modules) - self.loaded_modules
531 531 for modname in newly_loaded_modules:
532 532 _, pymtime = self._reloader.filename_and_mtime(sys.modules[modname])
533 533 if pymtime is not None:
534 534 self._reloader.modules_mtimes[modname] = pymtime
535 535
536 536 self.loaded_modules.update(newly_loaded_modules)
537 537
538 538
539 539 def load_ipython_extension(ip):
540 540 """Load the extension in IPython."""
541 541 auto_reload = AutoreloadMagics(ip)
542 542 ip.register_magics(auto_reload)
543 543 ip.events.register('pre_run_cell', auto_reload.pre_run_cell)
544 544 ip.events.register('post_execute', auto_reload.post_execute_hook)
General Comments 0
You need to be logged in to leave comments. Login now