##// END OF EJS Templates
Fix autoreload tests
Thomas Kluyver -
Show More
@@ -1,509 +1,508 b''
1 1 """IPython extension to reload modules before executing user code.
2 2
3 3 ``autoreload`` reloads modules automatically before entering the execution of
4 4 code typed at the IPython prompt.
5 5
6 6 This makes for example the following workflow possible:
7 7
8 8 .. sourcecode:: ipython
9 9
10 10 In [1]: %load_ext autoreload
11 11
12 12 In [2]: %autoreload 2
13 13
14 14 In [3]: from foo import some_function
15 15
16 16 In [4]: some_function()
17 17 Out[4]: 42
18 18
19 19 In [5]: # open foo.py in an editor and change some_function to return 43
20 20
21 21 In [6]: some_function()
22 22 Out[6]: 43
23 23
24 24 The module was reloaded without reloading it explicitly, and the object
25 25 imported with ``from foo import ...`` was also updated.
26 26
27 27 Usage
28 28 =====
29 29
30 30 The following magic commands are provided:
31 31
32 32 ``%autoreload``
33 33
34 34 Reload all modules (except those excluded by ``%aimport``)
35 35 automatically now.
36 36
37 37 ``%autoreload 0``
38 38
39 39 Disable automatic reloading.
40 40
41 41 ``%autoreload 1``
42 42
43 43 Reload all modules imported with ``%aimport`` every time before
44 44 executing the Python code typed.
45 45
46 46 ``%autoreload 2``
47 47
48 48 Reload all modules (except those excluded by ``%aimport``) every
49 49 time before executing the Python code typed.
50 50
51 51 ``%aimport``
52 52
53 53 List modules which are to be automatically imported or not to be imported.
54 54
55 55 ``%aimport foo``
56 56
57 57 Import module 'foo' and mark it to be autoreloaded for ``%autoreload 1``
58 58
59 59 ``%aimport -foo``
60 60
61 61 Mark module 'foo' to not be autoreloaded.
62 62
63 63 Caveats
64 64 =======
65 65
66 66 Reloading Python modules in a reliable way is in general difficult,
67 67 and unexpected things may occur. ``%autoreload`` tries to work around
68 68 common pitfalls by replacing function code objects and parts of
69 69 classes previously in the module with new versions. This makes the
70 70 following things to work:
71 71
72 72 - Functions and classes imported via 'from xxx import foo' are upgraded
73 73 to new versions when 'xxx' is reloaded.
74 74
75 75 - Methods and properties of classes are upgraded on reload, so that
76 76 calling 'c.foo()' on an object 'c' created before the reload causes
77 77 the new code for 'foo' to be executed.
78 78
79 79 Some of the known remaining caveats are:
80 80
81 81 - Replacing code objects does not always succeed: changing a @property
82 82 in a class to an ordinary method or a method to a member variable
83 83 can cause problems (but in old objects only).
84 84
85 85 - Functions that are removed (eg. via monkey-patching) from a module
86 86 before it is reloaded are not upgraded.
87 87
88 88 - C extension modules cannot be reloaded, and so cannot be autoreloaded.
89 89 """
90 90 from __future__ import print_function
91 91
92 92 skip_doctest = True
93 93
94 94 #-----------------------------------------------------------------------------
95 95 # Copyright (C) 2000 Thomas Heller
96 96 # Copyright (C) 2008 Pauli Virtanen <pav@iki.fi>
97 97 # Copyright (C) 2012 The IPython Development Team
98 98 #
99 99 # Distributed under the terms of the BSD License. The full license is in
100 100 # the file COPYING, distributed as part of this software.
101 101 #-----------------------------------------------------------------------------
102 102 #
103 103 # This IPython module is written by Pauli Virtanen, based on the autoreload
104 104 # code by Thomas Heller.
105 105
106 106 #-----------------------------------------------------------------------------
107 107 # Imports
108 108 #-----------------------------------------------------------------------------
109 109
110 110 import os
111 111 import sys
112 112 import traceback
113 113 import types
114 114 import weakref
115 115
116 116 try:
117 117 # Reload is not defined by default in Python3.
118 118 reload
119 119 except NameError:
120 120 from imp import reload
121 121
122 122 from IPython.utils import openpy
123 123 from IPython.utils.py3compat import PY3
124 124
125 125 #------------------------------------------------------------------------------
126 126 # Autoreload functionality
127 127 #------------------------------------------------------------------------------
128 128
129 129 class ModuleReloader(object):
130 130 enabled = False
131 131 """Whether this reloader is enabled"""
132 132
133 133 failed = {}
134 134 """Modules that failed to reload: {module: mtime-on-failed-reload, ...}"""
135 135
136 136 modules = {}
137 137 """Modules specially marked as autoreloadable."""
138 138
139 139 skip_modules = {}
140 140 """Modules specially marked as not autoreloadable."""
141 141
142 142 check_all = True
143 143 """Autoreload all modules, not just those listed in 'modules'"""
144 144
145 145 old_objects = {}
146 146 """(module-name, name) -> weakref, for replacing old code objects"""
147 147
148 148 def mark_module_skipped(self, module_name):
149 149 """Skip reloading the named module in the future"""
150 150 try:
151 151 del self.modules[module_name]
152 152 except KeyError:
153 153 pass
154 154 self.skip_modules[module_name] = True
155 155
156 156 def mark_module_reloadable(self, module_name):
157 157 """Reload the named module in the future (if it is imported)"""
158 158 try:
159 159 del self.skip_modules[module_name]
160 160 except KeyError:
161 161 pass
162 162 self.modules[module_name] = True
163 163
164 164 def aimport_module(self, module_name):
165 165 """Import a module, and mark it reloadable
166 166
167 167 Returns
168 168 -------
169 169 top_module : module
170 170 The imported module if it is top-level, or the top-level
171 171 top_name : module
172 172 Name of top_module
173 173
174 174 """
175 175 self.mark_module_reloadable(module_name)
176 176
177 177 __import__(module_name)
178 178 top_name = module_name.split('.')[0]
179 179 top_module = sys.modules[top_name]
180 180 return top_module, top_name
181 181
182 182 def check(self, check_all=False):
183 183 """Check whether some modules need to be reloaded."""
184 184
185 185 if not self.enabled and not check_all:
186 186 return
187 187
188 188 if check_all or self.check_all:
189 modules = sys.modules.keys()
189 modules = list(sys.modules.keys())
190 190 else:
191 modules = self.modules.keys()
191 modules = list(self.modules.keys())
192 192
193 193 for modname in modules:
194 194 m = sys.modules.get(modname, None)
195 195
196 196 if modname in self.skip_modules:
197 197 continue
198 198
199 199 if not hasattr(m, '__file__'):
200 200 continue
201 201
202 202 if m.__name__ == '__main__':
203 203 # we cannot reload(__main__)
204 204 continue
205 205
206 206 filename = m.__file__
207 207 path, ext = os.path.splitext(filename)
208 208
209 209 if ext.lower() == '.py':
210 210 pyc_filename = openpy.cache_from_source(filename)
211 211 py_filename = filename
212 212 else:
213 213 pyc_filename = filename
214 214 try:
215 215 py_filename = openpy.source_from_cache(filename)
216 216 except ValueError:
217 217 continue
218 218
219 219 try:
220 220 pymtime = os.stat(py_filename).st_mtime
221 221 if pymtime <= os.stat(pyc_filename).st_mtime:
222 222 continue
223 223 if self.failed.get(py_filename, None) == pymtime:
224 224 continue
225 225 except OSError:
226 226 continue
227 227
228 228 try:
229 229 superreload(m, reload, self.old_objects)
230 230 if py_filename in self.failed:
231 231 del self.failed[py_filename]
232 232 except:
233 233 print("[autoreload of %s failed: %s]" % (
234 234 modname, traceback.format_exc(1)), file=sys.stderr)
235 235 self.failed[py_filename] = pymtime
236 236
237 237 #------------------------------------------------------------------------------
238 238 # superreload
239 239 #------------------------------------------------------------------------------
240 240
241 241 if PY3:
242 242 func_attrs = ['__code__', '__defaults__', '__doc__',
243 243 '__closure__', '__globals__', '__dict__']
244 244 else:
245 245 func_attrs = ['func_code', 'func_defaults', 'func_doc',
246 246 'func_closure', 'func_globals', 'func_dict']
247 247
248 248
249 249 def update_function(old, new):
250 250 """Upgrade the code object of a function"""
251 251 for name in func_attrs:
252 252 try:
253 253 setattr(old, name, getattr(new, name))
254 254 except (AttributeError, TypeError):
255 255 pass
256 256
257 257
258 258 def update_class(old, new):
259 259 """Replace stuff in the __dict__ of a class, and upgrade
260 260 method code objects"""
261 for key in old.__dict__.keys():
261 for key in list(old.__dict__.keys()):
262 262 old_obj = getattr(old, key)
263 263
264 264 try:
265 265 new_obj = getattr(new, key)
266 266 except AttributeError:
267 267 # obsolete attribute: remove it
268 268 try:
269 269 delattr(old, key)
270 270 except (AttributeError, TypeError):
271 271 pass
272 272 continue
273 273
274 274 if update_generic(old_obj, new_obj): continue
275 275
276 276 try:
277 277 setattr(old, key, getattr(new, key))
278 278 except (AttributeError, TypeError):
279 279 pass # skip non-writable attributes
280 280
281 281
282 282 def update_property(old, new):
283 283 """Replace get/set/del functions of a property"""
284 284 update_generic(old.fdel, new.fdel)
285 285 update_generic(old.fget, new.fget)
286 286 update_generic(old.fset, new.fset)
287 287
288 288
289 289 def isinstance2(a, b, typ):
290 290 return isinstance(a, typ) and isinstance(b, typ)
291 291
292 292
293 293 UPDATE_RULES = [
294 294 (lambda a, b: isinstance2(a, b, type),
295 295 update_class),
296 296 (lambda a, b: isinstance2(a, b, types.FunctionType),
297 297 update_function),
298 298 (lambda a, b: isinstance2(a, b, property),
299 299 update_property),
300 300 ]
301 301
302 302
303 303 if PY3:
304 304 UPDATE_RULES.extend([(lambda a, b: isinstance2(a, b, types.MethodType),
305 305 lambda a, b: update_function(a.__func__, b.__func__)),
306 306 ])
307 307 else:
308 308 UPDATE_RULES.extend([(lambda a, b: isinstance2(a, b, types.ClassType),
309 309 update_class),
310 310 (lambda a, b: isinstance2(a, b, types.MethodType),
311 311 lambda a, b: update_function(a.__func__, b.__func__)),
312 312 ])
313 313
314 314
315 315 def update_generic(a, b):
316 316 for type_check, update in UPDATE_RULES:
317 317 if type_check(a, b):
318 318 update(a, b)
319 319 return True
320 320 return False
321 321
322 322
323 323 class StrongRef(object):
324 324 def __init__(self, obj):
325 325 self.obj = obj
326 326 def __call__(self):
327 327 return self.obj
328 328
329 329
330 330 def superreload(module, reload=reload, old_objects={}):
331 331 """Enhanced version of the builtin reload function.
332 332
333 333 superreload remembers objects previously in the module, and
334 334
335 335 - upgrades the class dictionary of every old class in the module
336 336 - upgrades the code object of every old function and method
337 337 - clears the module's namespace before reloading
338 338
339 339 """
340 340
341 341 # collect old objects in the module
342 for name, obj in module.__dict__.items():
342 for name, obj in list(module.__dict__.items()):
343 343 if not hasattr(obj, '__module__') or obj.__module__ != module.__name__:
344 344 continue
345 345 key = (module.__name__, name)
346 346 try:
347 347 old_objects.setdefault(key, []).append(weakref.ref(obj))
348 348 except TypeError:
349 349 # weakref doesn't work for all types;
350 350 # create strong references for 'important' cases
351 351 if not PY3 and isinstance(obj, types.ClassType):
352 352 old_objects.setdefault(key, []).append(StrongRef(obj))
353 353
354 354 # reload module
355 355 try:
356 356 # clear namespace first from old cruft
357 357 old_dict = module.__dict__.copy()
358 358 old_name = module.__name__
359 359 module.__dict__.clear()
360 360 module.__dict__['__name__'] = old_name
361 361 module.__dict__['__loader__'] = old_dict['__loader__']
362 362 except (TypeError, AttributeError, KeyError):
363 363 pass
364 364
365 365 try:
366 366 module = reload(module)
367 367 except:
368 368 # restore module dictionary on failed reload
369 369 module.__dict__.update(old_dict)
370 370 raise
371 371
372 372 # iterate over all objects and update functions & classes
373 for name, new_obj in module.__dict__.items():
373 for name, new_obj in list(module.__dict__.items()):
374 374 key = (module.__name__, name)
375 375 if key not in old_objects: continue
376 376
377 377 new_refs = []
378 378 for old_ref in old_objects[key]:
379 379 old_obj = old_ref()
380 380 if old_obj is None: continue
381 381 new_refs.append(old_ref)
382 382 update_generic(old_obj, new_obj)
383 383
384 384 if new_refs:
385 385 old_objects[key] = new_refs
386 386 else:
387 387 del old_objects[key]
388 388
389 389 return module
390 390
391 391 #------------------------------------------------------------------------------
392 392 # IPython connectivity
393 393 #------------------------------------------------------------------------------
394 394
395 395 from IPython.core.hooks import TryNext
396 396 from IPython.core.magic import Magics, magics_class, line_magic
397 397
398 398 @magics_class
399 399 class AutoreloadMagics(Magics):
400 400 def __init__(self, *a, **kw):
401 401 super(AutoreloadMagics, self).__init__(*a, **kw)
402 402 self._reloader = ModuleReloader()
403 403 self._reloader.check_all = False
404 404
405 405 @line_magic
406 406 def autoreload(self, parameter_s=''):
407 407 r"""%autoreload => Reload modules automatically
408 408
409 409 %autoreload
410 410 Reload all modules (except those excluded by %aimport) automatically
411 411 now.
412 412
413 413 %autoreload 0
414 414 Disable automatic reloading.
415 415
416 416 %autoreload 1
417 417 Reload all modules imported with %aimport every time before executing
418 418 the Python code typed.
419 419
420 420 %autoreload 2
421 421 Reload all modules (except those excluded by %aimport) every time
422 422 before executing the Python code typed.
423 423
424 424 Reloading Python modules in a reliable way is in general
425 425 difficult, and unexpected things may occur. %autoreload tries to
426 426 work around common pitfalls by replacing function code objects and
427 427 parts of classes previously in the module with new versions. This
428 428 makes the following things to work:
429 429
430 430 - Functions and classes imported via 'from xxx import foo' are upgraded
431 431 to new versions when 'xxx' is reloaded.
432 432
433 433 - Methods and properties of classes are upgraded on reload, so that
434 434 calling 'c.foo()' on an object 'c' created before the reload causes
435 435 the new code for 'foo' to be executed.
436 436
437 437 Some of the known remaining caveats are:
438 438
439 439 - Replacing code objects does not always succeed: changing a @property
440 440 in a class to an ordinary method or a method to a member variable
441 441 can cause problems (but in old objects only).
442 442
443 443 - Functions that are removed (eg. via monkey-patching) from a module
444 444 before it is reloaded are not upgraded.
445 445
446 446 - C extension modules cannot be reloaded, and so cannot be
447 447 autoreloaded.
448 448
449 449 """
450 450 if parameter_s == '':
451 451 self._reloader.check(True)
452 452 elif parameter_s == '0':
453 453 self._reloader.enabled = False
454 454 elif parameter_s == '1':
455 455 self._reloader.check_all = False
456 456 self._reloader.enabled = True
457 457 elif parameter_s == '2':
458 458 self._reloader.check_all = True
459 459 self._reloader.enabled = True
460 460
461 461 @line_magic
462 462 def aimport(self, parameter_s='', stream=None):
463 463 """%aimport => Import modules for automatic reloading.
464 464
465 465 %aimport
466 466 List modules to automatically import and not to import.
467 467
468 468 %aimport foo
469 469 Import module 'foo' and mark it to be autoreloaded for %autoreload 1
470 470
471 471 %aimport -foo
472 472 Mark module 'foo' to not be autoreloaded for %autoreload 1
473 473 """
474 474 modname = parameter_s
475 475 if not modname:
476 to_reload = self._reloader.modules.keys()
477 to_reload.sort()
478 to_skip = self._reloader.skip_modules.keys()
476 to_reload = sorted(self._reloader.modules.keys())
477 to_skip = sorted(self._reloader.skip_modules.keys())
479 478 to_skip.sort()
480 479 if stream is None:
481 480 stream = sys.stdout
482 481 if self._reloader.check_all:
483 482 stream.write("Modules to reload:\nall-except-skipped\n")
484 483 else:
485 484 stream.write("Modules to reload:\n%s\n" % ' '.join(to_reload))
486 485 stream.write("\nModules to skip:\n%s\n" % ' '.join(to_skip))
487 486 elif modname.startswith('-'):
488 487 modname = modname[1:]
489 488 self._reloader.mark_module_skipped(modname)
490 489 else:
491 490 top_module, top_name = self._reloader.aimport_module(modname)
492 491
493 492 # Inject module to user namespace
494 493 self.shell.push({top_name: top_module})
495 494
496 495 def pre_run_code_hook(self, ip):
497 496 if not self._reloader.enabled:
498 497 raise TryNext
499 498 try:
500 499 self._reloader.check()
501 500 except:
502 501 pass
503 502
504 503
505 504 def load_ipython_extension(ip):
506 505 """Load the extension in IPython."""
507 506 auto_reload = AutoreloadMagics(ip)
508 507 ip.register_magics(auto_reload)
509 508 ip.set_hook('pre_run_code_hook', auto_reload.pre_run_code_hook)
General Comments 0
You need to be logged in to leave comments. Login now