##// END OF EJS Templates
Use callbacks system for autoreload
Thomas Kluyver -
Show More
@@ -1,507 +1,505
1 """IPython extension to reload modules before executing user code.
1 """IPython extension to reload modules before executing user code.
2
2
3 ``autoreload`` reloads modules automatically before entering the execution of
3 ``autoreload`` reloads modules automatically before entering the execution of
4 code typed at the IPython prompt.
4 code typed at the IPython prompt.
5
5
6 This makes for example the following workflow possible:
6 This makes for example the following workflow possible:
7
7
8 .. sourcecode:: ipython
8 .. sourcecode:: ipython
9
9
10 In [1]: %load_ext autoreload
10 In [1]: %load_ext autoreload
11
11
12 In [2]: %autoreload 2
12 In [2]: %autoreload 2
13
13
14 In [3]: from foo import some_function
14 In [3]: from foo import some_function
15
15
16 In [4]: some_function()
16 In [4]: some_function()
17 Out[4]: 42
17 Out[4]: 42
18
18
19 In [5]: # open foo.py in an editor and change some_function to return 43
19 In [5]: # open foo.py in an editor and change some_function to return 43
20
20
21 In [6]: some_function()
21 In [6]: some_function()
22 Out[6]: 43
22 Out[6]: 43
23
23
24 The module was reloaded without reloading it explicitly, and the object
24 The module was reloaded without reloading it explicitly, and the object
25 imported with ``from foo import ...`` was also updated.
25 imported with ``from foo import ...`` was also updated.
26
26
27 Usage
27 Usage
28 =====
28 =====
29
29
30 The following magic commands are provided:
30 The following magic commands are provided:
31
31
32 ``%autoreload``
32 ``%autoreload``
33
33
34 Reload all modules (except those excluded by ``%aimport``)
34 Reload all modules (except those excluded by ``%aimport``)
35 automatically now.
35 automatically now.
36
36
37 ``%autoreload 0``
37 ``%autoreload 0``
38
38
39 Disable automatic reloading.
39 Disable automatic reloading.
40
40
41 ``%autoreload 1``
41 ``%autoreload 1``
42
42
43 Reload all modules imported with ``%aimport`` every time before
43 Reload all modules imported with ``%aimport`` every time before
44 executing the Python code typed.
44 executing the Python code typed.
45
45
46 ``%autoreload 2``
46 ``%autoreload 2``
47
47
48 Reload all modules (except those excluded by ``%aimport``) every
48 Reload all modules (except those excluded by ``%aimport``) every
49 time before executing the Python code typed.
49 time before executing the Python code typed.
50
50
51 ``%aimport``
51 ``%aimport``
52
52
53 List modules which are to be automatically imported or not to be imported.
53 List modules which are to be automatically imported or not to be imported.
54
54
55 ``%aimport foo``
55 ``%aimport foo``
56
56
57 Import module 'foo' and mark it to be autoreloaded for ``%autoreload 1``
57 Import module 'foo' and mark it to be autoreloaded for ``%autoreload 1``
58
58
59 ``%aimport -foo``
59 ``%aimport -foo``
60
60
61 Mark module 'foo' to not be autoreloaded.
61 Mark module 'foo' to not be autoreloaded.
62
62
63 Caveats
63 Caveats
64 =======
64 =======
65
65
66 Reloading Python modules in a reliable way is in general difficult,
66 Reloading Python modules in a reliable way is in general difficult,
67 and unexpected things may occur. ``%autoreload`` tries to work around
67 and unexpected things may occur. ``%autoreload`` tries to work around
68 common pitfalls by replacing function code objects and parts of
68 common pitfalls by replacing function code objects and parts of
69 classes previously in the module with new versions. This makes the
69 classes previously in the module with new versions. This makes the
70 following things to work:
70 following things to work:
71
71
72 - Functions and classes imported via 'from xxx import foo' are upgraded
72 - Functions and classes imported via 'from xxx import foo' are upgraded
73 to new versions when 'xxx' is reloaded.
73 to new versions when 'xxx' is reloaded.
74
74
75 - Methods and properties of classes are upgraded on reload, so that
75 - Methods and properties of classes are upgraded on reload, so that
76 calling 'c.foo()' on an object 'c' created before the reload causes
76 calling 'c.foo()' on an object 'c' created before the reload causes
77 the new code for 'foo' to be executed.
77 the new code for 'foo' to be executed.
78
78
79 Some of the known remaining caveats are:
79 Some of the known remaining caveats are:
80
80
81 - Replacing code objects does not always succeed: changing a @property
81 - Replacing code objects does not always succeed: changing a @property
82 in a class to an ordinary method or a method to a member variable
82 in a class to an ordinary method or a method to a member variable
83 can cause problems (but in old objects only).
83 can cause problems (but in old objects only).
84
84
85 - Functions that are removed (eg. via monkey-patching) from a module
85 - Functions that are removed (eg. via monkey-patching) from a module
86 before it is reloaded are not upgraded.
86 before it is reloaded are not upgraded.
87
87
88 - C extension modules cannot be reloaded, and so cannot be autoreloaded.
88 - C extension modules cannot be reloaded, and so cannot be autoreloaded.
89 """
89 """
90 from __future__ import print_function
90 from __future__ import print_function
91
91
92 skip_doctest = True
92 skip_doctest = True
93
93
94 #-----------------------------------------------------------------------------
94 #-----------------------------------------------------------------------------
95 # Copyright (C) 2000 Thomas Heller
95 # Copyright (C) 2000 Thomas Heller
96 # Copyright (C) 2008 Pauli Virtanen <pav@iki.fi>
96 # Copyright (C) 2008 Pauli Virtanen <pav@iki.fi>
97 # Copyright (C) 2012 The IPython Development Team
97 # Copyright (C) 2012 The IPython Development Team
98 #
98 #
99 # Distributed under the terms of the BSD License. The full license is in
99 # Distributed under the terms of the BSD License. The full license is in
100 # the file COPYING, distributed as part of this software.
100 # the file COPYING, distributed as part of this software.
101 #-----------------------------------------------------------------------------
101 #-----------------------------------------------------------------------------
102 #
102 #
103 # This IPython module is written by Pauli Virtanen, based on the autoreload
103 # This IPython module is written by Pauli Virtanen, based on the autoreload
104 # code by Thomas Heller.
104 # code by Thomas Heller.
105
105
106 #-----------------------------------------------------------------------------
106 #-----------------------------------------------------------------------------
107 # Imports
107 # Imports
108 #-----------------------------------------------------------------------------
108 #-----------------------------------------------------------------------------
109
109
110 import os
110 import os
111 import sys
111 import sys
112 import traceback
112 import traceback
113 import types
113 import types
114 import weakref
114 import weakref
115
115
116 try:
116 try:
117 # Reload is not defined by default in Python3.
117 # Reload is not defined by default in Python3.
118 reload
118 reload
119 except NameError:
119 except NameError:
120 from imp import reload
120 from imp import reload
121
121
122 from IPython.utils import openpy
122 from IPython.utils import openpy
123 from IPython.utils.py3compat import PY3
123 from IPython.utils.py3compat import PY3
124
124
125 #------------------------------------------------------------------------------
125 #------------------------------------------------------------------------------
126 # Autoreload functionality
126 # Autoreload functionality
127 #------------------------------------------------------------------------------
127 #------------------------------------------------------------------------------
128
128
129 class ModuleReloader(object):
129 class ModuleReloader(object):
130 enabled = False
130 enabled = False
131 """Whether this reloader is enabled"""
131 """Whether this reloader is enabled"""
132
132
133 failed = {}
133 failed = {}
134 """Modules that failed to reload: {module: mtime-on-failed-reload, ...}"""
134 """Modules that failed to reload: {module: mtime-on-failed-reload, ...}"""
135
135
136 modules = {}
136 modules = {}
137 """Modules specially marked as autoreloadable."""
137 """Modules specially marked as autoreloadable."""
138
138
139 skip_modules = {}
139 skip_modules = {}
140 """Modules specially marked as not autoreloadable."""
140 """Modules specially marked as not autoreloadable."""
141
141
142 check_all = True
142 check_all = True
143 """Autoreload all modules, not just those listed in 'modules'"""
143 """Autoreload all modules, not just those listed in 'modules'"""
144
144
145 old_objects = {}
145 old_objects = {}
146 """(module-name, name) -> weakref, for replacing old code objects"""
146 """(module-name, name) -> weakref, for replacing old code objects"""
147
147
148 def mark_module_skipped(self, module_name):
148 def mark_module_skipped(self, module_name):
149 """Skip reloading the named module in the future"""
149 """Skip reloading the named module in the future"""
150 try:
150 try:
151 del self.modules[module_name]
151 del self.modules[module_name]
152 except KeyError:
152 except KeyError:
153 pass
153 pass
154 self.skip_modules[module_name] = True
154 self.skip_modules[module_name] = True
155
155
156 def mark_module_reloadable(self, module_name):
156 def mark_module_reloadable(self, module_name):
157 """Reload the named module in the future (if it is imported)"""
157 """Reload the named module in the future (if it is imported)"""
158 try:
158 try:
159 del self.skip_modules[module_name]
159 del self.skip_modules[module_name]
160 except KeyError:
160 except KeyError:
161 pass
161 pass
162 self.modules[module_name] = True
162 self.modules[module_name] = True
163
163
164 def aimport_module(self, module_name):
164 def aimport_module(self, module_name):
165 """Import a module, and mark it reloadable
165 """Import a module, and mark it reloadable
166
166
167 Returns
167 Returns
168 -------
168 -------
169 top_module : module
169 top_module : module
170 The imported module if it is top-level, or the top-level
170 The imported module if it is top-level, or the top-level
171 top_name : module
171 top_name : module
172 Name of top_module
172 Name of top_module
173
173
174 """
174 """
175 self.mark_module_reloadable(module_name)
175 self.mark_module_reloadable(module_name)
176
176
177 __import__(module_name)
177 __import__(module_name)
178 top_name = module_name.split('.')[0]
178 top_name = module_name.split('.')[0]
179 top_module = sys.modules[top_name]
179 top_module = sys.modules[top_name]
180 return top_module, top_name
180 return top_module, top_name
181
181
182 def check(self, check_all=False):
182 def check(self, check_all=False):
183 """Check whether some modules need to be reloaded."""
183 """Check whether some modules need to be reloaded."""
184
184
185 if not self.enabled and not check_all:
185 if not self.enabled and not check_all:
186 return
186 return
187
187
188 if check_all or self.check_all:
188 if check_all or self.check_all:
189 modules = list(sys.modules.keys())
189 modules = list(sys.modules.keys())
190 else:
190 else:
191 modules = list(self.modules.keys())
191 modules = list(self.modules.keys())
192
192
193 for modname in modules:
193 for modname in modules:
194 m = sys.modules.get(modname, None)
194 m = sys.modules.get(modname, None)
195
195
196 if modname in self.skip_modules:
196 if modname in self.skip_modules:
197 continue
197 continue
198
198
199 if not hasattr(m, '__file__'):
199 if not hasattr(m, '__file__'):
200 continue
200 continue
201
201
202 if m.__name__ == '__main__':
202 if m.__name__ == '__main__':
203 # we cannot reload(__main__)
203 # we cannot reload(__main__)
204 continue
204 continue
205
205
206 filename = m.__file__
206 filename = m.__file__
207 path, ext = os.path.splitext(filename)
207 path, ext = os.path.splitext(filename)
208
208
209 if ext.lower() == '.py':
209 if ext.lower() == '.py':
210 pyc_filename = openpy.cache_from_source(filename)
210 pyc_filename = openpy.cache_from_source(filename)
211 py_filename = filename
211 py_filename = filename
212 else:
212 else:
213 pyc_filename = filename
213 pyc_filename = filename
214 try:
214 try:
215 py_filename = openpy.source_from_cache(filename)
215 py_filename = openpy.source_from_cache(filename)
216 except ValueError:
216 except ValueError:
217 continue
217 continue
218
218
219 try:
219 try:
220 pymtime = os.stat(py_filename).st_mtime
220 pymtime = os.stat(py_filename).st_mtime
221 if pymtime <= os.stat(pyc_filename).st_mtime:
221 if pymtime <= os.stat(pyc_filename).st_mtime:
222 continue
222 continue
223 if self.failed.get(py_filename, None) == pymtime:
223 if self.failed.get(py_filename, None) == pymtime:
224 continue
224 continue
225 except OSError:
225 except OSError:
226 continue
226 continue
227
227
228 try:
228 try:
229 superreload(m, reload, self.old_objects)
229 superreload(m, reload, self.old_objects)
230 if py_filename in self.failed:
230 if py_filename in self.failed:
231 del self.failed[py_filename]
231 del self.failed[py_filename]
232 except:
232 except:
233 print("[autoreload of %s failed: %s]" % (
233 print("[autoreload of %s failed: %s]" % (
234 modname, traceback.format_exc(1)), file=sys.stderr)
234 modname, traceback.format_exc(1)), file=sys.stderr)
235 self.failed[py_filename] = pymtime
235 self.failed[py_filename] = pymtime
236
236
237 #------------------------------------------------------------------------------
237 #------------------------------------------------------------------------------
238 # superreload
238 # superreload
239 #------------------------------------------------------------------------------
239 #------------------------------------------------------------------------------
240
240
241 if PY3:
241 if PY3:
242 func_attrs = ['__code__', '__defaults__', '__doc__',
242 func_attrs = ['__code__', '__defaults__', '__doc__',
243 '__closure__', '__globals__', '__dict__']
243 '__closure__', '__globals__', '__dict__']
244 else:
244 else:
245 func_attrs = ['func_code', 'func_defaults', 'func_doc',
245 func_attrs = ['func_code', 'func_defaults', 'func_doc',
246 'func_closure', 'func_globals', 'func_dict']
246 'func_closure', 'func_globals', 'func_dict']
247
247
248
248
249 def update_function(old, new):
249 def update_function(old, new):
250 """Upgrade the code object of a function"""
250 """Upgrade the code object of a function"""
251 for name in func_attrs:
251 for name in func_attrs:
252 try:
252 try:
253 setattr(old, name, getattr(new, name))
253 setattr(old, name, getattr(new, name))
254 except (AttributeError, TypeError):
254 except (AttributeError, TypeError):
255 pass
255 pass
256
256
257
257
258 def update_class(old, new):
258 def update_class(old, new):
259 """Replace stuff in the __dict__ of a class, and upgrade
259 """Replace stuff in the __dict__ of a class, and upgrade
260 method code objects"""
260 method code objects"""
261 for key in list(old.__dict__.keys()):
261 for key in list(old.__dict__.keys()):
262 old_obj = getattr(old, key)
262 old_obj = getattr(old, key)
263
263
264 try:
264 try:
265 new_obj = getattr(new, key)
265 new_obj = getattr(new, key)
266 except AttributeError:
266 except AttributeError:
267 # obsolete attribute: remove it
267 # obsolete attribute: remove it
268 try:
268 try:
269 delattr(old, key)
269 delattr(old, key)
270 except (AttributeError, TypeError):
270 except (AttributeError, TypeError):
271 pass
271 pass
272 continue
272 continue
273
273
274 if update_generic(old_obj, new_obj): continue
274 if update_generic(old_obj, new_obj): continue
275
275
276 try:
276 try:
277 setattr(old, key, getattr(new, key))
277 setattr(old, key, getattr(new, key))
278 except (AttributeError, TypeError):
278 except (AttributeError, TypeError):
279 pass # skip non-writable attributes
279 pass # skip non-writable attributes
280
280
281
281
282 def update_property(old, new):
282 def update_property(old, new):
283 """Replace get/set/del functions of a property"""
283 """Replace get/set/del functions of a property"""
284 update_generic(old.fdel, new.fdel)
284 update_generic(old.fdel, new.fdel)
285 update_generic(old.fget, new.fget)
285 update_generic(old.fget, new.fget)
286 update_generic(old.fset, new.fset)
286 update_generic(old.fset, new.fset)
287
287
288
288
289 def isinstance2(a, b, typ):
289 def isinstance2(a, b, typ):
290 return isinstance(a, typ) and isinstance(b, typ)
290 return isinstance(a, typ) and isinstance(b, typ)
291
291
292
292
293 UPDATE_RULES = [
293 UPDATE_RULES = [
294 (lambda a, b: isinstance2(a, b, type),
294 (lambda a, b: isinstance2(a, b, type),
295 update_class),
295 update_class),
296 (lambda a, b: isinstance2(a, b, types.FunctionType),
296 (lambda a, b: isinstance2(a, b, types.FunctionType),
297 update_function),
297 update_function),
298 (lambda a, b: isinstance2(a, b, property),
298 (lambda a, b: isinstance2(a, b, property),
299 update_property),
299 update_property),
300 ]
300 ]
301
301
302
302
303 if PY3:
303 if PY3:
304 UPDATE_RULES.extend([(lambda a, b: isinstance2(a, b, types.MethodType),
304 UPDATE_RULES.extend([(lambda a, b: isinstance2(a, b, types.MethodType),
305 lambda a, b: update_function(a.__func__, b.__func__)),
305 lambda a, b: update_function(a.__func__, b.__func__)),
306 ])
306 ])
307 else:
307 else:
308 UPDATE_RULES.extend([(lambda a, b: isinstance2(a, b, types.ClassType),
308 UPDATE_RULES.extend([(lambda a, b: isinstance2(a, b, types.ClassType),
309 update_class),
309 update_class),
310 (lambda a, b: isinstance2(a, b, types.MethodType),
310 (lambda a, b: isinstance2(a, b, types.MethodType),
311 lambda a, b: update_function(a.__func__, b.__func__)),
311 lambda a, b: update_function(a.__func__, b.__func__)),
312 ])
312 ])
313
313
314
314
315 def update_generic(a, b):
315 def update_generic(a, b):
316 for type_check, update in UPDATE_RULES:
316 for type_check, update in UPDATE_RULES:
317 if type_check(a, b):
317 if type_check(a, b):
318 update(a, b)
318 update(a, b)
319 return True
319 return True
320 return False
320 return False
321
321
322
322
323 class StrongRef(object):
323 class StrongRef(object):
324 def __init__(self, obj):
324 def __init__(self, obj):
325 self.obj = obj
325 self.obj = obj
326 def __call__(self):
326 def __call__(self):
327 return self.obj
327 return self.obj
328
328
329
329
330 def superreload(module, reload=reload, old_objects={}):
330 def superreload(module, reload=reload, old_objects={}):
331 """Enhanced version of the builtin reload function.
331 """Enhanced version of the builtin reload function.
332
332
333 superreload remembers objects previously in the module, and
333 superreload remembers objects previously in the module, and
334
334
335 - upgrades the class dictionary of every old class in the module
335 - upgrades the class dictionary of every old class in the module
336 - upgrades the code object of every old function and method
336 - upgrades the code object of every old function and method
337 - clears the module's namespace before reloading
337 - clears the module's namespace before reloading
338
338
339 """
339 """
340
340
341 # collect old objects in the module
341 # collect old objects in the module
342 for name, obj in list(module.__dict__.items()):
342 for name, obj in list(module.__dict__.items()):
343 if not hasattr(obj, '__module__') or obj.__module__ != module.__name__:
343 if not hasattr(obj, '__module__') or obj.__module__ != module.__name__:
344 continue
344 continue
345 key = (module.__name__, name)
345 key = (module.__name__, name)
346 try:
346 try:
347 old_objects.setdefault(key, []).append(weakref.ref(obj))
347 old_objects.setdefault(key, []).append(weakref.ref(obj))
348 except TypeError:
348 except TypeError:
349 # weakref doesn't work for all types;
349 # weakref doesn't work for all types;
350 # create strong references for 'important' cases
350 # create strong references for 'important' cases
351 if not PY3 and isinstance(obj, types.ClassType):
351 if not PY3 and isinstance(obj, types.ClassType):
352 old_objects.setdefault(key, []).append(StrongRef(obj))
352 old_objects.setdefault(key, []).append(StrongRef(obj))
353
353
354 # reload module
354 # reload module
355 try:
355 try:
356 # clear namespace first from old cruft
356 # clear namespace first from old cruft
357 old_dict = module.__dict__.copy()
357 old_dict = module.__dict__.copy()
358 old_name = module.__name__
358 old_name = module.__name__
359 module.__dict__.clear()
359 module.__dict__.clear()
360 module.__dict__['__name__'] = old_name
360 module.__dict__['__name__'] = old_name
361 module.__dict__['__loader__'] = old_dict['__loader__']
361 module.__dict__['__loader__'] = old_dict['__loader__']
362 except (TypeError, AttributeError, KeyError):
362 except (TypeError, AttributeError, KeyError):
363 pass
363 pass
364
364
365 try:
365 try:
366 module = reload(module)
366 module = reload(module)
367 except:
367 except:
368 # restore module dictionary on failed reload
368 # restore module dictionary on failed reload
369 module.__dict__.update(old_dict)
369 module.__dict__.update(old_dict)
370 raise
370 raise
371
371
372 # iterate over all objects and update functions & classes
372 # iterate over all objects and update functions & classes
373 for name, new_obj in list(module.__dict__.items()):
373 for name, new_obj in list(module.__dict__.items()):
374 key = (module.__name__, name)
374 key = (module.__name__, name)
375 if key not in old_objects: continue
375 if key not in old_objects: continue
376
376
377 new_refs = []
377 new_refs = []
378 for old_ref in old_objects[key]:
378 for old_ref in old_objects[key]:
379 old_obj = old_ref()
379 old_obj = old_ref()
380 if old_obj is None: continue
380 if old_obj is None: continue
381 new_refs.append(old_ref)
381 new_refs.append(old_ref)
382 update_generic(old_obj, new_obj)
382 update_generic(old_obj, new_obj)
383
383
384 if new_refs:
384 if new_refs:
385 old_objects[key] = new_refs
385 old_objects[key] = new_refs
386 else:
386 else:
387 del old_objects[key]
387 del old_objects[key]
388
388
389 return module
389 return module
390
390
391 #------------------------------------------------------------------------------
391 #------------------------------------------------------------------------------
392 # IPython connectivity
392 # IPython connectivity
393 #------------------------------------------------------------------------------
393 #------------------------------------------------------------------------------
394
394
395 from IPython.core.hooks import TryNext
396 from IPython.core.magic import Magics, magics_class, line_magic
395 from IPython.core.magic import Magics, magics_class, line_magic
397
396
398 @magics_class
397 @magics_class
399 class AutoreloadMagics(Magics):
398 class AutoreloadMagics(Magics):
400 def __init__(self, *a, **kw):
399 def __init__(self, *a, **kw):
401 super(AutoreloadMagics, self).__init__(*a, **kw)
400 super(AutoreloadMagics, self).__init__(*a, **kw)
402 self._reloader = ModuleReloader()
401 self._reloader = ModuleReloader()
403 self._reloader.check_all = False
402 self._reloader.check_all = False
404
403
405 @line_magic
404 @line_magic
406 def autoreload(self, parameter_s=''):
405 def autoreload(self, parameter_s=''):
407 r"""%autoreload => Reload modules automatically
406 r"""%autoreload => Reload modules automatically
408
407
409 %autoreload
408 %autoreload
410 Reload all modules (except those excluded by %aimport) automatically
409 Reload all modules (except those excluded by %aimport) automatically
411 now.
410 now.
412
411
413 %autoreload 0
412 %autoreload 0
414 Disable automatic reloading.
413 Disable automatic reloading.
415
414
416 %autoreload 1
415 %autoreload 1
417 Reload all modules imported with %aimport every time before executing
416 Reload all modules imported with %aimport every time before executing
418 the Python code typed.
417 the Python code typed.
419
418
420 %autoreload 2
419 %autoreload 2
421 Reload all modules (except those excluded by %aimport) every time
420 Reload all modules (except those excluded by %aimport) every time
422 before executing the Python code typed.
421 before executing the Python code typed.
423
422
424 Reloading Python modules in a reliable way is in general
423 Reloading Python modules in a reliable way is in general
425 difficult, and unexpected things may occur. %autoreload tries to
424 difficult, and unexpected things may occur. %autoreload tries to
426 work around common pitfalls by replacing function code objects and
425 work around common pitfalls by replacing function code objects and
427 parts of classes previously in the module with new versions. This
426 parts of classes previously in the module with new versions. This
428 makes the following things to work:
427 makes the following things to work:
429
428
430 - Functions and classes imported via 'from xxx import foo' are upgraded
429 - Functions and classes imported via 'from xxx import foo' are upgraded
431 to new versions when 'xxx' is reloaded.
430 to new versions when 'xxx' is reloaded.
432
431
433 - Methods and properties of classes are upgraded on reload, so that
432 - Methods and properties of classes are upgraded on reload, so that
434 calling 'c.foo()' on an object 'c' created before the reload causes
433 calling 'c.foo()' on an object 'c' created before the reload causes
435 the new code for 'foo' to be executed.
434 the new code for 'foo' to be executed.
436
435
437 Some of the known remaining caveats are:
436 Some of the known remaining caveats are:
438
437
439 - Replacing code objects does not always succeed: changing a @property
438 - Replacing code objects does not always succeed: changing a @property
440 in a class to an ordinary method or a method to a member variable
439 in a class to an ordinary method or a method to a member variable
441 can cause problems (but in old objects only).
440 can cause problems (but in old objects only).
442
441
443 - Functions that are removed (eg. via monkey-patching) from a module
442 - Functions that are removed (eg. via monkey-patching) from a module
444 before it is reloaded are not upgraded.
443 before it is reloaded are not upgraded.
445
444
446 - C extension modules cannot be reloaded, and so cannot be
445 - C extension modules cannot be reloaded, and so cannot be
447 autoreloaded.
446 autoreloaded.
448
447
449 """
448 """
450 if parameter_s == '':
449 if parameter_s == '':
451 self._reloader.check(True)
450 self._reloader.check(True)
452 elif parameter_s == '0':
451 elif parameter_s == '0':
453 self._reloader.enabled = False
452 self._reloader.enabled = False
454 elif parameter_s == '1':
453 elif parameter_s == '1':
455 self._reloader.check_all = False
454 self._reloader.check_all = False
456 self._reloader.enabled = True
455 self._reloader.enabled = True
457 elif parameter_s == '2':
456 elif parameter_s == '2':
458 self._reloader.check_all = True
457 self._reloader.check_all = True
459 self._reloader.enabled = True
458 self._reloader.enabled = True
460
459
461 @line_magic
460 @line_magic
462 def aimport(self, parameter_s='', stream=None):
461 def aimport(self, parameter_s='', stream=None):
463 """%aimport => Import modules for automatic reloading.
462 """%aimport => Import modules for automatic reloading.
464
463
465 %aimport
464 %aimport
466 List modules to automatically import and not to import.
465 List modules to automatically import and not to import.
467
466
468 %aimport foo
467 %aimport foo
469 Import module 'foo' and mark it to be autoreloaded for %autoreload 1
468 Import module 'foo' and mark it to be autoreloaded for %autoreload 1
470
469
471 %aimport -foo
470 %aimport -foo
472 Mark module 'foo' to not be autoreloaded for %autoreload 1
471 Mark module 'foo' to not be autoreloaded for %autoreload 1
473 """
472 """
474 modname = parameter_s
473 modname = parameter_s
475 if not modname:
474 if not modname:
476 to_reload = sorted(self._reloader.modules.keys())
475 to_reload = sorted(self._reloader.modules.keys())
477 to_skip = sorted(self._reloader.skip_modules.keys())
476 to_skip = sorted(self._reloader.skip_modules.keys())
478 if stream is None:
477 if stream is None:
479 stream = sys.stdout
478 stream = sys.stdout
480 if self._reloader.check_all:
479 if self._reloader.check_all:
481 stream.write("Modules to reload:\nall-except-skipped\n")
480 stream.write("Modules to reload:\nall-except-skipped\n")
482 else:
481 else:
483 stream.write("Modules to reload:\n%s\n" % ' '.join(to_reload))
482 stream.write("Modules to reload:\n%s\n" % ' '.join(to_reload))
484 stream.write("\nModules to skip:\n%s\n" % ' '.join(to_skip))
483 stream.write("\nModules to skip:\n%s\n" % ' '.join(to_skip))
485 elif modname.startswith('-'):
484 elif modname.startswith('-'):
486 modname = modname[1:]
485 modname = modname[1:]
487 self._reloader.mark_module_skipped(modname)
486 self._reloader.mark_module_skipped(modname)
488 else:
487 else:
489 top_module, top_name = self._reloader.aimport_module(modname)
488 top_module, top_name = self._reloader.aimport_module(modname)
490
489
491 # Inject module to user namespace
490 # Inject module to user namespace
492 self.shell.push({top_name: top_module})
491 self.shell.push({top_name: top_module})
493
492
494 def pre_run_code_hook(self, ip):
493 def pre_execute_explicit(self):
495 if not self._reloader.enabled:
494 if self._reloader.enabled:
496 raise TryNext
497 try:
495 try:
498 self._reloader.check()
496 self._reloader.check()
499 except:
497 except:
500 pass
498 pass
501
499
502
500
503 def load_ipython_extension(ip):
501 def load_ipython_extension(ip):
504 """Load the extension in IPython."""
502 """Load the extension in IPython."""
505 auto_reload = AutoreloadMagics(ip)
503 auto_reload = AutoreloadMagics(ip)
506 ip.register_magics(auto_reload)
504 ip.register_magics(auto_reload)
507 ip.set_hook('pre_run_code_hook', auto_reload.pre_run_code_hook)
505 ip.callbacks.register('pre_execute_explicit', auto_reload.pre_execute_explicit)
@@ -1,322 +1,321
1 """Tests for autoreload extension.
1 """Tests for autoreload extension.
2 """
2 """
3 #-----------------------------------------------------------------------------
3 #-----------------------------------------------------------------------------
4 # Copyright (c) 2012 IPython Development Team.
4 # Copyright (c) 2012 IPython Development Team.
5 #
5 #
6 # Distributed under the terms of the Modified BSD License.
6 # Distributed under the terms of the Modified BSD License.
7 #
7 #
8 # The full license is in the file COPYING.txt, distributed with this software.
8 # The full license is in the file COPYING.txt, distributed with this software.
9 #-----------------------------------------------------------------------------
9 #-----------------------------------------------------------------------------
10
10
11 #-----------------------------------------------------------------------------
11 #-----------------------------------------------------------------------------
12 # Imports
12 # Imports
13 #-----------------------------------------------------------------------------
13 #-----------------------------------------------------------------------------
14
14
15 import os
15 import os
16 import sys
16 import sys
17 import tempfile
17 import tempfile
18 import shutil
18 import shutil
19 import random
19 import random
20 import time
20 import time
21
21
22 import nose.tools as nt
22 import nose.tools as nt
23 import IPython.testing.tools as tt
23 import IPython.testing.tools as tt
24
24
25 from IPython.extensions.autoreload import AutoreloadMagics
25 from IPython.extensions.autoreload import AutoreloadMagics
26 from IPython.core.hooks import TryNext
26 from IPython.core.callbacks import CallbackManager, pre_execute_explicit
27 from IPython.utils.py3compat import PY3
27 from IPython.utils.py3compat import PY3
28
28
29 if PY3:
29 if PY3:
30 from io import StringIO
30 from io import StringIO
31 else:
31 else:
32 from StringIO import StringIO
32 from StringIO import StringIO
33
33
34 #-----------------------------------------------------------------------------
34 #-----------------------------------------------------------------------------
35 # Test fixture
35 # Test fixture
36 #-----------------------------------------------------------------------------
36 #-----------------------------------------------------------------------------
37
37
38 noop = lambda *a, **kw: None
38 noop = lambda *a, **kw: None
39
39
40 class FakeShell(object):
40 class FakeShell(object):
41
41
42 def __init__(self):
42 def __init__(self):
43 self.ns = {}
43 self.ns = {}
44 self.callbacks = CallbackManager(self, {'pre_execute_explicit', pre_execute_explicit})
44 self.auto_magics = AutoreloadMagics(shell=self)
45 self.auto_magics = AutoreloadMagics(shell=self)
46 self.callbacks.register('pre_execute_explicit', self.auto_magics.pre_execute_explicit)
45
47
46 register_magics = set_hook = noop
48 register_magics = set_hook = noop
47
49
48 def run_code(self, code):
50 def run_code(self, code):
49 try:
51 self.callbacks.fire('pre_execute_explicit')
50 self.auto_magics.pre_run_code_hook(self)
51 except TryNext:
52 pass
53 exec(code, self.ns)
52 exec(code, self.ns)
54
53
55 def push(self, items):
54 def push(self, items):
56 self.ns.update(items)
55 self.ns.update(items)
57
56
58 def magic_autoreload(self, parameter):
57 def magic_autoreload(self, parameter):
59 self.auto_magics.autoreload(parameter)
58 self.auto_magics.autoreload(parameter)
60
59
61 def magic_aimport(self, parameter, stream=None):
60 def magic_aimport(self, parameter, stream=None):
62 self.auto_magics.aimport(parameter, stream=stream)
61 self.auto_magics.aimport(parameter, stream=stream)
63
62
64
63
65 class Fixture(object):
64 class Fixture(object):
66 """Fixture for creating test module files"""
65 """Fixture for creating test module files"""
67
66
68 test_dir = None
67 test_dir = None
69 old_sys_path = None
68 old_sys_path = None
70 filename_chars = "abcdefghijklmopqrstuvwxyz0123456789"
69 filename_chars = "abcdefghijklmopqrstuvwxyz0123456789"
71
70
72 def setUp(self):
71 def setUp(self):
73 self.test_dir = tempfile.mkdtemp()
72 self.test_dir = tempfile.mkdtemp()
74 self.old_sys_path = list(sys.path)
73 self.old_sys_path = list(sys.path)
75 sys.path.insert(0, self.test_dir)
74 sys.path.insert(0, self.test_dir)
76 self.shell = FakeShell()
75 self.shell = FakeShell()
77
76
78 def tearDown(self):
77 def tearDown(self):
79 shutil.rmtree(self.test_dir)
78 shutil.rmtree(self.test_dir)
80 sys.path = self.old_sys_path
79 sys.path = self.old_sys_path
81
80
82 self.test_dir = None
81 self.test_dir = None
83 self.old_sys_path = None
82 self.old_sys_path = None
84 self.shell = None
83 self.shell = None
85
84
86 def get_module(self):
85 def get_module(self):
87 module_name = "tmpmod_" + "".join(random.sample(self.filename_chars,20))
86 module_name = "tmpmod_" + "".join(random.sample(self.filename_chars,20))
88 if module_name in sys.modules:
87 if module_name in sys.modules:
89 del sys.modules[module_name]
88 del sys.modules[module_name]
90 file_name = os.path.join(self.test_dir, module_name + ".py")
89 file_name = os.path.join(self.test_dir, module_name + ".py")
91 return module_name, file_name
90 return module_name, file_name
92
91
93 def write_file(self, filename, content):
92 def write_file(self, filename, content):
94 """
93 """
95 Write a file, and force a timestamp difference of at least one second
94 Write a file, and force a timestamp difference of at least one second
96
95
97 Notes
96 Notes
98 -----
97 -----
99 Python's .pyc files record the timestamp of their compilation
98 Python's .pyc files record the timestamp of their compilation
100 with a time resolution of one second.
99 with a time resolution of one second.
101
100
102 Therefore, we need to force a timestamp difference between .py
101 Therefore, we need to force a timestamp difference between .py
103 and .pyc, without having the .py file be timestamped in the
102 and .pyc, without having the .py file be timestamped in the
104 future, and without changing the timestamp of the .pyc file
103 future, and without changing the timestamp of the .pyc file
105 (because that is stored in the file). The only reliable way
104 (because that is stored in the file). The only reliable way
106 to achieve this seems to be to sleep.
105 to achieve this seems to be to sleep.
107 """
106 """
108
107
109 # Sleep one second + eps
108 # Sleep one second + eps
110 time.sleep(1.05)
109 time.sleep(1.05)
111
110
112 # Write
111 # Write
113 f = open(filename, 'w')
112 f = open(filename, 'w')
114 try:
113 try:
115 f.write(content)
114 f.write(content)
116 finally:
115 finally:
117 f.close()
116 f.close()
118
117
119 def new_module(self, code):
118 def new_module(self, code):
120 mod_name, mod_fn = self.get_module()
119 mod_name, mod_fn = self.get_module()
121 f = open(mod_fn, 'w')
120 f = open(mod_fn, 'w')
122 try:
121 try:
123 f.write(code)
122 f.write(code)
124 finally:
123 finally:
125 f.close()
124 f.close()
126 return mod_name, mod_fn
125 return mod_name, mod_fn
127
126
128 #-----------------------------------------------------------------------------
127 #-----------------------------------------------------------------------------
129 # Test automatic reloading
128 # Test automatic reloading
130 #-----------------------------------------------------------------------------
129 #-----------------------------------------------------------------------------
131
130
132 class TestAutoreload(Fixture):
131 class TestAutoreload(Fixture):
133 def _check_smoketest(self, use_aimport=True):
132 def _check_smoketest(self, use_aimport=True):
134 """
133 """
135 Functional test for the automatic reloader using either
134 Functional test for the automatic reloader using either
136 '%autoreload 1' or '%autoreload 2'
135 '%autoreload 1' or '%autoreload 2'
137 """
136 """
138
137
139 mod_name, mod_fn = self.new_module("""
138 mod_name, mod_fn = self.new_module("""
140 x = 9
139 x = 9
141
140
142 z = 123 # this item will be deleted
141 z = 123 # this item will be deleted
143
142
144 def foo(y):
143 def foo(y):
145 return y + 3
144 return y + 3
146
145
147 class Baz(object):
146 class Baz(object):
148 def __init__(self, x):
147 def __init__(self, x):
149 self.x = x
148 self.x = x
150 def bar(self, y):
149 def bar(self, y):
151 return self.x + y
150 return self.x + y
152 @property
151 @property
153 def quux(self):
152 def quux(self):
154 return 42
153 return 42
155 def zzz(self):
154 def zzz(self):
156 '''This method will be deleted below'''
155 '''This method will be deleted below'''
157 return 99
156 return 99
158
157
159 class Bar: # old-style class: weakref doesn't work for it on Python < 2.7
158 class Bar: # old-style class: weakref doesn't work for it on Python < 2.7
160 def foo(self):
159 def foo(self):
161 return 1
160 return 1
162 """)
161 """)
163
162
164 #
163 #
165 # Import module, and mark for reloading
164 # Import module, and mark for reloading
166 #
165 #
167 if use_aimport:
166 if use_aimport:
168 self.shell.magic_autoreload("1")
167 self.shell.magic_autoreload("1")
169 self.shell.magic_aimport(mod_name)
168 self.shell.magic_aimport(mod_name)
170 stream = StringIO()
169 stream = StringIO()
171 self.shell.magic_aimport("", stream=stream)
170 self.shell.magic_aimport("", stream=stream)
172 nt.assert_true(("Modules to reload:\n%s" % mod_name) in
171 nt.assert_true(("Modules to reload:\n%s" % mod_name) in
173 stream.getvalue())
172 stream.getvalue())
174
173
175 nt.assert_raises(
174 nt.assert_raises(
176 ImportError,
175 ImportError,
177 self.shell.magic_aimport, "tmpmod_as318989e89ds")
176 self.shell.magic_aimport, "tmpmod_as318989e89ds")
178 else:
177 else:
179 self.shell.magic_autoreload("2")
178 self.shell.magic_autoreload("2")
180 self.shell.run_code("import %s" % mod_name)
179 self.shell.run_code("import %s" % mod_name)
181 stream = StringIO()
180 stream = StringIO()
182 self.shell.magic_aimport("", stream=stream)
181 self.shell.magic_aimport("", stream=stream)
183 nt.assert_true("Modules to reload:\nall-except-skipped" in
182 nt.assert_true("Modules to reload:\nall-except-skipped" in
184 stream.getvalue())
183 stream.getvalue())
185 nt.assert_in(mod_name, self.shell.ns)
184 nt.assert_in(mod_name, self.shell.ns)
186
185
187 mod = sys.modules[mod_name]
186 mod = sys.modules[mod_name]
188
187
189 #
188 #
190 # Test module contents
189 # Test module contents
191 #
190 #
192 old_foo = mod.foo
191 old_foo = mod.foo
193 old_obj = mod.Baz(9)
192 old_obj = mod.Baz(9)
194 old_obj2 = mod.Bar()
193 old_obj2 = mod.Bar()
195
194
196 def check_module_contents():
195 def check_module_contents():
197 nt.assert_equal(mod.x, 9)
196 nt.assert_equal(mod.x, 9)
198 nt.assert_equal(mod.z, 123)
197 nt.assert_equal(mod.z, 123)
199
198
200 nt.assert_equal(old_foo(0), 3)
199 nt.assert_equal(old_foo(0), 3)
201 nt.assert_equal(mod.foo(0), 3)
200 nt.assert_equal(mod.foo(0), 3)
202
201
203 obj = mod.Baz(9)
202 obj = mod.Baz(9)
204 nt.assert_equal(old_obj.bar(1), 10)
203 nt.assert_equal(old_obj.bar(1), 10)
205 nt.assert_equal(obj.bar(1), 10)
204 nt.assert_equal(obj.bar(1), 10)
206 nt.assert_equal(obj.quux, 42)
205 nt.assert_equal(obj.quux, 42)
207 nt.assert_equal(obj.zzz(), 99)
206 nt.assert_equal(obj.zzz(), 99)
208
207
209 obj2 = mod.Bar()
208 obj2 = mod.Bar()
210 nt.assert_equal(old_obj2.foo(), 1)
209 nt.assert_equal(old_obj2.foo(), 1)
211 nt.assert_equal(obj2.foo(), 1)
210 nt.assert_equal(obj2.foo(), 1)
212
211
213 check_module_contents()
212 check_module_contents()
214
213
215 #
214 #
216 # Simulate a failed reload: no reload should occur and exactly
215 # Simulate a failed reload: no reload should occur and exactly
217 # one error message should be printed
216 # one error message should be printed
218 #
217 #
219 self.write_file(mod_fn, """
218 self.write_file(mod_fn, """
220 a syntax error
219 a syntax error
221 """)
220 """)
222
221
223 with tt.AssertPrints(('[autoreload of %s failed:' % mod_name), channel='stderr'):
222 with tt.AssertPrints(('[autoreload of %s failed:' % mod_name), channel='stderr'):
224 self.shell.run_code("pass") # trigger reload
223 self.shell.run_code("pass") # trigger reload
225 with tt.AssertNotPrints(('[autoreload of %s failed:' % mod_name), channel='stderr'):
224 with tt.AssertNotPrints(('[autoreload of %s failed:' % mod_name), channel='stderr'):
226 self.shell.run_code("pass") # trigger another reload
225 self.shell.run_code("pass") # trigger another reload
227 check_module_contents()
226 check_module_contents()
228
227
229 #
228 #
230 # Rewrite module (this time reload should succeed)
229 # Rewrite module (this time reload should succeed)
231 #
230 #
232 self.write_file(mod_fn, """
231 self.write_file(mod_fn, """
233 x = 10
232 x = 10
234
233
235 def foo(y):
234 def foo(y):
236 return y + 4
235 return y + 4
237
236
238 class Baz(object):
237 class Baz(object):
239 def __init__(self, x):
238 def __init__(self, x):
240 self.x = x
239 self.x = x
241 def bar(self, y):
240 def bar(self, y):
242 return self.x + y + 1
241 return self.x + y + 1
243 @property
242 @property
244 def quux(self):
243 def quux(self):
245 return 43
244 return 43
246
245
247 class Bar: # old-style class
246 class Bar: # old-style class
248 def foo(self):
247 def foo(self):
249 return 2
248 return 2
250 """)
249 """)
251
250
252 def check_module_contents():
251 def check_module_contents():
253 nt.assert_equal(mod.x, 10)
252 nt.assert_equal(mod.x, 10)
254 nt.assert_false(hasattr(mod, 'z'))
253 nt.assert_false(hasattr(mod, 'z'))
255
254
256 nt.assert_equal(old_foo(0), 4) # superreload magic!
255 nt.assert_equal(old_foo(0), 4) # superreload magic!
257 nt.assert_equal(mod.foo(0), 4)
256 nt.assert_equal(mod.foo(0), 4)
258
257
259 obj = mod.Baz(9)
258 obj = mod.Baz(9)
260 nt.assert_equal(old_obj.bar(1), 11) # superreload magic!
259 nt.assert_equal(old_obj.bar(1), 11) # superreload magic!
261 nt.assert_equal(obj.bar(1), 11)
260 nt.assert_equal(obj.bar(1), 11)
262
261
263 nt.assert_equal(old_obj.quux, 43)
262 nt.assert_equal(old_obj.quux, 43)
264 nt.assert_equal(obj.quux, 43)
263 nt.assert_equal(obj.quux, 43)
265
264
266 nt.assert_false(hasattr(old_obj, 'zzz'))
265 nt.assert_false(hasattr(old_obj, 'zzz'))
267 nt.assert_false(hasattr(obj, 'zzz'))
266 nt.assert_false(hasattr(obj, 'zzz'))
268
267
269 obj2 = mod.Bar()
268 obj2 = mod.Bar()
270 nt.assert_equal(old_obj2.foo(), 2)
269 nt.assert_equal(old_obj2.foo(), 2)
271 nt.assert_equal(obj2.foo(), 2)
270 nt.assert_equal(obj2.foo(), 2)
272
271
273 self.shell.run_code("pass") # trigger reload
272 self.shell.run_code("pass") # trigger reload
274 check_module_contents()
273 check_module_contents()
275
274
276 #
275 #
277 # Another failure case: deleted file (shouldn't reload)
276 # Another failure case: deleted file (shouldn't reload)
278 #
277 #
279 os.unlink(mod_fn)
278 os.unlink(mod_fn)
280
279
281 self.shell.run_code("pass") # trigger reload
280 self.shell.run_code("pass") # trigger reload
282 check_module_contents()
281 check_module_contents()
283
282
284 #
283 #
285 # Disable autoreload and rewrite module: no reload should occur
284 # Disable autoreload and rewrite module: no reload should occur
286 #
285 #
287 if use_aimport:
286 if use_aimport:
288 self.shell.magic_aimport("-" + mod_name)
287 self.shell.magic_aimport("-" + mod_name)
289 stream = StringIO()
288 stream = StringIO()
290 self.shell.magic_aimport("", stream=stream)
289 self.shell.magic_aimport("", stream=stream)
291 nt.assert_true(("Modules to skip:\n%s" % mod_name) in
290 nt.assert_true(("Modules to skip:\n%s" % mod_name) in
292 stream.getvalue())
291 stream.getvalue())
293
292
294 # This should succeed, although no such module exists
293 # This should succeed, although no such module exists
295 self.shell.magic_aimport("-tmpmod_as318989e89ds")
294 self.shell.magic_aimport("-tmpmod_as318989e89ds")
296 else:
295 else:
297 self.shell.magic_autoreload("0")
296 self.shell.magic_autoreload("0")
298
297
299 self.write_file(mod_fn, """
298 self.write_file(mod_fn, """
300 x = -99
299 x = -99
301 """)
300 """)
302
301
303 self.shell.run_code("pass") # trigger reload
302 self.shell.run_code("pass") # trigger reload
304 self.shell.run_code("pass")
303 self.shell.run_code("pass")
305 check_module_contents()
304 check_module_contents()
306
305
307 #
306 #
308 # Re-enable autoreload: reload should now occur
307 # Re-enable autoreload: reload should now occur
309 #
308 #
310 if use_aimport:
309 if use_aimport:
311 self.shell.magic_aimport(mod_name)
310 self.shell.magic_aimport(mod_name)
312 else:
311 else:
313 self.shell.magic_autoreload("")
312 self.shell.magic_autoreload("")
314
313
315 self.shell.run_code("pass") # trigger reload
314 self.shell.run_code("pass") # trigger reload
316 nt.assert_equal(mod.x, -99)
315 nt.assert_equal(mod.x, -99)
317
316
318 def test_smoketest_aimport(self):
317 def test_smoketest_aimport(self):
319 self._check_smoketest(use_aimport=True)
318 self._check_smoketest(use_aimport=True)
320
319
321 def test_smoketest_autoreload(self):
320 def test_smoketest_autoreload(self):
322 self._check_smoketest(use_aimport=False)
321 self._check_smoketest(use_aimport=False)
General Comments 0
You need to be logged in to leave comments. Login now