##// END OF EJS Templates
Merge pull request #5044 from takluyver/autoreload-nopyc...
Min RK -
r15689:4527ad96 merge
parent child Browse files
Show More
@@ -1,505 +1,536 b''
1 """IPython extension to reload modules before executing user code.
1 """IPython extension to reload modules before executing user code.
2
2
3 ``autoreload`` reloads modules automatically before entering the execution of
3 ``autoreload`` reloads modules automatically before entering the execution of
4 code typed at the IPython prompt.
4 code typed at the IPython prompt.
5
5
6 This makes for example the following workflow possible:
6 This makes for example the following workflow possible:
7
7
8 .. sourcecode:: ipython
8 .. sourcecode:: ipython
9
9
10 In [1]: %load_ext autoreload
10 In [1]: %load_ext autoreload
11
11
12 In [2]: %autoreload 2
12 In [2]: %autoreload 2
13
13
14 In [3]: from foo import some_function
14 In [3]: from foo import some_function
15
15
16 In [4]: some_function()
16 In [4]: some_function()
17 Out[4]: 42
17 Out[4]: 42
18
18
19 In [5]: # open foo.py in an editor and change some_function to return 43
19 In [5]: # open foo.py in an editor and change some_function to return 43
20
20
21 In [6]: some_function()
21 In [6]: some_function()
22 Out[6]: 43
22 Out[6]: 43
23
23
24 The module was reloaded without reloading it explicitly, and the object
24 The module was reloaded without reloading it explicitly, and the object
25 imported with ``from foo import ...`` was also updated.
25 imported with ``from foo import ...`` was also updated.
26
26
27 Usage
27 Usage
28 =====
28 =====
29
29
30 The following magic commands are provided:
30 The following magic commands are provided:
31
31
32 ``%autoreload``
32 ``%autoreload``
33
33
34 Reload all modules (except those excluded by ``%aimport``)
34 Reload all modules (except those excluded by ``%aimport``)
35 automatically now.
35 automatically now.
36
36
37 ``%autoreload 0``
37 ``%autoreload 0``
38
38
39 Disable automatic reloading.
39 Disable automatic reloading.
40
40
41 ``%autoreload 1``
41 ``%autoreload 1``
42
42
43 Reload all modules imported with ``%aimport`` every time before
43 Reload all modules imported with ``%aimport`` every time before
44 executing the Python code typed.
44 executing the Python code typed.
45
45
46 ``%autoreload 2``
46 ``%autoreload 2``
47
47
48 Reload all modules (except those excluded by ``%aimport``) every
48 Reload all modules (except those excluded by ``%aimport``) every
49 time before executing the Python code typed.
49 time before executing the Python code typed.
50
50
51 ``%aimport``
51 ``%aimport``
52
52
53 List modules which are to be automatically imported or not to be imported.
53 List modules which are to be automatically imported or not to be imported.
54
54
55 ``%aimport foo``
55 ``%aimport foo``
56
56
57 Import module 'foo' and mark it to be autoreloaded for ``%autoreload 1``
57 Import module 'foo' and mark it to be autoreloaded for ``%autoreload 1``
58
58
59 ``%aimport -foo``
59 ``%aimport -foo``
60
60
61 Mark module 'foo' to not be autoreloaded.
61 Mark module 'foo' to not be autoreloaded.
62
62
63 Caveats
63 Caveats
64 =======
64 =======
65
65
66 Reloading Python modules in a reliable way is in general difficult,
66 Reloading Python modules in a reliable way is in general difficult,
67 and unexpected things may occur. ``%autoreload`` tries to work around
67 and unexpected things may occur. ``%autoreload`` tries to work around
68 common pitfalls by replacing function code objects and parts of
68 common pitfalls by replacing function code objects and parts of
69 classes previously in the module with new versions. This makes the
69 classes previously in the module with new versions. This makes the
70 following things to work:
70 following things to work:
71
71
72 - Functions and classes imported via 'from xxx import foo' are upgraded
72 - Functions and classes imported via 'from xxx import foo' are upgraded
73 to new versions when 'xxx' is reloaded.
73 to new versions when 'xxx' is reloaded.
74
74
75 - Methods and properties of classes are upgraded on reload, so that
75 - Methods and properties of classes are upgraded on reload, so that
76 calling 'c.foo()' on an object 'c' created before the reload causes
76 calling 'c.foo()' on an object 'c' created before the reload causes
77 the new code for 'foo' to be executed.
77 the new code for 'foo' to be executed.
78
78
79 Some of the known remaining caveats are:
79 Some of the known remaining caveats are:
80
80
81 - Replacing code objects does not always succeed: changing a @property
81 - Replacing code objects does not always succeed: changing a @property
82 in a class to an ordinary method or a method to a member variable
82 in a class to an ordinary method or a method to a member variable
83 can cause problems (but in old objects only).
83 can cause problems (but in old objects only).
84
84
85 - Functions that are removed (eg. via monkey-patching) from a module
85 - Functions that are removed (eg. via monkey-patching) from a module
86 before it is reloaded are not upgraded.
86 before it is reloaded are not upgraded.
87
87
88 - C extension modules cannot be reloaded, and so cannot be autoreloaded.
88 - C extension modules cannot be reloaded, and so cannot be autoreloaded.
89 """
89 """
90 from __future__ import print_function
90 from __future__ import print_function
91
91
92 skip_doctest = True
92 skip_doctest = True
93
93
94 #-----------------------------------------------------------------------------
94 #-----------------------------------------------------------------------------
95 # Copyright (C) 2000 Thomas Heller
95 # Copyright (C) 2000 Thomas Heller
96 # Copyright (C) 2008 Pauli Virtanen <pav@iki.fi>
96 # Copyright (C) 2008 Pauli Virtanen <pav@iki.fi>
97 # Copyright (C) 2012 The IPython Development Team
97 # Copyright (C) 2012 The IPython Development Team
98 #
98 #
99 # Distributed under the terms of the BSD License. The full license is in
99 # Distributed under the terms of the BSD License. The full license is in
100 # the file COPYING, distributed as part of this software.
100 # the file COPYING, distributed as part of this software.
101 #-----------------------------------------------------------------------------
101 #-----------------------------------------------------------------------------
102 #
102 #
103 # This IPython module is written by Pauli Virtanen, based on the autoreload
103 # This IPython module is written by Pauli Virtanen, based on the autoreload
104 # code by Thomas Heller.
104 # code by Thomas Heller.
105
105
106 #-----------------------------------------------------------------------------
106 #-----------------------------------------------------------------------------
107 # Imports
107 # Imports
108 #-----------------------------------------------------------------------------
108 #-----------------------------------------------------------------------------
109
109
110 import os
110 import os
111 import sys
111 import sys
112 import traceback
112 import traceback
113 import types
113 import types
114 import weakref
114 import weakref
115
115
116 try:
116 try:
117 # Reload is not defined by default in Python3.
117 # Reload is not defined by default in Python3.
118 reload
118 reload
119 except NameError:
119 except NameError:
120 from imp import reload
120 from imp import reload
121
121
122 from IPython.utils import openpy
122 from IPython.utils import openpy
123 from IPython.utils.py3compat import PY3
123 from IPython.utils.py3compat import PY3
124
124
125 #------------------------------------------------------------------------------
125 #------------------------------------------------------------------------------
126 # Autoreload functionality
126 # Autoreload functionality
127 #------------------------------------------------------------------------------
127 #------------------------------------------------------------------------------
128
128
129 class ModuleReloader(object):
129 class ModuleReloader(object):
130 enabled = False
130 enabled = False
131 """Whether this reloader is enabled"""
131 """Whether this reloader is enabled"""
132
132
133 failed = {}
134 """Modules that failed to reload: {module: mtime-on-failed-reload, ...}"""
135
136 modules = {}
137 """Modules specially marked as autoreloadable."""
138
139 skip_modules = {}
140 """Modules specially marked as not autoreloadable."""
141
142 check_all = True
133 check_all = True
143 """Autoreload all modules, not just those listed in 'modules'"""
134 """Autoreload all modules, not just those listed in 'modules'"""
144
135
145 old_objects = {}
136 def __init__(self):
146 """(module-name, name) -> weakref, for replacing old code objects"""
137 # Modules that failed to reload: {module: mtime-on-failed-reload, ...}
138 self.failed = {}
139 # Modules specially marked as autoreloadable.
140 self.modules = {}
141 # Modules specially marked as not autoreloadable.
142 self.skip_modules = {}
143 # (module-name, name) -> weakref, for replacing old code objects
144 self.old_objects = {}
145 # Module modification timestamps
146 self.modules_mtimes = {}
147
148 # Cache module modification times
149 self.check(check_all=True, do_reload=False)
147
150
148 def mark_module_skipped(self, module_name):
151 def mark_module_skipped(self, module_name):
149 """Skip reloading the named module in the future"""
152 """Skip reloading the named module in the future"""
150 try:
153 try:
151 del self.modules[module_name]
154 del self.modules[module_name]
152 except KeyError:
155 except KeyError:
153 pass
156 pass
154 self.skip_modules[module_name] = True
157 self.skip_modules[module_name] = True
155
158
156 def mark_module_reloadable(self, module_name):
159 def mark_module_reloadable(self, module_name):
157 """Reload the named module in the future (if it is imported)"""
160 """Reload the named module in the future (if it is imported)"""
158 try:
161 try:
159 del self.skip_modules[module_name]
162 del self.skip_modules[module_name]
160 except KeyError:
163 except KeyError:
161 pass
164 pass
162 self.modules[module_name] = True
165 self.modules[module_name] = True
163
166
164 def aimport_module(self, module_name):
167 def aimport_module(self, module_name):
165 """Import a module, and mark it reloadable
168 """Import a module, and mark it reloadable
166
169
167 Returns
170 Returns
168 -------
171 -------
169 top_module : module
172 top_module : module
170 The imported module if it is top-level, or the top-level
173 The imported module if it is top-level, or the top-level
171 top_name : module
174 top_name : module
172 Name of top_module
175 Name of top_module
173
176
174 """
177 """
175 self.mark_module_reloadable(module_name)
178 self.mark_module_reloadable(module_name)
176
179
177 __import__(module_name)
180 __import__(module_name)
178 top_name = module_name.split('.')[0]
181 top_name = module_name.split('.')[0]
179 top_module = sys.modules[top_name]
182 top_module = sys.modules[top_name]
180 return top_module, top_name
183 return top_module, top_name
181
184
182 def check(self, check_all=False):
185 def filename_and_mtime(self, module):
186 if not hasattr(module, '__file__'):
187 return None, None
188
189 if module.__name__ == '__main__':
190 # we cannot reload(__main__)
191 return None, None
192
193 filename = module.__file__
194 path, ext = os.path.splitext(filename)
195
196 if ext.lower() == '.py':
197 py_filename = filename
198 else:
199 try:
200 py_filename = openpy.source_from_cache(filename)
201 except ValueError:
202 return None, None
203
204 try:
205 pymtime = os.stat(py_filename).st_mtime
206 except OSError:
207 return None, None
208
209 return py_filename, pymtime
210
211 def check(self, check_all=False, do_reload=True):
183 """Check whether some modules need to be reloaded."""
212 """Check whether some modules need to be reloaded."""
184
213
185 if not self.enabled and not check_all:
214 if not self.enabled and not check_all:
186 return
215 return
187
216
188 if check_all or self.check_all:
217 if check_all or self.check_all:
189 modules = list(sys.modules.keys())
218 modules = list(sys.modules.keys())
190 else:
219 else:
191 modules = list(self.modules.keys())
220 modules = list(self.modules.keys())
192
221
193 for modname in modules:
222 for modname in modules:
194 m = sys.modules.get(modname, None)
223 m = sys.modules.get(modname, None)
195
224
196 if modname in self.skip_modules:
225 if modname in self.skip_modules:
197 continue
226 continue
198
227
199 if not hasattr(m, '__file__'):
228 py_filename, pymtime = self.filename_and_mtime(m)
229 if py_filename is None:
200 continue
230 continue
201
231
202 if m.__name__ == '__main__':
203 # we cannot reload(__main__)
204 continue
205
206 filename = m.__file__
207 path, ext = os.path.splitext(filename)
208
209 if ext.lower() == '.py':
210 pyc_filename = openpy.cache_from_source(filename)
211 py_filename = filename
212 else:
213 pyc_filename = filename
214 try:
215 py_filename = openpy.source_from_cache(filename)
216 except ValueError:
217 continue
218
219 try:
232 try:
220 pymtime = os.stat(py_filename).st_mtime
233 if pymtime <= self.modules_mtimes[modname]:
221 if pymtime <= os.stat(pyc_filename).st_mtime:
222 continue
234 continue
235 except KeyError:
236 self.modules_mtimes[modname] = pymtime
237 continue
238 else:
223 if self.failed.get(py_filename, None) == pymtime:
239 if self.failed.get(py_filename, None) == pymtime:
224 continue
240 continue
225 except OSError:
226 continue
227
241
228 try:
242 self.modules_mtimes[modname] = pymtime
229 superreload(m, reload, self.old_objects)
243
230 if py_filename in self.failed:
244 # If we've reached this point, we should try to reload the module
231 del self.failed[py_filename]
245 if do_reload:
232 except:
246 try:
233 print("[autoreload of %s failed: %s]" % (
247 superreload(m, reload, self.old_objects)
234 modname, traceback.format_exc(1)), file=sys.stderr)
248 if py_filename in self.failed:
235 self.failed[py_filename] = pymtime
249 del self.failed[py_filename]
250 except:
251 print("[autoreload of %s failed: %s]" % (
252 modname, traceback.format_exc(1)), file=sys.stderr)
253 self.failed[py_filename] = pymtime
236
254
237 #------------------------------------------------------------------------------
255 #------------------------------------------------------------------------------
238 # superreload
256 # superreload
239 #------------------------------------------------------------------------------
257 #------------------------------------------------------------------------------
240
258
241 if PY3:
259 if PY3:
242 func_attrs = ['__code__', '__defaults__', '__doc__',
260 func_attrs = ['__code__', '__defaults__', '__doc__',
243 '__closure__', '__globals__', '__dict__']
261 '__closure__', '__globals__', '__dict__']
244 else:
262 else:
245 func_attrs = ['func_code', 'func_defaults', 'func_doc',
263 func_attrs = ['func_code', 'func_defaults', 'func_doc',
246 'func_closure', 'func_globals', 'func_dict']
264 'func_closure', 'func_globals', 'func_dict']
247
265
248
266
249 def update_function(old, new):
267 def update_function(old, new):
250 """Upgrade the code object of a function"""
268 """Upgrade the code object of a function"""
251 for name in func_attrs:
269 for name in func_attrs:
252 try:
270 try:
253 setattr(old, name, getattr(new, name))
271 setattr(old, name, getattr(new, name))
254 except (AttributeError, TypeError):
272 except (AttributeError, TypeError):
255 pass
273 pass
256
274
257
275
258 def update_class(old, new):
276 def update_class(old, new):
259 """Replace stuff in the __dict__ of a class, and upgrade
277 """Replace stuff in the __dict__ of a class, and upgrade
260 method code objects"""
278 method code objects"""
261 for key in list(old.__dict__.keys()):
279 for key in list(old.__dict__.keys()):
262 old_obj = getattr(old, key)
280 old_obj = getattr(old, key)
263
281
264 try:
282 try:
265 new_obj = getattr(new, key)
283 new_obj = getattr(new, key)
266 except AttributeError:
284 except AttributeError:
267 # obsolete attribute: remove it
285 # obsolete attribute: remove it
268 try:
286 try:
269 delattr(old, key)
287 delattr(old, key)
270 except (AttributeError, TypeError):
288 except (AttributeError, TypeError):
271 pass
289 pass
272 continue
290 continue
273
291
274 if update_generic(old_obj, new_obj): continue
292 if update_generic(old_obj, new_obj): continue
275
293
276 try:
294 try:
277 setattr(old, key, getattr(new, key))
295 setattr(old, key, getattr(new, key))
278 except (AttributeError, TypeError):
296 except (AttributeError, TypeError):
279 pass # skip non-writable attributes
297 pass # skip non-writable attributes
280
298
281
299
282 def update_property(old, new):
300 def update_property(old, new):
283 """Replace get/set/del functions of a property"""
301 """Replace get/set/del functions of a property"""
284 update_generic(old.fdel, new.fdel)
302 update_generic(old.fdel, new.fdel)
285 update_generic(old.fget, new.fget)
303 update_generic(old.fget, new.fget)
286 update_generic(old.fset, new.fset)
304 update_generic(old.fset, new.fset)
287
305
288
306
289 def isinstance2(a, b, typ):
307 def isinstance2(a, b, typ):
290 return isinstance(a, typ) and isinstance(b, typ)
308 return isinstance(a, typ) and isinstance(b, typ)
291
309
292
310
293 UPDATE_RULES = [
311 UPDATE_RULES = [
294 (lambda a, b: isinstance2(a, b, type),
312 (lambda a, b: isinstance2(a, b, type),
295 update_class),
313 update_class),
296 (lambda a, b: isinstance2(a, b, types.FunctionType),
314 (lambda a, b: isinstance2(a, b, types.FunctionType),
297 update_function),
315 update_function),
298 (lambda a, b: isinstance2(a, b, property),
316 (lambda a, b: isinstance2(a, b, property),
299 update_property),
317 update_property),
300 ]
318 ]
301
319
302
320
303 if PY3:
321 if PY3:
304 UPDATE_RULES.extend([(lambda a, b: isinstance2(a, b, types.MethodType),
322 UPDATE_RULES.extend([(lambda a, b: isinstance2(a, b, types.MethodType),
305 lambda a, b: update_function(a.__func__, b.__func__)),
323 lambda a, b: update_function(a.__func__, b.__func__)),
306 ])
324 ])
307 else:
325 else:
308 UPDATE_RULES.extend([(lambda a, b: isinstance2(a, b, types.ClassType),
326 UPDATE_RULES.extend([(lambda a, b: isinstance2(a, b, types.ClassType),
309 update_class),
327 update_class),
310 (lambda a, b: isinstance2(a, b, types.MethodType),
328 (lambda a, b: isinstance2(a, b, types.MethodType),
311 lambda a, b: update_function(a.__func__, b.__func__)),
329 lambda a, b: update_function(a.__func__, b.__func__)),
312 ])
330 ])
313
331
314
332
315 def update_generic(a, b):
333 def update_generic(a, b):
316 for type_check, update in UPDATE_RULES:
334 for type_check, update in UPDATE_RULES:
317 if type_check(a, b):
335 if type_check(a, b):
318 update(a, b)
336 update(a, b)
319 return True
337 return True
320 return False
338 return False
321
339
322
340
323 class StrongRef(object):
341 class StrongRef(object):
324 def __init__(self, obj):
342 def __init__(self, obj):
325 self.obj = obj
343 self.obj = obj
326 def __call__(self):
344 def __call__(self):
327 return self.obj
345 return self.obj
328
346
329
347
330 def superreload(module, reload=reload, old_objects={}):
348 def superreload(module, reload=reload, old_objects={}):
331 """Enhanced version of the builtin reload function.
349 """Enhanced version of the builtin reload function.
332
350
333 superreload remembers objects previously in the module, and
351 superreload remembers objects previously in the module, and
334
352
335 - upgrades the class dictionary of every old class in the module
353 - upgrades the class dictionary of every old class in the module
336 - upgrades the code object of every old function and method
354 - upgrades the code object of every old function and method
337 - clears the module's namespace before reloading
355 - clears the module's namespace before reloading
338
356
339 """
357 """
340
358
341 # collect old objects in the module
359 # collect old objects in the module
342 for name, obj in list(module.__dict__.items()):
360 for name, obj in list(module.__dict__.items()):
343 if not hasattr(obj, '__module__') or obj.__module__ != module.__name__:
361 if not hasattr(obj, '__module__') or obj.__module__ != module.__name__:
344 continue
362 continue
345 key = (module.__name__, name)
363 key = (module.__name__, name)
346 try:
364 try:
347 old_objects.setdefault(key, []).append(weakref.ref(obj))
365 old_objects.setdefault(key, []).append(weakref.ref(obj))
348 except TypeError:
366 except TypeError:
349 # weakref doesn't work for all types;
367 # weakref doesn't work for all types;
350 # create strong references for 'important' cases
368 # create strong references for 'important' cases
351 if not PY3 and isinstance(obj, types.ClassType):
369 if not PY3 and isinstance(obj, types.ClassType):
352 old_objects.setdefault(key, []).append(StrongRef(obj))
370 old_objects.setdefault(key, []).append(StrongRef(obj))
353
371
354 # reload module
372 # reload module
355 try:
373 try:
356 # clear namespace first from old cruft
374 # clear namespace first from old cruft
357 old_dict = module.__dict__.copy()
375 old_dict = module.__dict__.copy()
358 old_name = module.__name__
376 old_name = module.__name__
359 module.__dict__.clear()
377 module.__dict__.clear()
360 module.__dict__['__name__'] = old_name
378 module.__dict__['__name__'] = old_name
361 module.__dict__['__loader__'] = old_dict['__loader__']
379 module.__dict__['__loader__'] = old_dict['__loader__']
362 except (TypeError, AttributeError, KeyError):
380 except (TypeError, AttributeError, KeyError):
363 pass
381 pass
364
382
365 try:
383 try:
366 module = reload(module)
384 module = reload(module)
367 except:
385 except:
368 # restore module dictionary on failed reload
386 # restore module dictionary on failed reload
369 module.__dict__.update(old_dict)
387 module.__dict__.update(old_dict)
370 raise
388 raise
371
389
372 # iterate over all objects and update functions & classes
390 # iterate over all objects and update functions & classes
373 for name, new_obj in list(module.__dict__.items()):
391 for name, new_obj in list(module.__dict__.items()):
374 key = (module.__name__, name)
392 key = (module.__name__, name)
375 if key not in old_objects: continue
393 if key not in old_objects: continue
376
394
377 new_refs = []
395 new_refs = []
378 for old_ref in old_objects[key]:
396 for old_ref in old_objects[key]:
379 old_obj = old_ref()
397 old_obj = old_ref()
380 if old_obj is None: continue
398 if old_obj is None: continue
381 new_refs.append(old_ref)
399 new_refs.append(old_ref)
382 update_generic(old_obj, new_obj)
400 update_generic(old_obj, new_obj)
383
401
384 if new_refs:
402 if new_refs:
385 old_objects[key] = new_refs
403 old_objects[key] = new_refs
386 else:
404 else:
387 del old_objects[key]
405 del old_objects[key]
388
406
389 return module
407 return module
390
408
391 #------------------------------------------------------------------------------
409 #------------------------------------------------------------------------------
392 # IPython connectivity
410 # IPython connectivity
393 #------------------------------------------------------------------------------
411 #------------------------------------------------------------------------------
394
412
395 from IPython.core.magic import Magics, magics_class, line_magic
413 from IPython.core.magic import Magics, magics_class, line_magic
396
414
397 @magics_class
415 @magics_class
398 class AutoreloadMagics(Magics):
416 class AutoreloadMagics(Magics):
399 def __init__(self, *a, **kw):
417 def __init__(self, *a, **kw):
400 super(AutoreloadMagics, self).__init__(*a, **kw)
418 super(AutoreloadMagics, self).__init__(*a, **kw)
401 self._reloader = ModuleReloader()
419 self._reloader = ModuleReloader()
402 self._reloader.check_all = False
420 self._reloader.check_all = False
421 self.loaded_modules = set(sys.modules)
403
422
404 @line_magic
423 @line_magic
405 def autoreload(self, parameter_s=''):
424 def autoreload(self, parameter_s=''):
406 r"""%autoreload => Reload modules automatically
425 r"""%autoreload => Reload modules automatically
407
426
408 %autoreload
427 %autoreload
409 Reload all modules (except those excluded by %aimport) automatically
428 Reload all modules (except those excluded by %aimport) automatically
410 now.
429 now.
411
430
412 %autoreload 0
431 %autoreload 0
413 Disable automatic reloading.
432 Disable automatic reloading.
414
433
415 %autoreload 1
434 %autoreload 1
416 Reload all modules imported with %aimport every time before executing
435 Reload all modules imported with %aimport every time before executing
417 the Python code typed.
436 the Python code typed.
418
437
419 %autoreload 2
438 %autoreload 2
420 Reload all modules (except those excluded by %aimport) every time
439 Reload all modules (except those excluded by %aimport) every time
421 before executing the Python code typed.
440 before executing the Python code typed.
422
441
423 Reloading Python modules in a reliable way is in general
442 Reloading Python modules in a reliable way is in general
424 difficult, and unexpected things may occur. %autoreload tries to
443 difficult, and unexpected things may occur. %autoreload tries to
425 work around common pitfalls by replacing function code objects and
444 work around common pitfalls by replacing function code objects and
426 parts of classes previously in the module with new versions. This
445 parts of classes previously in the module with new versions. This
427 makes the following things to work:
446 makes the following things to work:
428
447
429 - Functions and classes imported via 'from xxx import foo' are upgraded
448 - Functions and classes imported via 'from xxx import foo' are upgraded
430 to new versions when 'xxx' is reloaded.
449 to new versions when 'xxx' is reloaded.
431
450
432 - Methods and properties of classes are upgraded on reload, so that
451 - Methods and properties of classes are upgraded on reload, so that
433 calling 'c.foo()' on an object 'c' created before the reload causes
452 calling 'c.foo()' on an object 'c' created before the reload causes
434 the new code for 'foo' to be executed.
453 the new code for 'foo' to be executed.
435
454
436 Some of the known remaining caveats are:
455 Some of the known remaining caveats are:
437
456
438 - Replacing code objects does not always succeed: changing a @property
457 - Replacing code objects does not always succeed: changing a @property
439 in a class to an ordinary method or a method to a member variable
458 in a class to an ordinary method or a method to a member variable
440 can cause problems (but in old objects only).
459 can cause problems (but in old objects only).
441
460
442 - Functions that are removed (eg. via monkey-patching) from a module
461 - Functions that are removed (eg. via monkey-patching) from a module
443 before it is reloaded are not upgraded.
462 before it is reloaded are not upgraded.
444
463
445 - C extension modules cannot be reloaded, and so cannot be
464 - C extension modules cannot be reloaded, and so cannot be
446 autoreloaded.
465 autoreloaded.
447
466
448 """
467 """
449 if parameter_s == '':
468 if parameter_s == '':
450 self._reloader.check(True)
469 self._reloader.check(True)
451 elif parameter_s == '0':
470 elif parameter_s == '0':
452 self._reloader.enabled = False
471 self._reloader.enabled = False
453 elif parameter_s == '1':
472 elif parameter_s == '1':
454 self._reloader.check_all = False
473 self._reloader.check_all = False
455 self._reloader.enabled = True
474 self._reloader.enabled = True
456 elif parameter_s == '2':
475 elif parameter_s == '2':
457 self._reloader.check_all = True
476 self._reloader.check_all = True
458 self._reloader.enabled = True
477 self._reloader.enabled = True
459
478
460 @line_magic
479 @line_magic
461 def aimport(self, parameter_s='', stream=None):
480 def aimport(self, parameter_s='', stream=None):
462 """%aimport => Import modules for automatic reloading.
481 """%aimport => Import modules for automatic reloading.
463
482
464 %aimport
483 %aimport
465 List modules to automatically import and not to import.
484 List modules to automatically import and not to import.
466
485
467 %aimport foo
486 %aimport foo
468 Import module 'foo' and mark it to be autoreloaded for %autoreload 1
487 Import module 'foo' and mark it to be autoreloaded for %autoreload 1
469
488
470 %aimport -foo
489 %aimport -foo
471 Mark module 'foo' to not be autoreloaded for %autoreload 1
490 Mark module 'foo' to not be autoreloaded for %autoreload 1
472 """
491 """
473 modname = parameter_s
492 modname = parameter_s
474 if not modname:
493 if not modname:
475 to_reload = sorted(self._reloader.modules.keys())
494 to_reload = sorted(self._reloader.modules.keys())
476 to_skip = sorted(self._reloader.skip_modules.keys())
495 to_skip = sorted(self._reloader.skip_modules.keys())
477 if stream is None:
496 if stream is None:
478 stream = sys.stdout
497 stream = sys.stdout
479 if self._reloader.check_all:
498 if self._reloader.check_all:
480 stream.write("Modules to reload:\nall-except-skipped\n")
499 stream.write("Modules to reload:\nall-except-skipped\n")
481 else:
500 else:
482 stream.write("Modules to reload:\n%s\n" % ' '.join(to_reload))
501 stream.write("Modules to reload:\n%s\n" % ' '.join(to_reload))
483 stream.write("\nModules to skip:\n%s\n" % ' '.join(to_skip))
502 stream.write("\nModules to skip:\n%s\n" % ' '.join(to_skip))
484 elif modname.startswith('-'):
503 elif modname.startswith('-'):
485 modname = modname[1:]
504 modname = modname[1:]
486 self._reloader.mark_module_skipped(modname)
505 self._reloader.mark_module_skipped(modname)
487 else:
506 else:
488 top_module, top_name = self._reloader.aimport_module(modname)
507 top_module, top_name = self._reloader.aimport_module(modname)
489
508
490 # Inject module to user namespace
509 # Inject module to user namespace
491 self.shell.push({top_name: top_module})
510 self.shell.push({top_name: top_module})
492
511
493 def pre_run_cell(self):
512 def pre_run_cell(self):
494 if self._reloader.enabled:
513 if self._reloader.enabled:
495 try:
514 try:
496 self._reloader.check()
515 self._reloader.check()
497 except:
516 except:
498 pass
517 pass
499
518
519 def post_execute_hook(self):
520 """Cache the modification times of any modules imported in this execution
521 """
522 newly_loaded_modules = set(sys.modules) - self.loaded_modules
523 for modname in newly_loaded_modules:
524 _, pymtime = self._reloader.filename_and_mtime(sys.modules[modname])
525 if pymtime is not None:
526 self._reloader.modules_mtimes[modname] = pymtime
527
528 self.loaded_modules.update(newly_loaded_modules)
529
500
530
501 def load_ipython_extension(ip):
531 def load_ipython_extension(ip):
502 """Load the extension in IPython."""
532 """Load the extension in IPython."""
503 auto_reload = AutoreloadMagics(ip)
533 auto_reload = AutoreloadMagics(ip)
504 ip.register_magics(auto_reload)
534 ip.register_magics(auto_reload)
505 ip.events.register('pre_run_cell', auto_reload.pre_run_cell)
535 ip.events.register('pre_run_cell', auto_reload.pre_run_cell)
536 ip.events.register('post_execute', auto_reload.post_execute_hook)
@@ -1,321 +1,321 b''
1 """Tests for autoreload extension.
1 """Tests for autoreload extension.
2 """
2 """
3 #-----------------------------------------------------------------------------
3 #-----------------------------------------------------------------------------
4 # Copyright (c) 2012 IPython Development Team.
4 # Copyright (c) 2012 IPython Development Team.
5 #
5 #
6 # Distributed under the terms of the Modified BSD License.
6 # Distributed under the terms of the Modified BSD License.
7 #
7 #
8 # The full license is in the file COPYING.txt, distributed with this software.
8 # The full license is in the file COPYING.txt, distributed with this software.
9 #-----------------------------------------------------------------------------
9 #-----------------------------------------------------------------------------
10
10
11 #-----------------------------------------------------------------------------
11 #-----------------------------------------------------------------------------
12 # Imports
12 # Imports
13 #-----------------------------------------------------------------------------
13 #-----------------------------------------------------------------------------
14
14
15 import os
15 import os
16 import sys
16 import sys
17 import tempfile
17 import tempfile
18 import shutil
18 import shutil
19 import random
19 import random
20 import time
20 import time
21
21
22 import nose.tools as nt
22 import nose.tools as nt
23 import IPython.testing.tools as tt
23 import IPython.testing.tools as tt
24
24
25 from IPython.extensions.autoreload import AutoreloadMagics
25 from IPython.extensions.autoreload import AutoreloadMagics
26 from IPython.core.events import EventManager, pre_run_cell
26 from IPython.core.events import EventManager, pre_run_cell
27 from IPython.utils.py3compat import PY3
27 from IPython.utils.py3compat import PY3
28
28
29 if PY3:
29 if PY3:
30 from io import StringIO
30 from io import StringIO
31 else:
31 else:
32 from StringIO import StringIO
32 from StringIO import StringIO
33
33
34 #-----------------------------------------------------------------------------
34 #-----------------------------------------------------------------------------
35 # Test fixture
35 # Test fixture
36 #-----------------------------------------------------------------------------
36 #-----------------------------------------------------------------------------
37
37
38 noop = lambda *a, **kw: None
38 noop = lambda *a, **kw: None
39
39
40 class FakeShell(object):
40 class FakeShell(object):
41
41
42 def __init__(self):
42 def __init__(self):
43 self.ns = {}
43 self.ns = {}
44 self.events = EventManager(self, {'pre_run_cell', pre_run_cell})
44 self.events = EventManager(self, {'pre_run_cell', pre_run_cell})
45 self.auto_magics = AutoreloadMagics(shell=self)
45 self.auto_magics = AutoreloadMagics(shell=self)
46 self.events.register('pre_run_cell', self.auto_magics.pre_run_cell)
46 self.events.register('pre_run_cell', self.auto_magics.pre_run_cell)
47
47
48 register_magics = set_hook = noop
48 register_magics = set_hook = noop
49
49
50 def run_code(self, code):
50 def run_code(self, code):
51 self.events.trigger('pre_run_cell')
51 self.events.trigger('pre_run_cell')
52 exec(code, self.ns)
52 exec(code, self.ns)
53 self.auto_magics.post_execute_hook()
53
54
54 def push(self, items):
55 def push(self, items):
55 self.ns.update(items)
56 self.ns.update(items)
56
57
57 def magic_autoreload(self, parameter):
58 def magic_autoreload(self, parameter):
58 self.auto_magics.autoreload(parameter)
59 self.auto_magics.autoreload(parameter)
59
60
60 def magic_aimport(self, parameter, stream=None):
61 def magic_aimport(self, parameter, stream=None):
61 self.auto_magics.aimport(parameter, stream=stream)
62 self.auto_magics.aimport(parameter, stream=stream)
63 self.auto_magics.post_execute_hook()
62
64
63
65
64 class Fixture(object):
66 class Fixture(object):
65 """Fixture for creating test module files"""
67 """Fixture for creating test module files"""
66
68
67 test_dir = None
69 test_dir = None
68 old_sys_path = None
70 old_sys_path = None
69 filename_chars = "abcdefghijklmopqrstuvwxyz0123456789"
71 filename_chars = "abcdefghijklmopqrstuvwxyz0123456789"
70
72
71 def setUp(self):
73 def setUp(self):
72 self.test_dir = tempfile.mkdtemp()
74 self.test_dir = tempfile.mkdtemp()
73 self.old_sys_path = list(sys.path)
75 self.old_sys_path = list(sys.path)
74 sys.path.insert(0, self.test_dir)
76 sys.path.insert(0, self.test_dir)
75 self.shell = FakeShell()
77 self.shell = FakeShell()
76
78
77 def tearDown(self):
79 def tearDown(self):
78 shutil.rmtree(self.test_dir)
80 shutil.rmtree(self.test_dir)
79 sys.path = self.old_sys_path
81 sys.path = self.old_sys_path
80
82
81 self.test_dir = None
83 self.test_dir = None
82 self.old_sys_path = None
84 self.old_sys_path = None
83 self.shell = None
85 self.shell = None
84
86
85 def get_module(self):
87 def get_module(self):
86 module_name = "tmpmod_" + "".join(random.sample(self.filename_chars,20))
88 module_name = "tmpmod_" + "".join(random.sample(self.filename_chars,20))
87 if module_name in sys.modules:
89 if module_name in sys.modules:
88 del sys.modules[module_name]
90 del sys.modules[module_name]
89 file_name = os.path.join(self.test_dir, module_name + ".py")
91 file_name = os.path.join(self.test_dir, module_name + ".py")
90 return module_name, file_name
92 return module_name, file_name
91
93
92 def write_file(self, filename, content):
94 def write_file(self, filename, content):
93 """
95 """
94 Write a file, and force a timestamp difference of at least one second
96 Write a file, and force a timestamp difference of at least one second
95
97
96 Notes
98 Notes
97 -----
99 -----
98 Python's .pyc files record the timestamp of their compilation
100 Python's .pyc files record the timestamp of their compilation
99 with a time resolution of one second.
101 with a time resolution of one second.
100
102
101 Therefore, we need to force a timestamp difference between .py
103 Therefore, we need to force a timestamp difference between .py
102 and .pyc, without having the .py file be timestamped in the
104 and .pyc, without having the .py file be timestamped in the
103 future, and without changing the timestamp of the .pyc file
105 future, and without changing the timestamp of the .pyc file
104 (because that is stored in the file). The only reliable way
106 (because that is stored in the file). The only reliable way
105 to achieve this seems to be to sleep.
107 to achieve this seems to be to sleep.
106 """
108 """
107
109
108 # Sleep one second + eps
110 # Sleep one second + eps
109 time.sleep(1.05)
111 time.sleep(1.05)
110
112
111 # Write
113 # Write
112 f = open(filename, 'w')
114 f = open(filename, 'w')
113 try:
115 try:
114 f.write(content)
116 f.write(content)
115 finally:
117 finally:
116 f.close()
118 f.close()
117
119
118 def new_module(self, code):
120 def new_module(self, code):
119 mod_name, mod_fn = self.get_module()
121 mod_name, mod_fn = self.get_module()
120 f = open(mod_fn, 'w')
122 f = open(mod_fn, 'w')
121 try:
123 try:
122 f.write(code)
124 f.write(code)
123 finally:
125 finally:
124 f.close()
126 f.close()
125 return mod_name, mod_fn
127 return mod_name, mod_fn
126
128
127 #-----------------------------------------------------------------------------
129 #-----------------------------------------------------------------------------
128 # Test automatic reloading
130 # Test automatic reloading
129 #-----------------------------------------------------------------------------
131 #-----------------------------------------------------------------------------
130
132
131 class TestAutoreload(Fixture):
133 class TestAutoreload(Fixture):
132 def _check_smoketest(self, use_aimport=True):
134 def _check_smoketest(self, use_aimport=True):
133 """
135 """
134 Functional test for the automatic reloader using either
136 Functional test for the automatic reloader using either
135 '%autoreload 1' or '%autoreload 2'
137 '%autoreload 1' or '%autoreload 2'
136 """
138 """
137
139
138 mod_name, mod_fn = self.new_module("""
140 mod_name, mod_fn = self.new_module("""
139 x = 9
141 x = 9
140
142
141 z = 123 # this item will be deleted
143 z = 123 # this item will be deleted
142
144
143 def foo(y):
145 def foo(y):
144 return y + 3
146 return y + 3
145
147
146 class Baz(object):
148 class Baz(object):
147 def __init__(self, x):
149 def __init__(self, x):
148 self.x = x
150 self.x = x
149 def bar(self, y):
151 def bar(self, y):
150 return self.x + y
152 return self.x + y
151 @property
153 @property
152 def quux(self):
154 def quux(self):
153 return 42
155 return 42
154 def zzz(self):
156 def zzz(self):
155 '''This method will be deleted below'''
157 '''This method will be deleted below'''
156 return 99
158 return 99
157
159
158 class Bar: # old-style class: weakref doesn't work for it on Python < 2.7
160 class Bar: # old-style class: weakref doesn't work for it on Python < 2.7
159 def foo(self):
161 def foo(self):
160 return 1
162 return 1
161 """)
163 """)
162
164
163 #
165 #
164 # Import module, and mark for reloading
166 # Import module, and mark for reloading
165 #
167 #
166 if use_aimport:
168 if use_aimport:
167 self.shell.magic_autoreload("1")
169 self.shell.magic_autoreload("1")
168 self.shell.magic_aimport(mod_name)
170 self.shell.magic_aimport(mod_name)
169 stream = StringIO()
171 stream = StringIO()
170 self.shell.magic_aimport("", stream=stream)
172 self.shell.magic_aimport("", stream=stream)
171 nt.assert_true(("Modules to reload:\n%s" % mod_name) in
173 nt.assert_in(("Modules to reload:\n%s" % mod_name), stream.getvalue())
172 stream.getvalue())
173
174
174 nt.assert_raises(
175 with nt.assert_raises(ImportError):
175 ImportError,
176 self.shell.magic_aimport("tmpmod_as318989e89ds")
176 self.shell.magic_aimport, "tmpmod_as318989e89ds")
177 else:
177 else:
178 self.shell.magic_autoreload("2")
178 self.shell.magic_autoreload("2")
179 self.shell.run_code("import %s" % mod_name)
179 self.shell.run_code("import %s" % mod_name)
180 stream = StringIO()
180 stream = StringIO()
181 self.shell.magic_aimport("", stream=stream)
181 self.shell.magic_aimport("", stream=stream)
182 nt.assert_true("Modules to reload:\nall-except-skipped" in
182 nt.assert_true("Modules to reload:\nall-except-skipped" in
183 stream.getvalue())
183 stream.getvalue())
184 nt.assert_in(mod_name, self.shell.ns)
184 nt.assert_in(mod_name, self.shell.ns)
185
185
186 mod = sys.modules[mod_name]
186 mod = sys.modules[mod_name]
187
187
188 #
188 #
189 # Test module contents
189 # Test module contents
190 #
190 #
191 old_foo = mod.foo
191 old_foo = mod.foo
192 old_obj = mod.Baz(9)
192 old_obj = mod.Baz(9)
193 old_obj2 = mod.Bar()
193 old_obj2 = mod.Bar()
194
194
195 def check_module_contents():
195 def check_module_contents():
196 nt.assert_equal(mod.x, 9)
196 nt.assert_equal(mod.x, 9)
197 nt.assert_equal(mod.z, 123)
197 nt.assert_equal(mod.z, 123)
198
198
199 nt.assert_equal(old_foo(0), 3)
199 nt.assert_equal(old_foo(0), 3)
200 nt.assert_equal(mod.foo(0), 3)
200 nt.assert_equal(mod.foo(0), 3)
201
201
202 obj = mod.Baz(9)
202 obj = mod.Baz(9)
203 nt.assert_equal(old_obj.bar(1), 10)
203 nt.assert_equal(old_obj.bar(1), 10)
204 nt.assert_equal(obj.bar(1), 10)
204 nt.assert_equal(obj.bar(1), 10)
205 nt.assert_equal(obj.quux, 42)
205 nt.assert_equal(obj.quux, 42)
206 nt.assert_equal(obj.zzz(), 99)
206 nt.assert_equal(obj.zzz(), 99)
207
207
208 obj2 = mod.Bar()
208 obj2 = mod.Bar()
209 nt.assert_equal(old_obj2.foo(), 1)
209 nt.assert_equal(old_obj2.foo(), 1)
210 nt.assert_equal(obj2.foo(), 1)
210 nt.assert_equal(obj2.foo(), 1)
211
211
212 check_module_contents()
212 check_module_contents()
213
213
214 #
214 #
215 # Simulate a failed reload: no reload should occur and exactly
215 # Simulate a failed reload: no reload should occur and exactly
216 # one error message should be printed
216 # one error message should be printed
217 #
217 #
218 self.write_file(mod_fn, """
218 self.write_file(mod_fn, """
219 a syntax error
219 a syntax error
220 """)
220 """)
221
221
222 with tt.AssertPrints(('[autoreload of %s failed:' % mod_name), channel='stderr'):
222 with tt.AssertPrints(('[autoreload of %s failed:' % mod_name), channel='stderr'):
223 self.shell.run_code("pass") # trigger reload
223 self.shell.run_code("pass") # trigger reload
224 with tt.AssertNotPrints(('[autoreload of %s failed:' % mod_name), channel='stderr'):
224 with tt.AssertNotPrints(('[autoreload of %s failed:' % mod_name), channel='stderr'):
225 self.shell.run_code("pass") # trigger another reload
225 self.shell.run_code("pass") # trigger another reload
226 check_module_contents()
226 check_module_contents()
227
227
228 #
228 #
229 # Rewrite module (this time reload should succeed)
229 # Rewrite module (this time reload should succeed)
230 #
230 #
231 self.write_file(mod_fn, """
231 self.write_file(mod_fn, """
232 x = 10
232 x = 10
233
233
234 def foo(y):
234 def foo(y):
235 return y + 4
235 return y + 4
236
236
237 class Baz(object):
237 class Baz(object):
238 def __init__(self, x):
238 def __init__(self, x):
239 self.x = x
239 self.x = x
240 def bar(self, y):
240 def bar(self, y):
241 return self.x + y + 1
241 return self.x + y + 1
242 @property
242 @property
243 def quux(self):
243 def quux(self):
244 return 43
244 return 43
245
245
246 class Bar: # old-style class
246 class Bar: # old-style class
247 def foo(self):
247 def foo(self):
248 return 2
248 return 2
249 """)
249 """)
250
250
251 def check_module_contents():
251 def check_module_contents():
252 nt.assert_equal(mod.x, 10)
252 nt.assert_equal(mod.x, 10)
253 nt.assert_false(hasattr(mod, 'z'))
253 nt.assert_false(hasattr(mod, 'z'))
254
254
255 nt.assert_equal(old_foo(0), 4) # superreload magic!
255 nt.assert_equal(old_foo(0), 4) # superreload magic!
256 nt.assert_equal(mod.foo(0), 4)
256 nt.assert_equal(mod.foo(0), 4)
257
257
258 obj = mod.Baz(9)
258 obj = mod.Baz(9)
259 nt.assert_equal(old_obj.bar(1), 11) # superreload magic!
259 nt.assert_equal(old_obj.bar(1), 11) # superreload magic!
260 nt.assert_equal(obj.bar(1), 11)
260 nt.assert_equal(obj.bar(1), 11)
261
261
262 nt.assert_equal(old_obj.quux, 43)
262 nt.assert_equal(old_obj.quux, 43)
263 nt.assert_equal(obj.quux, 43)
263 nt.assert_equal(obj.quux, 43)
264
264
265 nt.assert_false(hasattr(old_obj, 'zzz'))
265 nt.assert_false(hasattr(old_obj, 'zzz'))
266 nt.assert_false(hasattr(obj, 'zzz'))
266 nt.assert_false(hasattr(obj, 'zzz'))
267
267
268 obj2 = mod.Bar()
268 obj2 = mod.Bar()
269 nt.assert_equal(old_obj2.foo(), 2)
269 nt.assert_equal(old_obj2.foo(), 2)
270 nt.assert_equal(obj2.foo(), 2)
270 nt.assert_equal(obj2.foo(), 2)
271
271
272 self.shell.run_code("pass") # trigger reload
272 self.shell.run_code("pass") # trigger reload
273 check_module_contents()
273 check_module_contents()
274
274
275 #
275 #
276 # Another failure case: deleted file (shouldn't reload)
276 # Another failure case: deleted file (shouldn't reload)
277 #
277 #
278 os.unlink(mod_fn)
278 os.unlink(mod_fn)
279
279
280 self.shell.run_code("pass") # trigger reload
280 self.shell.run_code("pass") # trigger reload
281 check_module_contents()
281 check_module_contents()
282
282
283 #
283 #
284 # Disable autoreload and rewrite module: no reload should occur
284 # Disable autoreload and rewrite module: no reload should occur
285 #
285 #
286 if use_aimport:
286 if use_aimport:
287 self.shell.magic_aimport("-" + mod_name)
287 self.shell.magic_aimport("-" + mod_name)
288 stream = StringIO()
288 stream = StringIO()
289 self.shell.magic_aimport("", stream=stream)
289 self.shell.magic_aimport("", stream=stream)
290 nt.assert_true(("Modules to skip:\n%s" % mod_name) in
290 nt.assert_true(("Modules to skip:\n%s" % mod_name) in
291 stream.getvalue())
291 stream.getvalue())
292
292
293 # This should succeed, although no such module exists
293 # This should succeed, although no such module exists
294 self.shell.magic_aimport("-tmpmod_as318989e89ds")
294 self.shell.magic_aimport("-tmpmod_as318989e89ds")
295 else:
295 else:
296 self.shell.magic_autoreload("0")
296 self.shell.magic_autoreload("0")
297
297
298 self.write_file(mod_fn, """
298 self.write_file(mod_fn, """
299 x = -99
299 x = -99
300 """)
300 """)
301
301
302 self.shell.run_code("pass") # trigger reload
302 self.shell.run_code("pass") # trigger reload
303 self.shell.run_code("pass")
303 self.shell.run_code("pass")
304 check_module_contents()
304 check_module_contents()
305
305
306 #
306 #
307 # Re-enable autoreload: reload should now occur
307 # Re-enable autoreload: reload should now occur
308 #
308 #
309 if use_aimport:
309 if use_aimport:
310 self.shell.magic_aimport(mod_name)
310 self.shell.magic_aimport(mod_name)
311 else:
311 else:
312 self.shell.magic_autoreload("")
312 self.shell.magic_autoreload("")
313
313
314 self.shell.run_code("pass") # trigger reload
314 self.shell.run_code("pass") # trigger reload
315 nt.assert_equal(mod.x, -99)
315 nt.assert_equal(mod.x, -99)
316
316
317 def test_smoketest_aimport(self):
317 def test_smoketest_aimport(self):
318 self._check_smoketest(use_aimport=True)
318 self._check_smoketest(use_aimport=True)
319
319
320 def test_smoketest_autoreload(self):
320 def test_smoketest_autoreload(self):
321 self._check_smoketest(use_aimport=False)
321 self._check_smoketest(use_aimport=False)
General Comments 0
You need to be logged in to leave comments. Login now