##// END OF EJS Templates
Several fixes to autoreload extension for new API....
Fernando Perez -
Show More
@@ -1,538 +1,538 b''
1 """IPython extension to reload modules before executing user code.
1 """IPython extension to reload modules before executing user code.
2
2
3 ``autoreload`` reloads modules automatically before entering the execution of
3 ``autoreload`` reloads modules automatically before entering the execution of
4 code typed at the IPython prompt.
4 code typed at the IPython prompt.
5
5
6 This makes for example the following workflow possible:
6 This makes for example the following workflow possible:
7
7
8 .. sourcecode:: ipython
8 .. sourcecode:: ipython
9
9
10 In [1]: %load_ext autoreload
10 In [1]: %load_ext autoreload
11
11
12 In [2]: %autoreload 2
12 In [2]: %autoreload 2
13
13
14 In [3]: from foo import some_function
14 In [3]: from foo import some_function
15
15
16 In [4]: some_function()
16 In [4]: some_function()
17 Out[4]: 42
17 Out[4]: 42
18
18
19 In [5]: # open foo.py in an editor and change some_function to return 43
19 In [5]: # open foo.py in an editor and change some_function to return 43
20
20
21 In [6]: some_function()
21 In [6]: some_function()
22 Out[6]: 43
22 Out[6]: 43
23
23
24 The module was reloaded without reloading it explicitly, and the object
24 The module was reloaded without reloading it explicitly, and the object
25 imported with ``from foo import ...`` was also updated.
25 imported with ``from foo import ...`` was also updated.
26
26
27 Usage
27 Usage
28 =====
28 =====
29
29
30 The following magic commands are provided:
30 The following magic commands are provided:
31
31
32 ``%autoreload``
32 ``%autoreload``
33
33
34 Reload all modules (except those excluded by ``%aimport``)
34 Reload all modules (except those excluded by ``%aimport``)
35 automatically now.
35 automatically now.
36
36
37 ``%autoreload 0``
37 ``%autoreload 0``
38
38
39 Disable automatic reloading.
39 Disable automatic reloading.
40
40
41 ``%autoreload 1``
41 ``%autoreload 1``
42
42
43 Reload all modules imported with ``%aimport`` every time before
43 Reload all modules imported with ``%aimport`` every time before
44 executing the Python code typed.
44 executing the Python code typed.
45
45
46 ``%autoreload 2``
46 ``%autoreload 2``
47
47
48 Reload all modules (except those excluded by ``%aimport``) every
48 Reload all modules (except those excluded by ``%aimport``) every
49 time before executing the Python code typed.
49 time before executing the Python code typed.
50
50
51 ``%aimport``
51 ``%aimport``
52
52
53 List modules which are to be automatically imported or not to be imported.
53 List modules which are to be automatically imported or not to be imported.
54
54
55 ``%aimport foo``
55 ``%aimport foo``
56
56
57 Import module 'foo' and mark it to be autoreloaded for ``%autoreload 1``
57 Import module 'foo' and mark it to be autoreloaded for ``%autoreload 1``
58
58
59 ``%aimport -foo``
59 ``%aimport -foo``
60
60
61 Mark module 'foo' to not be autoreloaded.
61 Mark module 'foo' to not be autoreloaded.
62
62
63 Caveats
63 Caveats
64 =======
64 =======
65
65
66 Reloading Python modules in a reliable way is in general difficult,
66 Reloading Python modules in a reliable way is in general difficult,
67 and unexpected things may occur. ``%autoreload`` tries to work around
67 and unexpected things may occur. ``%autoreload`` tries to work around
68 common pitfalls by replacing function code objects and parts of
68 common pitfalls by replacing function code objects and parts of
69 classes previously in the module with new versions. This makes the
69 classes previously in the module with new versions. This makes the
70 following things to work:
70 following things to work:
71
71
72 - Functions and classes imported via 'from xxx import foo' are upgraded
72 - Functions and classes imported via 'from xxx import foo' are upgraded
73 to new versions when 'xxx' is reloaded.
73 to new versions when 'xxx' is reloaded.
74
74
75 - Methods and properties of classes are upgraded on reload, so that
75 - Methods and properties of classes are upgraded on reload, so that
76 calling 'c.foo()' on an object 'c' created before the reload causes
76 calling 'c.foo()' on an object 'c' created before the reload causes
77 the new code for 'foo' to be executed.
77 the new code for 'foo' to be executed.
78
78
79 Some of the known remaining caveats are:
79 Some of the known remaining caveats are:
80
80
81 - Replacing code objects does not always succeed: changing a @property
81 - Replacing code objects does not always succeed: changing a @property
82 in a class to an ordinary method or a method to a member variable
82 in a class to an ordinary method or a method to a member variable
83 can cause problems (but in old objects only).
83 can cause problems (but in old objects only).
84
84
85 - Functions that are removed (eg. via monkey-patching) from a module
85 - Functions that are removed (eg. via monkey-patching) from a module
86 before it is reloaded are not upgraded.
86 before it is reloaded are not upgraded.
87
87
88 - C extension modules cannot be reloaded, and so cannot be autoreloaded.
88 - C extension modules cannot be reloaded, and so cannot be autoreloaded.
89 """
89 """
90
90
91 skip_doctest = True
91 skip_doctest = True
92
92
93 #-----------------------------------------------------------------------------
93 #-----------------------------------------------------------------------------
94 # Copyright (C) 2000 Thomas Heller
94 # Copyright (C) 2000 Thomas Heller
95 # Copyright (C) 2008 Pauli Virtanen <pav@iki.fi>
95 # Copyright (C) 2008 Pauli Virtanen <pav@iki.fi>
96 # Copyright (C) 2012 The IPython Development Team
96 # Copyright (C) 2012 The IPython Development Team
97 #
97 #
98 # Distributed under the terms of the BSD License. The full license is in
98 # Distributed under the terms of the BSD License. The full license is in
99 # the file COPYING, distributed as part of this software.
99 # the file COPYING, distributed as part of this software.
100 #-----------------------------------------------------------------------------
100 #-----------------------------------------------------------------------------
101 #
101 #
102 # This IPython module is written by Pauli Virtanen, based on the autoreload
102 # This IPython module is written by Pauli Virtanen, based on the autoreload
103 # code by Thomas Heller.
103 # code by Thomas Heller.
104
104
105 #-----------------------------------------------------------------------------
105 #-----------------------------------------------------------------------------
106 # Imports
106 # Imports
107 #-----------------------------------------------------------------------------
107 #-----------------------------------------------------------------------------
108 import atexit
108 import atexit
109 import imp
109 import imp
110 import inspect
110 import inspect
111 import os
111 import os
112 import sys
112 import sys
113 import threading
113 import threading
114 import time
114 import time
115 import traceback
115 import traceback
116 import types
116 import types
117 import weakref
117 import weakref
118
118
119 try:
119 try:
120 # Reload is not defined by default in Python3.
120 # Reload is not defined by default in Python3.
121 reload
121 reload
122 except NameError:
122 except NameError:
123 from imp import reload
123 from imp import reload
124
124
125 from IPython.utils import pyfile
125 from IPython.utils import pyfile
126 from IPython.utils.py3compat import PY3
126 from IPython.utils.py3compat import PY3
127
127
128 #------------------------------------------------------------------------------
128 #------------------------------------------------------------------------------
129 # Autoreload functionality
129 # Autoreload functionality
130 #------------------------------------------------------------------------------
130 #------------------------------------------------------------------------------
131
131
132 def _get_compiled_ext():
132 def _get_compiled_ext():
133 """Official way to get the extension of compiled files (.pyc or .pyo)"""
133 """Official way to get the extension of compiled files (.pyc or .pyo)"""
134 for ext, mode, typ in imp.get_suffixes():
134 for ext, mode, typ in imp.get_suffixes():
135 if typ == imp.PY_COMPILED:
135 if typ == imp.PY_COMPILED:
136 return ext
136 return ext
137
137
138
138
139 PY_COMPILED_EXT = _get_compiled_ext()
139 PY_COMPILED_EXT = _get_compiled_ext()
140
140
141
141
142 class ModuleReloader(object):
142 class ModuleReloader(object):
143 enabled = False
143 enabled = False
144 """Whether this reloader is enabled"""
144 """Whether this reloader is enabled"""
145
145
146 failed = {}
146 failed = {}
147 """Modules that failed to reload: {module: mtime-on-failed-reload, ...}"""
147 """Modules that failed to reload: {module: mtime-on-failed-reload, ...}"""
148
148
149 modules = {}
149 modules = {}
150 """Modules specially marked as autoreloadable."""
150 """Modules specially marked as autoreloadable."""
151
151
152 skip_modules = {}
152 skip_modules = {}
153 """Modules specially marked as not autoreloadable."""
153 """Modules specially marked as not autoreloadable."""
154
154
155 check_all = True
155 check_all = True
156 """Autoreload all modules, not just those listed in 'modules'"""
156 """Autoreload all modules, not just those listed in 'modules'"""
157
157
158 old_objects = {}
158 old_objects = {}
159 """(module-name, name) -> weakref, for replacing old code objects"""
159 """(module-name, name) -> weakref, for replacing old code objects"""
160
160
161 def mark_module_skipped(self, module_name):
161 def mark_module_skipped(self, module_name):
162 """Skip reloading the named module in the future"""
162 """Skip reloading the named module in the future"""
163 try:
163 try:
164 del self.modules[module_name]
164 del self.modules[module_name]
165 except KeyError:
165 except KeyError:
166 pass
166 pass
167 self.skip_modules[module_name] = True
167 self.skip_modules[module_name] = True
168
168
169 def mark_module_reloadable(self, module_name):
169 def mark_module_reloadable(self, module_name):
170 """Reload the named module in the future (if it is imported)"""
170 """Reload the named module in the future (if it is imported)"""
171 try:
171 try:
172 del self.skip_modules[module_name]
172 del self.skip_modules[module_name]
173 except KeyError:
173 except KeyError:
174 pass
174 pass
175 self.modules[module_name] = True
175 self.modules[module_name] = True
176
176
177 def aimport_module(self, module_name):
177 def aimport_module(self, module_name):
178 """Import a module, and mark it reloadable
178 """Import a module, and mark it reloadable
179
179
180 Returns
180 Returns
181 -------
181 -------
182 top_module : module
182 top_module : module
183 The imported module if it is top-level, or the top-level
183 The imported module if it is top-level, or the top-level
184 top_name : module
184 top_name : module
185 Name of top_module
185 Name of top_module
186
186
187 """
187 """
188 self.mark_module_reloadable(module_name)
188 self.mark_module_reloadable(module_name)
189
189
190 __import__(module_name)
190 __import__(module_name)
191 top_name = module_name.split('.')[0]
191 top_name = module_name.split('.')[0]
192 top_module = sys.modules[top_name]
192 top_module = sys.modules[top_name]
193 return top_module, top_name
193 return top_module, top_name
194
194
195 def check(self, check_all=False):
195 def check(self, check_all=False):
196 """Check whether some modules need to be reloaded."""
196 """Check whether some modules need to be reloaded."""
197
197
198 if not self.enabled and not check_all:
198 if not self.enabled and not check_all:
199 return
199 return
200
200
201 if check_all or self.check_all:
201 if check_all or self.check_all:
202 modules = sys.modules.keys()
202 modules = sys.modules.keys()
203 else:
203 else:
204 modules = self.modules.keys()
204 modules = self.modules.keys()
205
205
206 for modname in modules:
206 for modname in modules:
207 m = sys.modules.get(modname, None)
207 m = sys.modules.get(modname, None)
208
208
209 if modname in self.skip_modules:
209 if modname in self.skip_modules:
210 continue
210 continue
211
211
212 if not hasattr(m, '__file__'):
212 if not hasattr(m, '__file__'):
213 continue
213 continue
214
214
215 if m.__name__ == '__main__':
215 if m.__name__ == '__main__':
216 # we cannot reload(__main__)
216 # we cannot reload(__main__)
217 continue
217 continue
218
218
219 filename = m.__file__
219 filename = m.__file__
220 path, ext = os.path.splitext(filename)
220 path, ext = os.path.splitext(filename)
221
221
222 if ext.lower() == '.py':
222 if ext.lower() == '.py':
223 ext = PY_COMPILED_EXT
223 ext = PY_COMPILED_EXT
224 pyc_filename = pyfile.cache_from_source(filename)
224 pyc_filename = pyfile.cache_from_source(filename)
225 py_filename = filename
225 py_filename = filename
226 else:
226 else:
227 pyc_filename = filename
227 pyc_filename = filename
228 try:
228 try:
229 py_filename = pyfile.source_from_cache(filename)
229 py_filename = pyfile.source_from_cache(filename)
230 except ValueError:
230 except ValueError:
231 continue
231 continue
232
232
233 try:
233 try:
234 pymtime = os.stat(py_filename).st_mtime
234 pymtime = os.stat(py_filename).st_mtime
235 if pymtime <= os.stat(pyc_filename).st_mtime:
235 if pymtime <= os.stat(pyc_filename).st_mtime:
236 continue
236 continue
237 if self.failed.get(py_filename, None) == pymtime:
237 if self.failed.get(py_filename, None) == pymtime:
238 continue
238 continue
239 except OSError:
239 except OSError:
240 continue
240 continue
241
241
242 try:
242 try:
243 superreload(m, reload, self.old_objects)
243 superreload(m, reload, self.old_objects)
244 if py_filename in self.failed:
244 if py_filename in self.failed:
245 del self.failed[py_filename]
245 del self.failed[py_filename]
246 except:
246 except:
247 print >> sys.stderr, "[autoreload of %s failed: %s]" % (
247 print >> sys.stderr, "[autoreload of %s failed: %s]" % (
248 modname, traceback.format_exc(1))
248 modname, traceback.format_exc(1))
249 self.failed[py_filename] = pymtime
249 self.failed[py_filename] = pymtime
250
250
251 #------------------------------------------------------------------------------
251 #------------------------------------------------------------------------------
252 # superreload
252 # superreload
253 #------------------------------------------------------------------------------
253 #------------------------------------------------------------------------------
254
254
255 if PY3:
255 if PY3:
256 func_attrs = ['__code__', '__defaults__', '__doc__',
256 func_attrs = ['__code__', '__defaults__', '__doc__',
257 '__closure__', '__globals__', '__dict__']
257 '__closure__', '__globals__', '__dict__']
258 else:
258 else:
259 func_attrs = ['func_code', 'func_defaults', 'func_doc',
259 func_attrs = ['func_code', 'func_defaults', 'func_doc',
260 'func_closure', 'func_globals', 'func_dict']
260 'func_closure', 'func_globals', 'func_dict']
261
261
262
262
263 def update_function(old, new):
263 def update_function(old, new):
264 """Upgrade the code object of a function"""
264 """Upgrade the code object of a function"""
265 for name in func_attrs:
265 for name in func_attrs:
266 try:
266 try:
267 setattr(old, name, getattr(new, name))
267 setattr(old, name, getattr(new, name))
268 except (AttributeError, TypeError):
268 except (AttributeError, TypeError):
269 pass
269 pass
270
270
271
271
272 def update_class(old, new):
272 def update_class(old, new):
273 """Replace stuff in the __dict__ of a class, and upgrade
273 """Replace stuff in the __dict__ of a class, and upgrade
274 method code objects"""
274 method code objects"""
275 for key in old.__dict__.keys():
275 for key in old.__dict__.keys():
276 old_obj = getattr(old, key)
276 old_obj = getattr(old, key)
277
277
278 try:
278 try:
279 new_obj = getattr(new, key)
279 new_obj = getattr(new, key)
280 except AttributeError:
280 except AttributeError:
281 # obsolete attribute: remove it
281 # obsolete attribute: remove it
282 try:
282 try:
283 delattr(old, key)
283 delattr(old, key)
284 except (AttributeError, TypeError):
284 except (AttributeError, TypeError):
285 pass
285 pass
286 continue
286 continue
287
287
288 if update_generic(old_obj, new_obj): continue
288 if update_generic(old_obj, new_obj): continue
289
289
290 try:
290 try:
291 setattr(old, key, getattr(new, key))
291 setattr(old, key, getattr(new, key))
292 except (AttributeError, TypeError):
292 except (AttributeError, TypeError):
293 pass # skip non-writable attributes
293 pass # skip non-writable attributes
294
294
295
295
296 def update_property(old, new):
296 def update_property(old, new):
297 """Replace get/set/del functions of a property"""
297 """Replace get/set/del functions of a property"""
298 update_generic(old.fdel, new.fdel)
298 update_generic(old.fdel, new.fdel)
299 update_generic(old.fget, new.fget)
299 update_generic(old.fget, new.fget)
300 update_generic(old.fset, new.fset)
300 update_generic(old.fset, new.fset)
301
301
302
302
303 def isinstance2(a, b, typ):
303 def isinstance2(a, b, typ):
304 return isinstance(a, typ) and isinstance(b, typ)
304 return isinstance(a, typ) and isinstance(b, typ)
305
305
306
306
307 UPDATE_RULES = [
307 UPDATE_RULES = [
308 (lambda a, b: isinstance2(a, b, type),
308 (lambda a, b: isinstance2(a, b, type),
309 update_class),
309 update_class),
310 (lambda a, b: isinstance2(a, b, types.FunctionType),
310 (lambda a, b: isinstance2(a, b, types.FunctionType),
311 update_function),
311 update_function),
312 (lambda a, b: isinstance2(a, b, property),
312 (lambda a, b: isinstance2(a, b, property),
313 update_property),
313 update_property),
314 ]
314 ]
315
315
316
316
317 if PY3:
317 if PY3:
318 UPDATE_RULES.extend([(lambda a, b: isinstance2(a, b, types.MethodType),
318 UPDATE_RULES.extend([(lambda a, b: isinstance2(a, b, types.MethodType),
319 lambda a, b: update_function(a.__func__, b.__func__)),
319 lambda a, b: update_function(a.__func__, b.__func__)),
320 ])
320 ])
321 else:
321 else:
322 UPDATE_RULES.extend([(lambda a, b: isinstance2(a, b, types.ClassType),
322 UPDATE_RULES.extend([(lambda a, b: isinstance2(a, b, types.ClassType),
323 update_class),
323 update_class),
324 (lambda a, b: isinstance2(a, b, types.MethodType),
324 (lambda a, b: isinstance2(a, b, types.MethodType),
325 lambda a, b: update_function(a.im_func, b.im_func)),
325 lambda a, b: update_function(a.im_func, b.im_func)),
326 ])
326 ])
327
327
328
328
329 def update_generic(a, b):
329 def update_generic(a, b):
330 for type_check, update in UPDATE_RULES:
330 for type_check, update in UPDATE_RULES:
331 if type_check(a, b):
331 if type_check(a, b):
332 update(a, b)
332 update(a, b)
333 return True
333 return True
334 return False
334 return False
335
335
336
336
337 class StrongRef(object):
337 class StrongRef(object):
338 def __init__(self, obj):
338 def __init__(self, obj):
339 self.obj = obj
339 self.obj = obj
340 def __call__(self):
340 def __call__(self):
341 return self.obj
341 return self.obj
342
342
343
343
344 def superreload(module, reload=reload, old_objects={}):
344 def superreload(module, reload=reload, old_objects={}):
345 """Enhanced version of the builtin reload function.
345 """Enhanced version of the builtin reload function.
346
346
347 superreload remembers objects previously in the module, and
347 superreload remembers objects previously in the module, and
348
348
349 - upgrades the class dictionary of every old class in the module
349 - upgrades the class dictionary of every old class in the module
350 - upgrades the code object of every old function and method
350 - upgrades the code object of every old function and method
351 - clears the module's namespace before reloading
351 - clears the module's namespace before reloading
352
352
353 """
353 """
354
354
355 # collect old objects in the module
355 # collect old objects in the module
356 for name, obj in module.__dict__.items():
356 for name, obj in module.__dict__.items():
357 if not hasattr(obj, '__module__') or obj.__module__ != module.__name__:
357 if not hasattr(obj, '__module__') or obj.__module__ != module.__name__:
358 continue
358 continue
359 key = (module.__name__, name)
359 key = (module.__name__, name)
360 try:
360 try:
361 old_objects.setdefault(key, []).append(weakref.ref(obj))
361 old_objects.setdefault(key, []).append(weakref.ref(obj))
362 except TypeError:
362 except TypeError:
363 # weakref doesn't work for all types;
363 # weakref doesn't work for all types;
364 # create strong references for 'important' cases
364 # create strong references for 'important' cases
365 if not PY3 and isinstance(obj, types.ClassType):
365 if not PY3 and isinstance(obj, types.ClassType):
366 old_objects.setdefault(key, []).append(StrongRef(obj))
366 old_objects.setdefault(key, []).append(StrongRef(obj))
367
367
368 # reload module
368 # reload module
369 try:
369 try:
370 # clear namespace first from old cruft
370 # clear namespace first from old cruft
371 old_dict = module.__dict__.copy()
371 old_dict = module.__dict__.copy()
372 old_name = module.__name__
372 old_name = module.__name__
373 module.__dict__.clear()
373 module.__dict__.clear()
374 module.__dict__['__name__'] = old_name
374 module.__dict__['__name__'] = old_name
375 except (TypeError, AttributeError, KeyError):
375 except (TypeError, AttributeError, KeyError):
376 pass
376 pass
377
377
378 try:
378 try:
379 module = reload(module)
379 module = reload(module)
380 except:
380 except:
381 # restore module dictionary on failed reload
381 # restore module dictionary on failed reload
382 module.__dict__.update(old_dict)
382 module.__dict__.update(old_dict)
383 raise
383 raise
384
384
385 # iterate over all objects and update functions & classes
385 # iterate over all objects and update functions & classes
386 for name, new_obj in module.__dict__.items():
386 for name, new_obj in module.__dict__.items():
387 key = (module.__name__, name)
387 key = (module.__name__, name)
388 if key not in old_objects: continue
388 if key not in old_objects: continue
389
389
390 new_refs = []
390 new_refs = []
391 for old_ref in old_objects[key]:
391 for old_ref in old_objects[key]:
392 old_obj = old_ref()
392 old_obj = old_ref()
393 if old_obj is None: continue
393 if old_obj is None: continue
394 new_refs.append(old_ref)
394 new_refs.append(old_ref)
395 update_generic(old_obj, new_obj)
395 update_generic(old_obj, new_obj)
396
396
397 if new_refs:
397 if new_refs:
398 old_objects[key] = new_refs
398 old_objects[key] = new_refs
399 else:
399 else:
400 del old_objects[key]
400 del old_objects[key]
401
401
402 return module
402 return module
403
403
404 #------------------------------------------------------------------------------
404 #------------------------------------------------------------------------------
405 # IPython connectivity
405 # IPython connectivity
406 #------------------------------------------------------------------------------
406 #------------------------------------------------------------------------------
407
407
408 from IPython.core.hooks import TryNext
408 from IPython.core.hooks import TryNext
409 from IPython.core.magic import Magics, register_magics, line_magic
409 from IPython.core.magic import Magics, register_magics, line_magic
410 from IPython.core.plugin import Plugin
410 from IPython.core.plugin import Plugin
411
411
412 @register_magics
412 @register_magics
413 class AutoreloadMagics(Magics):
413 class AutoreloadMagics(Magics):
414 def __init__(self, *a, **kw):
414 def __init__(self, *a, **kw):
415 super(AutoreloadMagics, self).__init__(*a, **kw)
415 super(AutoreloadMagics, self).__init__(*a, **kw)
416 self._reloader = ModuleReloader()
416 self._reloader = ModuleReloader()
417 self._reloader.check_all = False
417 self._reloader.check_all = False
418
418
419 @line_magic
419 @line_magic
420 def autoreload(self, parameter_s=''):
420 def autoreload(self, parameter_s=''):
421 r"""%autoreload => Reload modules automatically
421 r"""%autoreload => Reload modules automatically
422
422
423 %autoreload
423 %autoreload
424 Reload all modules (except those excluded by %aimport) automatically
424 Reload all modules (except those excluded by %aimport) automatically
425 now.
425 now.
426
426
427 %autoreload 0
427 %autoreload 0
428 Disable automatic reloading.
428 Disable automatic reloading.
429
429
430 %autoreload 1
430 %autoreload 1
431 Reload all modules imported with %aimport every time before executing
431 Reload all modules imported with %aimport every time before executing
432 the Python code typed.
432 the Python code typed.
433
433
434 %autoreload 2
434 %autoreload 2
435 Reload all modules (except those excluded by %aimport) every time
435 Reload all modules (except those excluded by %aimport) every time
436 before executing the Python code typed.
436 before executing the Python code typed.
437
437
438 Reloading Python modules in a reliable way is in general
438 Reloading Python modules in a reliable way is in general
439 difficult, and unexpected things may occur. %autoreload tries to
439 difficult, and unexpected things may occur. %autoreload tries to
440 work around common pitfalls by replacing function code objects and
440 work around common pitfalls by replacing function code objects and
441 parts of classes previously in the module with new versions. This
441 parts of classes previously in the module with new versions. This
442 makes the following things to work:
442 makes the following things to work:
443
443
444 - Functions and classes imported via 'from xxx import foo' are upgraded
444 - Functions and classes imported via 'from xxx import foo' are upgraded
445 to new versions when 'xxx' is reloaded.
445 to new versions when 'xxx' is reloaded.
446
446
447 - Methods and properties of classes are upgraded on reload, so that
447 - Methods and properties of classes are upgraded on reload, so that
448 calling 'c.foo()' on an object 'c' created before the reload causes
448 calling 'c.foo()' on an object 'c' created before the reload causes
449 the new code for 'foo' to be executed.
449 the new code for 'foo' to be executed.
450
450
451 Some of the known remaining caveats are:
451 Some of the known remaining caveats are:
452
452
453 - Replacing code objects does not always succeed: changing a @property
453 - Replacing code objects does not always succeed: changing a @property
454 in a class to an ordinary method or a method to a member variable
454 in a class to an ordinary method or a method to a member variable
455 can cause problems (but in old objects only).
455 can cause problems (but in old objects only).
456
456
457 - Functions that are removed (eg. via monkey-patching) from a module
457 - Functions that are removed (eg. via monkey-patching) from a module
458 before it is reloaded are not upgraded.
458 before it is reloaded are not upgraded.
459
459
460 - C extension modules cannot be reloaded, and so cannot be
460 - C extension modules cannot be reloaded, and so cannot be
461 autoreloaded.
461 autoreloaded.
462
462
463 """
463 """
464 if parameter_s == '':
464 if parameter_s == '':
465 self._reloader.check(True)
465 self._reloader.check(True)
466 elif parameter_s == '0':
466 elif parameter_s == '0':
467 self._reloader.enabled = False
467 self._reloader.enabled = False
468 elif parameter_s == '1':
468 elif parameter_s == '1':
469 self._reloader.check_all = False
469 self._reloader.check_all = False
470 self._reloader.enabled = True
470 self._reloader.enabled = True
471 elif parameter_s == '2':
471 elif parameter_s == '2':
472 self._reloader.check_all = True
472 self._reloader.check_all = True
473 self._reloader.enabled = True
473 self._reloader.enabled = True
474
474
475 @line_magic
475 @line_magic
476 def aimport(self, parameter_s='', stream=None):
476 def aimport(self, parameter_s='', stream=None):
477 """%aimport => Import modules for automatic reloading.
477 """%aimport => Import modules for automatic reloading.
478
478
479 %aimport
479 %aimport
480 List modules to automatically import and not to import.
480 List modules to automatically import and not to import.
481
481
482 %aimport foo
482 %aimport foo
483 Import module 'foo' and mark it to be autoreloaded for %autoreload 1
483 Import module 'foo' and mark it to be autoreloaded for %autoreload 1
484
484
485 %aimport -foo
485 %aimport -foo
486 Mark module 'foo' to not be autoreloaded for %autoreload 1
486 Mark module 'foo' to not be autoreloaded for %autoreload 1
487
487
488 """
488 """
489
489
490 modname = parameter_s
490 modname = parameter_s
491 if not modname:
491 if not modname:
492 to_reload = self._reloader.modules.keys()
492 to_reload = self._reloader.modules.keys()
493 to_reload.sort()
493 to_reload.sort()
494 to_skip = self._reloader.skip_modules.keys()
494 to_skip = self._reloader.skip_modules.keys()
495 to_skip.sort()
495 to_skip.sort()
496 if stream is None:
496 if stream is None:
497 stream = sys.stdout
497 stream = sys.stdout
498 if self._reloader.check_all:
498 if self._reloader.check_all:
499 stream.write("Modules to reload:\nall-except-skipped\n")
499 stream.write("Modules to reload:\nall-except-skipped\n")
500 else:
500 else:
501 stream.write("Modules to reload:\n%s\n" % ' '.join(to_reload))
501 stream.write("Modules to reload:\n%s\n" % ' '.join(to_reload))
502 stream.write("\nModules to skip:\n%s\n" % ' '.join(to_skip))
502 stream.write("\nModules to skip:\n%s\n" % ' '.join(to_skip))
503 elif modname.startswith('-'):
503 elif modname.startswith('-'):
504 modname = modname[1:]
504 modname = modname[1:]
505 self._reloader.mark_module_skipped(modname)
505 self._reloader.mark_module_skipped(modname)
506 else:
506 else:
507 top_module, top_name = self._reloader.aimport_module(modname)
507 top_module, top_name = self._reloader.aimport_module(modname)
508
508
509 # Inject module to user namespace
509 # Inject module to user namespace
510 self.shell.push({top_name: top_module})
510 self.shell.push({top_name: top_module})
511
511
512 def pre_run_code_hook(self, ipself):
512 def pre_run_code_hook(self, ipself):
513 if not self._reloader.enabled:
513 if not self._reloader.enabled:
514 raise TryNext
514 raise TryNext
515 try:
515 try:
516 self._reloader.check()
516 self._reloader.check()
517 except:
517 except:
518 pass
518 pass
519
519
520
520
521 class AutoreloadPlugin(Plugin):
521 class AutoreloadPlugin(Plugin):
522 def __init__(self, shell=None, config=None):
522 def __init__(self, shell=None, config=None):
523 super(AutoreloadPlugin, self).__init__(shell=shell, config=config)
523 super(AutoreloadPlugin, self).__init__(shell=shell, config=config)
524 auto = AutoreloadMagics(shell)
524 self.auto_magics = AutoreloadMagics(shell)
525 self.shell.register_magics(auto)
525 shell.register_magics(self.auto_magics)
526 self.shell.set_hook('pre_run_code_hook', auto.pre_run_code_hook)
526 shell.set_hook('pre_run_code_hook', self.auto_magics.pre_run_code_hook)
527
527
528
528
529 _loaded = False
529 _loaded = False
530
530
531
531
532 def load_ipython_extension(ip):
532 def load_ipython_extension(ip):
533 """Load the extension in IPython."""
533 """Load the extension in IPython."""
534 global _loaded
534 global _loaded
535 if not _loaded:
535 if not _loaded:
536 plugin = AutoreloadPlugin(shell=ip, config=ip.config)
536 plugin = AutoreloadPlugin(shell=ip, config=ip.config)
537 ip.plugin_manager.register_plugin('autoreload', plugin)
537 ip.plugin_manager.register_plugin('autoreload', plugin)
538 _loaded = True
538 _loaded = True
@@ -1,300 +1,300 b''
1 import os
1 import os
2 import sys
2 import sys
3 import tempfile
3 import tempfile
4 import shutil
4 import shutil
5 import random
5 import random
6 import time
6 import time
7 from StringIO import StringIO
7 from StringIO import StringIO
8
8
9 import nose.tools as nt
9 import nose.tools as nt
10 import IPython.testing.tools as tt
10 import IPython.testing.tools as tt
11
11
12 from IPython.extensions.autoreload import AutoreloadInterface
12 from IPython.extensions.autoreload import AutoreloadPlugin
13 from IPython.core.hooks import TryNext
13 from IPython.core.hooks import TryNext
14
14
15 #-----------------------------------------------------------------------------
15 #-----------------------------------------------------------------------------
16 # Test fixture
16 # Test fixture
17 #-----------------------------------------------------------------------------
17 #-----------------------------------------------------------------------------
18
18
19 class FakeShell(object):
19 class FakeShell(object):
20 def __init__(self):
20 def __init__(self):
21 self.ns = {}
21 self.ns = {}
22 self.reloader = AutoreloadInterface()
22 self.reloader = AutoreloadPlugin(shell=get_ipython())
23
23
24 def run_code(self, code):
24 def run_code(self, code):
25 try:
25 try:
26 self.reloader.pre_run_code_hook(self)
26 self.reloader.auto_magics.pre_run_code_hook(self)
27 except TryNext:
27 except TryNext:
28 pass
28 pass
29 exec code in self.ns
29 exec code in self.ns
30
30
31 def push(self, items):
31 def push(self, items):
32 self.ns.update(items)
32 self.ns.update(items)
33
33
34 def magic_autoreload(self, parameter):
34 def magic_autoreload(self, parameter):
35 self.reloader.magic_autoreload(self, parameter)
35 self.reloader.auto_magics.autoreload(parameter)
36
36
37 def magic_aimport(self, parameter, stream=None):
37 def magic_aimport(self, parameter, stream=None):
38 self.reloader.magic_aimport(self, parameter, stream=stream)
38 self.reloader.auto_magics.aimport(parameter, stream=stream)
39
39
40
40
41 class Fixture(object):
41 class Fixture(object):
42 """Fixture for creating test module files"""
42 """Fixture for creating test module files"""
43
43
44 test_dir = None
44 test_dir = None
45 old_sys_path = None
45 old_sys_path = None
46 filename_chars = "abcdefghijklmopqrstuvwxyz0123456789"
46 filename_chars = "abcdefghijklmopqrstuvwxyz0123456789"
47
47
48 def setUp(self):
48 def setUp(self):
49 self.test_dir = tempfile.mkdtemp()
49 self.test_dir = tempfile.mkdtemp()
50 self.old_sys_path = list(sys.path)
50 self.old_sys_path = list(sys.path)
51 sys.path.insert(0, self.test_dir)
51 sys.path.insert(0, self.test_dir)
52 self.shell = FakeShell()
52 self.shell = FakeShell()
53
53
54 def tearDown(self):
54 def tearDown(self):
55 shutil.rmtree(self.test_dir)
55 shutil.rmtree(self.test_dir)
56 sys.path = self.old_sys_path
56 sys.path = self.old_sys_path
57 self.shell.reloader.enabled = False
57 self.shell.reloader.enabled = False
58
58
59 self.test_dir = None
59 self.test_dir = None
60 self.old_sys_path = None
60 self.old_sys_path = None
61 self.shell = None
61 self.shell = None
62
62
63 def get_module(self):
63 def get_module(self):
64 module_name = "tmpmod_" + "".join(random.sample(self.filename_chars,20))
64 module_name = "tmpmod_" + "".join(random.sample(self.filename_chars,20))
65 if module_name in sys.modules:
65 if module_name in sys.modules:
66 del sys.modules[module_name]
66 del sys.modules[module_name]
67 file_name = os.path.join(self.test_dir, module_name + ".py")
67 file_name = os.path.join(self.test_dir, module_name + ".py")
68 return module_name, file_name
68 return module_name, file_name
69
69
70 def write_file(self, filename, content):
70 def write_file(self, filename, content):
71 """
71 """
72 Write a file, and force a timestamp difference of at least one second
72 Write a file, and force a timestamp difference of at least one second
73
73
74 Notes
74 Notes
75 -----
75 -----
76 Python's .pyc files record the timestamp of their compilation
76 Python's .pyc files record the timestamp of their compilation
77 with a time resolution of one second.
77 with a time resolution of one second.
78
78
79 Therefore, we need to force a timestamp difference between .py
79 Therefore, we need to force a timestamp difference between .py
80 and .pyc, without having the .py file be timestamped in the
80 and .pyc, without having the .py file be timestamped in the
81 future, and without changing the timestamp of the .pyc file
81 future, and without changing the timestamp of the .pyc file
82 (because that is stored in the file). The only reliable way
82 (because that is stored in the file). The only reliable way
83 to achieve this seems to be to sleep.
83 to achieve this seems to be to sleep.
84
84
85 """
85 """
86
86
87 # Sleep one second + eps
87 # Sleep one second + eps
88 time.sleep(1.05)
88 time.sleep(1.05)
89
89
90 # Write
90 # Write
91 f = open(filename, 'w')
91 f = open(filename, 'w')
92 try:
92 try:
93 f.write(content)
93 f.write(content)
94 finally:
94 finally:
95 f.close()
95 f.close()
96
96
97 def new_module(self, code):
97 def new_module(self, code):
98 mod_name, mod_fn = self.get_module()
98 mod_name, mod_fn = self.get_module()
99 f = open(mod_fn, 'w')
99 f = open(mod_fn, 'w')
100 try:
100 try:
101 f.write(code)
101 f.write(code)
102 finally:
102 finally:
103 f.close()
103 f.close()
104 return mod_name, mod_fn
104 return mod_name, mod_fn
105
105
106 #-----------------------------------------------------------------------------
106 #-----------------------------------------------------------------------------
107 # Test automatic reloading
107 # Test automatic reloading
108 #-----------------------------------------------------------------------------
108 #-----------------------------------------------------------------------------
109
109
110 class TestAutoreload(Fixture):
110 class TestAutoreload(Fixture):
111 def _check_smoketest(self, use_aimport=True):
111 def _check_smoketest(self, use_aimport=True):
112 """
112 """
113 Functional test for the automatic reloader using either
113 Functional test for the automatic reloader using either
114 '%autoreload 1' or '%autoreload 2'
114 '%autoreload 1' or '%autoreload 2'
115 """
115 """
116
116
117 mod_name, mod_fn = self.new_module("""
117 mod_name, mod_fn = self.new_module("""
118 x = 9
118 x = 9
119
119
120 z = 123 # this item will be deleted
120 z = 123 # this item will be deleted
121
121
122 def foo(y):
122 def foo(y):
123 return y + 3
123 return y + 3
124
124
125 class Baz(object):
125 class Baz(object):
126 def __init__(self, x):
126 def __init__(self, x):
127 self.x = x
127 self.x = x
128 def bar(self, y):
128 def bar(self, y):
129 return self.x + y
129 return self.x + y
130 @property
130 @property
131 def quux(self):
131 def quux(self):
132 return 42
132 return 42
133 def zzz(self):
133 def zzz(self):
134 '''This method will be deleted below'''
134 '''This method will be deleted below'''
135 return 99
135 return 99
136
136
137 class Bar: # old-style class: weakref doesn't work for it on Python < 2.7
137 class Bar: # old-style class: weakref doesn't work for it on Python < 2.7
138 def foo(self):
138 def foo(self):
139 return 1
139 return 1
140 """)
140 """)
141
141
142 #
142 #
143 # Import module, and mark for reloading
143 # Import module, and mark for reloading
144 #
144 #
145 if use_aimport:
145 if use_aimport:
146 self.shell.magic_autoreload("1")
146 self.shell.magic_autoreload("1")
147 self.shell.magic_aimport(mod_name)
147 self.shell.magic_aimport(mod_name)
148 stream = StringIO()
148 stream = StringIO()
149 self.shell.magic_aimport("", stream=stream)
149 self.shell.magic_aimport("", stream=stream)
150 nt.assert_true(("Modules to reload:\n%s" % mod_name) in
150 nt.assert_true(("Modules to reload:\n%s" % mod_name) in
151 stream.getvalue())
151 stream.getvalue())
152
152
153 nt.assert_raises(
153 nt.assert_raises(
154 ImportError,
154 ImportError,
155 self.shell.magic_aimport, "tmpmod_as318989e89ds")
155 self.shell.magic_aimport, "tmpmod_as318989e89ds")
156 else:
156 else:
157 self.shell.magic_autoreload("2")
157 self.shell.magic_autoreload("2")
158 self.shell.run_code("import %s" % mod_name)
158 self.shell.run_code("import %s" % mod_name)
159 stream = StringIO()
159 stream = StringIO()
160 self.shell.magic_aimport("", stream=stream)
160 self.shell.magic_aimport("", stream=stream)
161 nt.assert_true("Modules to reload:\nall-except-skipped" in
161 nt.assert_true("Modules to reload:\nall-except-skipped" in
162 stream.getvalue())
162 stream.getvalue())
163 nt.assert_true(mod_name in self.shell.ns)
163 nt.assert_in(mod_name, self.shell.ns)
164
164
165 mod = sys.modules[mod_name]
165 mod = sys.modules[mod_name]
166
166
167 #
167 #
168 # Test module contents
168 # Test module contents
169 #
169 #
170 old_foo = mod.foo
170 old_foo = mod.foo
171 old_obj = mod.Baz(9)
171 old_obj = mod.Baz(9)
172 old_obj2 = mod.Bar()
172 old_obj2 = mod.Bar()
173
173
174 def check_module_contents():
174 def check_module_contents():
175 nt.assert_equal(mod.x, 9)
175 nt.assert_equal(mod.x, 9)
176 nt.assert_equal(mod.z, 123)
176 nt.assert_equal(mod.z, 123)
177
177
178 nt.assert_equal(old_foo(0), 3)
178 nt.assert_equal(old_foo(0), 3)
179 nt.assert_equal(mod.foo(0), 3)
179 nt.assert_equal(mod.foo(0), 3)
180
180
181 obj = mod.Baz(9)
181 obj = mod.Baz(9)
182 nt.assert_equal(old_obj.bar(1), 10)
182 nt.assert_equal(old_obj.bar(1), 10)
183 nt.assert_equal(obj.bar(1), 10)
183 nt.assert_equal(obj.bar(1), 10)
184 nt.assert_equal(obj.quux, 42)
184 nt.assert_equal(obj.quux, 42)
185 nt.assert_equal(obj.zzz(), 99)
185 nt.assert_equal(obj.zzz(), 99)
186
186
187 obj2 = mod.Bar()
187 obj2 = mod.Bar()
188 nt.assert_equal(old_obj2.foo(), 1)
188 nt.assert_equal(old_obj2.foo(), 1)
189 nt.assert_equal(obj2.foo(), 1)
189 nt.assert_equal(obj2.foo(), 1)
190
190
191 check_module_contents()
191 check_module_contents()
192
192
193 #
193 #
194 # Simulate a failed reload: no reload should occur and exactly
194 # Simulate a failed reload: no reload should occur and exactly
195 # one error message should be printed
195 # one error message should be printed
196 #
196 #
197 self.write_file(mod_fn, """
197 self.write_file(mod_fn, """
198 a syntax error
198 a syntax error
199 """)
199 """)
200
200
201 with tt.AssertPrints(('[autoreload of %s failed:' % mod_name), channel='stderr'):
201 with tt.AssertPrints(('[autoreload of %s failed:' % mod_name), channel='stderr'):
202 self.shell.run_code("pass") # trigger reload
202 self.shell.run_code("pass") # trigger reload
203 with tt.AssertNotPrints(('[autoreload of %s failed:' % mod_name), channel='stderr'):
203 with tt.AssertNotPrints(('[autoreload of %s failed:' % mod_name), channel='stderr'):
204 self.shell.run_code("pass") # trigger another reload
204 self.shell.run_code("pass") # trigger another reload
205 check_module_contents()
205 check_module_contents()
206
206
207 #
207 #
208 # Rewrite module (this time reload should succeed)
208 # Rewrite module (this time reload should succeed)
209 #
209 #
210 self.write_file(mod_fn, """
210 self.write_file(mod_fn, """
211 x = 10
211 x = 10
212
212
213 def foo(y):
213 def foo(y):
214 return y + 4
214 return y + 4
215
215
216 class Baz(object):
216 class Baz(object):
217 def __init__(self, x):
217 def __init__(self, x):
218 self.x = x
218 self.x = x
219 def bar(self, y):
219 def bar(self, y):
220 return self.x + y + 1
220 return self.x + y + 1
221 @property
221 @property
222 def quux(self):
222 def quux(self):
223 return 43
223 return 43
224
224
225 class Bar: # old-style class
225 class Bar: # old-style class
226 def foo(self):
226 def foo(self):
227 return 2
227 return 2
228 """)
228 """)
229
229
230 def check_module_contents():
230 def check_module_contents():
231 nt.assert_equal(mod.x, 10)
231 nt.assert_equal(mod.x, 10)
232 nt.assert_false(hasattr(mod, 'z'))
232 nt.assert_false(hasattr(mod, 'z'))
233
233
234 nt.assert_equal(old_foo(0), 4) # superreload magic!
234 nt.assert_equal(old_foo(0), 4) # superreload magic!
235 nt.assert_equal(mod.foo(0), 4)
235 nt.assert_equal(mod.foo(0), 4)
236
236
237 obj = mod.Baz(9)
237 obj = mod.Baz(9)
238 nt.assert_equal(old_obj.bar(1), 11) # superreload magic!
238 nt.assert_equal(old_obj.bar(1), 11) # superreload magic!
239 nt.assert_equal(obj.bar(1), 11)
239 nt.assert_equal(obj.bar(1), 11)
240
240
241 nt.assert_equal(old_obj.quux, 43)
241 nt.assert_equal(old_obj.quux, 43)
242 nt.assert_equal(obj.quux, 43)
242 nt.assert_equal(obj.quux, 43)
243
243
244 nt.assert_false(hasattr(old_obj, 'zzz'))
244 nt.assert_false(hasattr(old_obj, 'zzz'))
245 nt.assert_false(hasattr(obj, 'zzz'))
245 nt.assert_false(hasattr(obj, 'zzz'))
246
246
247 obj2 = mod.Bar()
247 obj2 = mod.Bar()
248 nt.assert_equal(old_obj2.foo(), 2)
248 nt.assert_equal(old_obj2.foo(), 2)
249 nt.assert_equal(obj2.foo(), 2)
249 nt.assert_equal(obj2.foo(), 2)
250
250
251 self.shell.run_code("pass") # trigger reload
251 self.shell.run_code("pass") # trigger reload
252 check_module_contents()
252 check_module_contents()
253
253
254 #
254 #
255 # Another failure case: deleted file (shouldn't reload)
255 # Another failure case: deleted file (shouldn't reload)
256 #
256 #
257 os.unlink(mod_fn)
257 os.unlink(mod_fn)
258
258
259 self.shell.run_code("pass") # trigger reload
259 self.shell.run_code("pass") # trigger reload
260 check_module_contents()
260 check_module_contents()
261
261
262 #
262 #
263 # Disable autoreload and rewrite module: no reload should occur
263 # Disable autoreload and rewrite module: no reload should occur
264 #
264 #
265 if use_aimport:
265 if use_aimport:
266 self.shell.magic_aimport("-" + mod_name)
266 self.shell.magic_aimport("-" + mod_name)
267 stream = StringIO()
267 stream = StringIO()
268 self.shell.magic_aimport("", stream=stream)
268 self.shell.magic_aimport("", stream=stream)
269 nt.assert_true(("Modules to skip:\n%s" % mod_name) in
269 nt.assert_true(("Modules to skip:\n%s" % mod_name) in
270 stream.getvalue())
270 stream.getvalue())
271
271
272 # This should succeed, although no such module exists
272 # This should succeed, although no such module exists
273 self.shell.magic_aimport("-tmpmod_as318989e89ds")
273 self.shell.magic_aimport("-tmpmod_as318989e89ds")
274 else:
274 else:
275 self.shell.magic_autoreload("0")
275 self.shell.magic_autoreload("0")
276
276
277 self.write_file(mod_fn, """
277 self.write_file(mod_fn, """
278 x = -99
278 x = -99
279 """)
279 """)
280
280
281 self.shell.run_code("pass") # trigger reload
281 self.shell.run_code("pass") # trigger reload
282 self.shell.run_code("pass")
282 self.shell.run_code("pass")
283 check_module_contents()
283 check_module_contents()
284
284
285 #
285 #
286 # Re-enable autoreload: reload should now occur
286 # Re-enable autoreload: reload should now occur
287 #
287 #
288 if use_aimport:
288 if use_aimport:
289 self.shell.magic_aimport(mod_name)
289 self.shell.magic_aimport(mod_name)
290 else:
290 else:
291 self.shell.magic_autoreload("")
291 self.shell.magic_autoreload("")
292
292
293 self.shell.run_code("pass") # trigger reload
293 self.shell.run_code("pass") # trigger reload
294 nt.assert_equal(mod.x, -99)
294 nt.assert_equal(mod.x, -99)
295
295
296 def test_smoketest_aimport(self):
296 def test_smoketest_aimport(self):
297 self._check_smoketest(use_aimport=True)
297 self._check_smoketest(use_aimport=True)
298
298
299 def test_smoketest_autoreload(self):
299 def test_smoketest_autoreload(self):
300 self._check_smoketest(use_aimport=False)
300 self._check_smoketest(use_aimport=False)
General Comments 0
You need to be logged in to leave comments. Login now