Pauli's autoreload patch to do proper "superreload", i.e. replace code objects of used function objects. Fixes #237691

IPython/Extensions/ipy_autoreload.py

@@ -19,6 +19,7 b' for documentation.'
 #------------------------------------------------------------------------------
 
 import time, os, threading, sys, types, imp, inspect, traceback, atexit
+import weakref
 
 def _get_compiled_ext():
     """Official way to get the extension of compiled files (.pyc or .pyo)"""
@@ -29,7 +30,7 b' def _get_compiled_ext():'
 PY_COMPILED_EXT = _get_compiled_ext()
 
 class ModuleReloader(object):
-    skipped = {}
+    failed = {}
     """Modules that failed to reload: {module: mtime-on-failed-reload, ...}"""
     
     modules = {}
@@ -40,6 +41,9 b' class ModuleReloader(object):'
 
     check_all = True
     """Autoreload all modules, not just those listed in 'modules'"""
+
+    old_objects = {}
+    """(module-name, name) -> weakref, for replacing old code objects"""
     
     def check(self, check_all=False):
         """Check whether some modules need to be reloaded."""
@@ -77,62 +81,152 b' class ModuleReloader(object):'
                 pymtime = os.stat(filename[:-1]).st_mtime
                 if pymtime <= os.stat(filename).st_mtime:
                     continue
-                if self.skipped.get(filename[:-1], None) == pymtime:
+                if self.failed.get(filename[:-1], None) == pymtime:
                     continue
             except OSError:
                 continue
             
             try:
-                superreload(m)
-                if filename[:-1] in self.skipped:
-                    del self.skipped[filename[:-1]]
+                superreload(m, reload, self.old_objects)
+                if filename[:-1] in self.failed:
+                    del self.failed[filename[:-1]]
             except:
-                self.skipped[filename[:-1]] = pymtime
+                print >> sys.stderr, "[autoreload of %s failed: %s]" % (
+                        modname, traceback.format_exc(1))
+                self.failed[filename[:-1]] = pymtime
 
-def update_function(old, new, attrnames):
-    for name in attrnames:
-        setattr(old, name, getattr(new, name))
+#------------------------------------------------------------------------------
+# superreload
+#------------------------------------------------------------------------------
 
-def superreload(module, reload=reload):
+def update_function(old, new):
+    """Upgrade the code object of a function"""
+    for name in ['func_code', 'func_defaults', 'func_doc',
+                 'func_closure', 'func_globals', 'func_dict']:
+        try:
+            setattr(old, name, getattr(new, name))
+        except (AttributeError, TypeError):
+            pass
+
+def update_class(old, new):
+    """Replace stuff in the __dict__ of a class, and upgrade
+    method code objects"""
+    for key in old.__dict__.keys():
+        old_obj = getattr(old, key)
+
+        try:
+            new_obj = getattr(new, key)
+        except AttributeError:
+            # obsolete attribute: remove it
+            try: 
+                delattr(old, key)
+            except (AttributeError, TypeError):
+                pass
+            continue
+        
+        if update_generic(old_obj, new_obj): continue
+
+        try:
+            setattr(old, key, getattr(new, key))
+        except (AttributeError, TypeError):
+            pass # skip non-writable attributes
+
+def update_property(old, new):
+    """Replace get/set/del functions of a property"""
+    update_generic(old.fdel, new.fdel)
+    update_generic(old.fget, new.fget)
+    update_generic(old.fset, new.fset)
+
+def isinstance2(a, b, typ):
+    return isinstance(a, typ) and isinstance(b, typ)
+
+UPDATE_RULES = [
+    (lambda a, b: isinstance2(a, b, types.ClassType),
+     update_class),
+    (lambda a, b: isinstance2(a, b, types.TypeType),
+     update_class),
+    (lambda a, b: isinstance2(a, b, types.FunctionType),
+     update_function), 
+    (lambda a, b: isinstance2(a, b, property),
+     update_property), 
+    (lambda a, b: isinstance2(a, b, types.MethodType),
+     lambda a, b: update_function(a.im_func, b.im_func)),
+]
+
+def update_generic(a, b):
+    for type_check, update in UPDATE_RULES:
+        if type_check(a, b):
+            update(a, b)
+            return True
+    return False
+
+class StrongRef(object):
+    def __init__(self, obj):
+        self.obj = obj
+    def __call__(self):
+        return self.obj
+
+def superreload(module, reload=reload, old_objects={}):
     """Enhanced version of the builtin reload function.
-    
-    superreload replaces the class dictionary of every top-level
-    class in the module with the new one automatically,
-    as well as every function's code object.
+   
+    superreload remembers objects previously in the module, and
+
+    - upgrades the class dictionary of every old class in the module
+    - upgrades the code object of every old function and method
+    - clears the module's namespace before reloading
     
     """
-
+    
+    # collect old objects in the module
+    for name, obj in module.__dict__.items():
+        if not hasattr(obj, '__module__') or obj.__module__ != module.__name__:
+            continue
+        key = (module.__name__, name)
+        try:
+            old_objects.setdefault(key, []).append(weakref.ref(obj))
+        except TypeError:
+            # weakref doesn't work for all types;
+            # create strong references for 'important' cases
+            if isinstance(obj, types.ClassType):
+                old_objects.setdefault(key, []).append(StrongRef(obj))
+
+    # reload module
+    try:
+        # clear namespace first from old cruft
+        old_name = module.__name__
+        module.__dict__.clear()
+        module.__dict__['__name__'] = old_name
+    except (TypeError, AttributeError, KeyError):
+        pass
     module = reload(module)
     
-    # iterate over all objects and update them
-    count = 0
+    # iterate over all objects and update functions & classes
     for name, new_obj in module.__dict__.items():
         key = (module.__name__, name)
-        if _old_objects.has_key(key):
-            for old_obj in _old_objects[key]:
-                if type(new_obj) == types.ClassType:
-                    old_obj.__dict__.update(new_obj.__dict__)
-                    count += 1
-                elif type(new_obj) == types.FunctionType:
-                    update_function(old_obj,
-                           new_obj,
-                           "func_code func_defaults func_doc".split())
-                    count += 1
-                elif type(new_obj) == types.MethodType:
-                    update_function(old_obj.im_func,
-                           new_obj.im_func,
-                           "func_code func_defaults func_doc".split())
-                    count += 1
-    
+        if key not in old_objects: continue
+
+        new_refs = []
+        for old_ref in old_objects[key]:
+            old_obj = old_ref()
+            if old_obj is None: continue
+            new_refs.append(old_ref)
+            update_generic(old_obj, new_obj)
+
+        if new_refs:
+            old_objects[key] = new_refs
+        else:
+            del old_objects[key]
+
     return module
 
 reloader = ModuleReloader()
 
 #------------------------------------------------------------------------------
-# IPython monkey-patching
+# IPython connectivity
 #------------------------------------------------------------------------------
+import IPython.ipapi
 
-import IPython.iplib
+ip = IPython.ipapi.get()
 
 autoreload_enabled = False
 
@@ -144,48 +238,50 b' def runcode_hook(self):'
     except:
         pass
 
-
 def enable_autoreload():
     global autoreload_enabled
     autoreload_enabled = True
-    
 
 def disable_autoreload():
     global autoreload_enabled
     autoreload_enabled = False
-    
-#------------------------------------------------------------------------------
-# IPython connectivity
-#------------------------------------------------------------------------------
-
-import IPython.ipapi
-ip = IPython.ipapi.get()
 
 def autoreload_f(self, parameter_s=''):
     r""" %autoreload => Reload modules automatically
-
+    
     %autoreload
-    Reload all modules (except thoses excluded by %aimport) automatically now.
-
+    Reload all modules (except those excluded by %aimport) automatically now.
+    
     %autoreload 1
     Reload all modules imported with %aimport every time before executing
     the Python code typed.
-
+    
     %autoreload 2
-    Reload all modules (except thoses excluded by %aimport) every time
+    Reload all modules (except those excluded by %aimport) every time
     before executing the Python code typed.
-
-    Reloading Python modules in a reliable way is in general
-    difficult, and unexpected things may occur. Some of the common
-    caveats relevant for 'autoreload' are:
-
-    - Modules are not reloaded in any specific order, and no dependency
-      analysis is done. For example, modules with 'from xxx import foo'
-      retain old versions of 'foo' when 'xxx' is autoreloaded.
-    - Functions or objects imported from the autoreloaded module to
-      the interactive namespace are not updated.
+    
+    Reloading Python modules in a reliable way is in general difficult,
+    and unexpected things may occur. %autoreload tries to work
+    around common pitfalls by replacing code objects of functions
+    previously in the module with new versions. This makes the following
+    things to work:
+
+    - Functions and classes imported via 'from xxx import foo' are upgraded
+      to new versions when 'xxx' is reloaded.
+    - Methods and properties of classes are upgraded on reload, so that
+      calling 'c.foo()' on an object 'c' created before the reload causes
+      the new code for 'foo' to be executed.
+    
+    Some of the known remaining caveats are:
+    
+    - Replacing code objects does not always succeed: changing a @property
+      in a class to an ordinary method or a method to a member variable
+      can cause problems (but in old objects only).
+    - Functions that are removed (eg. via monkey-patching) from a module
+      before it is reloaded are not upgraded.
     - C extension modules cannot be reloaded, and so cannot be
       autoreloaded.
+    
     """
     if parameter_s == '':
         reloader.check(True)
@@ -241,4 +337,4 b' def init():'
     ip.expose_magic('aimport', aimport_f)
     ip.set_hook('pre_runcode_hook', runcode_hook)
     
-init()
  No newline at end of file
+init()

doc/ChangeLog

@@ -1,3 +1,13 b''
[phone]  Ville Vainio <[e-mail]>
+
+	* Extensions/ipy_autoreload.py: Apply Pauli Virtanen's patch
+	to autoreloading that also replaces the changed code objects 
+	with new versions
+
+	* pspersistence.py: report UsageError on %store -w w/o arg,
+	and other usage pattern errors. Bug report by Johann Cohen-Tanugi.
+	
+	
 2008-06-03  Ville Vainio <[e-mail]>
 
 	* ipython.rst, ipython.1: remove -twisted from man page,