Fix https://bugs.launchpad.net/ipython/+bug/239054

- Made a more robust reset method to fully flush internal state and restore
the system to a clean state.  We'll need to see later if %reset should just
call this.  For now it's used to release resources.

IPython/tests/obj_del.py

@@ -0,0 +1,34 b''
+"""Test code for https://bugs.launchpad.net/ipython/+bug/239054
+
+WARNING: this script exits IPython!  It MUST be run in a subprocess.
+
+When you run the following script from CPython it prints:
+__init__ is here
+__del__ is here
+
+and creates the __del__.txt file
+
+When you run it from IPython it prints:
+__init__ is here
+
+When you exit() or Exit from IPython neothing is printed and no file is created
+(the file thing is to make sure __del__ is really never called and not that
+just the output is eaten).
+
+Note that if you call %reset in IPython then everything is Ok.
+
+IPython should do the equivalent of %reset and release all the references it
+holds before exit. This behavior is important when working with binding objects
+that rely on __del__. If the current behavior has some use case then I suggest
+to add a configuration option to IPython to control it.
+"""
+import sys
+
+class A(object):
+    def __del__(self):
+      print 'object A deleted'
+
+a = A()
+
+# Now, we force an exit, the caller will check that the del printout was given
+_ip.IP.ask_exit()

IPython/tests/test_iplib.py

@@ -0,0 +1,16 b''
+"""Tests for the key iplib module, where the main ipython class is defined.
+"""
+
+import nose.tools as nt
+
+def test_reset():
+    """reset must clear most namespaces."""
+    ip = _ip.IP
+    ip.reset()  # first, it should run without error
+    # Then, check that most namespaces end up empty
+    for ns in ip.ns_refs_table:
+        if ns is ip.user_ns:
+            # The user namespace is reset with some data, so we can't check for
+            # it being empty
+            continue
+        nt.assert_equals(len(ns),0)

IPython/ipapi.py

@@ -186,7 +186,6 b' class IPApi(object):'
         self.set_custom_exc = ip.set_custom_exc
 
         self.user_ns = ip.user_ns
-        self.user_ns['_ip'] = self
 
         self.set_crash_handler = ip.set_crash_handler
 

IPython/iplib.py

@@ -285,6 +285,13 b' class InteractiveShell(object,Magic):'
         # This is the namespace where all normal user variables live
         self.user_ns = user_ns
         self.user_global_ns = user_global_ns
+
+        # An auxiliary namespace that checks what parts of the user_ns were
+        # loaded at startup, so we can list later only variables defined in
+        # actual interactive use.  Since it is always a subset of user_ns, it
+        # doesn't need to be seaparately tracked in the ns_table
+        self.user_config_ns = {}
+
         # A namespace to keep track of internal data structures to prevent
         # them from cluttering user-visible stuff.  Will be updated later
         self.internal_ns = {}
@@ -294,6 +301,24 b' class InteractiveShell(object,Magic):'
         # of positional arguments of the alias.
         self.alias_table = {}
 
+        # Now that FakeModule produces a real module, we've run into a nasty
+        # problem: after script execution (via %run), the module where the user
+        # code ran is deleted.  Now that this object is a true module (needed
+        # so docetst and other tools work correctly), the Python module
+        # teardown mechanism runs over it, and sets to None every variable
+        # present in that module.  Top-level references to objects from the
+        # script survive, because the user_ns is updated with them.  However,
+        # calling functions defined in the script that use other things from
+        # the script will fail, because the function's closure had references
+        # to the original objects, which are now all None.  So we must protect
+        # these modules from deletion by keeping a cache.  To avoid keeping
+        # stale modules around (we only need the one from the last run), we use
+        # a dict keyed with the full path to the script, so only the last
+        # version of the module is held in the cache.  The %reset command will
+        # flush this cache.  See the cache_main_mod() and clear_main_mod_cache()
+        # methods for details on use.
+        self._user_main_modules = {}
+
         # A table holding all the namespaces IPython deals with, so that
         # introspection facilities can search easily.
         self.ns_table = {'user':user_ns,
@@ -302,9 +327,14 b' class InteractiveShell(object,Magic):'
                          'internal':self.internal_ns,
                          'builtin':__builtin__.__dict__
                          }
-        # The user namespace MUST have a pointer to the shell itself.
-        self.user_ns[name] = self
 
+        # Similarly, track all namespaces where references can be held and that
+        # we can safely clear (so it can NOT include builtin).  This one can be
+        # a simple list.
+        self.ns_refs_table = [ user_ns, user_global_ns, self.user_config_ns,
+                               self.alias_table, self.internal_ns,
+                               self._user_main_modules ]
+        
         # We need to insert into sys.modules something that looks like a
         # module but which accesses the IPython namespace, for shelve and
         # pickle to work interactively. Normally they rely on getting
@@ -329,32 +359,13 b' class InteractiveShell(object,Magic):'
                 #print "pickle hack in place"  # dbg
                 #print 'main_name:',main_name # dbg
                 sys.modules[main_name] = FakeModule(self.user_ns)
-
-        # Now that FakeModule produces a real module, we've run into a nasty
-        # problem: after script execution (via %run), the module where the user
-        # code ran is deleted.  Now that this object is a true module (needed
-        # so docetst and other tools work correctly), the Python module
-        # teardown mechanism runs over it, and sets to None every variable
-        # present in that module.  Top-level references to objects from the
-        # script survive, because the user_ns is updated with them.  However,
-        # calling functions defined in the script that use other things from
-        # the script will fail, because the function's closure had references
-        # to the original objects, which are now all None.  So we must protect
-        # these modules from deletion by keeping a cache.  To avoid keeping
-        # stale modules around (we only need the one from the last run), we use
-        # a dict keyed with the full path to the script, so only the last
-        # version of the module is held in the cache.  The %reset command will
-        # flush this cache.  See the cache_main_mod() and clear_main_mod_cache()
-        # methods for details on use.
-        self._user_main_modules = {}
         
         # List of input with multi-line handling.
-        # Fill its zero entry, user counter starts at 1
-        self.input_hist = InputList(['\n'])
+        self.input_hist = InputList()
         # This one will hold the 'raw' input history, without any
         # pre-processing.  This will allow users to retrieve the input just as
         # it was exactly typed in by the user, with %hist -r.
-        self.input_hist_raw = InputList(['\n'])
+        self.input_hist_raw = InputList()
 
         # list of visited directories
         try:
@@ -380,17 +391,7 b' class InteractiveShell(object,Magic):'
             no_alias[key] = 1
         no_alias.update(__builtin__.__dict__)
         self.no_alias = no_alias
-                
-        # make global variables for user access to these
-        self.user_ns['_ih'] = self.input_hist
-        self.user_ns['_oh'] = self.output_hist
-        self.user_ns['_dh'] = self.dir_hist
-
-        # user aliases to input and output histories
-        self.user_ns['In']  = self.input_hist
-        self.user_ns['Out'] = self.output_hist
 
-        self.user_ns['_sh'] = IPython.shadowns
         # Object variable to store code object waiting execution.  This is
         # used mainly by the multithreaded shells, but it can come in handy in
         # other situations.  No need to use a Queue here, since it's a single
@@ -583,11 +584,13 b' class InteractiveShell(object,Magic):'
         else:
             auto_alias = ()
         self.auto_alias = [s.split(None,1) for s in auto_alias]
-
         
         # Produce a public API instance
         self.api = IPython.ipapi.IPApi(self)
 
+        # Initialize all user-visible namespaces
+        self.init_namespaces()
+        
         # Call the actual (public) initializer
         self.init_auto_alias()
 
@@ -654,7 +657,6 b' class InteractiveShell(object,Magic):'
         # Load readline proper
         if rc.readline:
             self.init_readline()
-
         
         # local shortcut, this is used a LOT
         self.log = self.logger.log
@@ -721,6 +723,39 b' class InteractiveShell(object,Magic):'
         if batchrun and not self.rc.interact:
             self.ask_exit()            
 
+    def init_namespaces(self):
+        """Initialize all user-visible namespaces to their minimum defaults.
+
+        Certain history lists are also initialized here, as they effectively
+        act as user namespaces.
+
+        Note
+        ----
+        All data structures here are only filled in, they are NOT reset by this
+        method.  If they were not empty before, data will simply be added to
+        therm.
+        """
+        # The user namespace MUST have a pointer to the shell itself.
+        self.user_ns[self.name] = self
+
+        # Store the public api instance
+        self.user_ns['_ip'] = self.api
+
+        # make global variables for user access to the histories
+        self.user_ns['_ih'] = self.input_hist
+        self.user_ns['_oh'] = self.output_hist
+        self.user_ns['_dh'] = self.dir_hist
+
+        # user aliases to input and output histories
+        self.user_ns['In']  = self.input_hist
+        self.user_ns['Out'] = self.output_hist
+
+        self.user_ns['_sh'] = IPython.shadowns
+
+        # Fill the history zero entry, user counter starts at 1
+        self.input_hist.append('\n')
+        self.input_hist_raw.append('\n')
+
     def add_builtins(self):
         """Store ipython references into the builtin namespace.
 
@@ -1249,7 +1284,27 b' want to merge them back into the new files.""" % locals()'
             except OSError:
                 pass
 
+        # Clear all user namespaces to release all references cleanly.
+        self.reset()
+
+        # Run user hooks
         self.hooks.shutdown_hook()
+
+    def reset(self):
+        """Clear all internal namespaces.
+
+        Note that this is much more aggressive than %reset, since it clears
+        fully all namespaces, as well as all input/output lists.
+        """
+        for ns in self.ns_refs_table:
+            ns.clear()
+
+        # Clear input and output histories
+        self.input_hist[:] = []
+        self.input_hist_raw[:] = []
+        self.output_hist.clear()
+        # Restore the user namespaces to minimal usability
+        self.init_namespaces()
         
     def savehist(self):
         """Save input history to a file (via readline library)."""
@@ -2012,7 +2067,6 b' want to merge them back into the new files.""" % locals()'
             # NOT skip even a blank line if we are in a code block (more is
             # true)
             
-            
             if line or more:
                 # push to raw history, so hist line numbers stay in sync
                 self.input_hist_raw.append("# " + line + "\n")

IPython/ipmaker.py

@@ -107,8 +107,6 b' def make_IPython(argv=None,user_ns=None,user_global_ns=None,debug=1,'
         IP.user_ns['help'] = _Helper()
     except ImportError:
         warn('help() not available - check site.py')
-    IP.user_config_ns = {}
-
 
     if DEVDEBUG:
         # For developer debugging only (global flag)

IPython/testing/plugin/ipdoctest.py

@@ -72,7 +72,7 b' class py_file_finder(object):'
     def __call__(self,name):
         from IPython.genutils import get_py_filename
         try:
-            get_py_filename(name)
+            return get_py_filename(name)
         except IOError:
             test_dir = os.path.dirname(self.test_filename)
             new_path = os.path.join(test_dir,name)

IPython/tests/test_magic.py

@@ -80,6 +80,14 b' def doctest_hist_r():'
     """
 
 
+def test_obj_del():
+    """Test that object's __del__ methods are called on exit."""
+    test_dir = os.path.dirname(__file__)
+    del_file = os.path.join(test_dir,'obj_del.py')
+    out = _ip.IP.getoutput('ipython %s' % del_file)
+    nt.assert_equals(out,'object A deleted')
+
+
 def test_shist():
     # Simple tests of ShadowHist class - test generator.
     import os, shutil, tempfile