From 4f847f55846bf60cb514332594883d22c1079cab 2009-03-14 12:16:54 From: Fernando Perez Date: 2009-03-14 12:16:54 Subject: [PATCH] Fix https://bugs.launchpad.net/ipython/+bug/239054 - Made a more robust reset method to fully flush internal state and restore the system to a clean state. We'll need to see later if %reset should just call this. For now it's used to release resources. --- diff --git a/IPython/ipapi.py b/IPython/ipapi.py index 860565b..da6bcf4 100644 --- a/IPython/ipapi.py +++ b/IPython/ipapi.py @@ -186,7 +186,6 @@ class IPApi(object): self.set_custom_exc = ip.set_custom_exc self.user_ns = ip.user_ns - self.user_ns['_ip'] = self self.set_crash_handler = ip.set_crash_handler diff --git a/IPython/iplib.py b/IPython/iplib.py index e54fe24..b974ba5 100644 --- a/IPython/iplib.py +++ b/IPython/iplib.py @@ -285,6 +285,13 @@ class InteractiveShell(object,Magic): # This is the namespace where all normal user variables live self.user_ns = user_ns self.user_global_ns = user_global_ns + + # An auxiliary namespace that checks what parts of the user_ns were + # loaded at startup, so we can list later only variables defined in + # actual interactive use. Since it is always a subset of user_ns, it + # doesn't need to be seaparately tracked in the ns_table + self.user_config_ns = {} + # A namespace to keep track of internal data structures to prevent # them from cluttering user-visible stuff. Will be updated later self.internal_ns = {} @@ -294,6 +301,24 @@ class InteractiveShell(object,Magic): # of positional arguments of the alias. self.alias_table = {} + # Now that FakeModule produces a real module, we've run into a nasty + # problem: after script execution (via %run), the module where the user + # code ran is deleted. Now that this object is a true module (needed + # so docetst and other tools work correctly), the Python module + # teardown mechanism runs over it, and sets to None every variable + # present in that module. Top-level references to objects from the + # script survive, because the user_ns is updated with them. However, + # calling functions defined in the script that use other things from + # the script will fail, because the function's closure had references + # to the original objects, which are now all None. So we must protect + # these modules from deletion by keeping a cache. To avoid keeping + # stale modules around (we only need the one from the last run), we use + # a dict keyed with the full path to the script, so only the last + # version of the module is held in the cache. The %reset command will + # flush this cache. See the cache_main_mod() and clear_main_mod_cache() + # methods for details on use. + self._user_main_modules = {} + # A table holding all the namespaces IPython deals with, so that # introspection facilities can search easily. self.ns_table = {'user':user_ns, @@ -302,9 +327,14 @@ class InteractiveShell(object,Magic): 'internal':self.internal_ns, 'builtin':__builtin__.__dict__ } - # The user namespace MUST have a pointer to the shell itself. - self.user_ns[name] = self + # Similarly, track all namespaces where references can be held and that + # we can safely clear (so it can NOT include builtin). This one can be + # a simple list. + self.ns_refs_table = [ user_ns, user_global_ns, self.user_config_ns, + self.alias_table, self.internal_ns, + self._user_main_modules ] + # We need to insert into sys.modules something that looks like a # module but which accesses the IPython namespace, for shelve and # pickle to work interactively. Normally they rely on getting @@ -329,32 +359,13 @@ class InteractiveShell(object,Magic): #print "pickle hack in place" # dbg #print 'main_name:',main_name # dbg sys.modules[main_name] = FakeModule(self.user_ns) - - # Now that FakeModule produces a real module, we've run into a nasty - # problem: after script execution (via %run), the module where the user - # code ran is deleted. Now that this object is a true module (needed - # so docetst and other tools work correctly), the Python module - # teardown mechanism runs over it, and sets to None every variable - # present in that module. Top-level references to objects from the - # script survive, because the user_ns is updated with them. However, - # calling functions defined in the script that use other things from - # the script will fail, because the function's closure had references - # to the original objects, which are now all None. So we must protect - # these modules from deletion by keeping a cache. To avoid keeping - # stale modules around (we only need the one from the last run), we use - # a dict keyed with the full path to the script, so only the last - # version of the module is held in the cache. The %reset command will - # flush this cache. See the cache_main_mod() and clear_main_mod_cache() - # methods for details on use. - self._user_main_modules = {} # List of input with multi-line handling. - # Fill its zero entry, user counter starts at 1 - self.input_hist = InputList(['\n']) + self.input_hist = InputList() # This one will hold the 'raw' input history, without any # pre-processing. This will allow users to retrieve the input just as # it was exactly typed in by the user, with %hist -r. - self.input_hist_raw = InputList(['\n']) + self.input_hist_raw = InputList() # list of visited directories try: @@ -380,17 +391,7 @@ class InteractiveShell(object,Magic): no_alias[key] = 1 no_alias.update(__builtin__.__dict__) self.no_alias = no_alias - - # make global variables for user access to these - self.user_ns['_ih'] = self.input_hist - self.user_ns['_oh'] = self.output_hist - self.user_ns['_dh'] = self.dir_hist - - # user aliases to input and output histories - self.user_ns['In'] = self.input_hist - self.user_ns['Out'] = self.output_hist - self.user_ns['_sh'] = IPython.shadowns # Object variable to store code object waiting execution. This is # used mainly by the multithreaded shells, but it can come in handy in # other situations. No need to use a Queue here, since it's a single @@ -583,11 +584,13 @@ class InteractiveShell(object,Magic): else: auto_alias = () self.auto_alias = [s.split(None,1) for s in auto_alias] - # Produce a public API instance self.api = IPython.ipapi.IPApi(self) + # Initialize all user-visible namespaces + self.init_namespaces() + # Call the actual (public) initializer self.init_auto_alias() @@ -654,7 +657,6 @@ class InteractiveShell(object,Magic): # Load readline proper if rc.readline: self.init_readline() - # local shortcut, this is used a LOT self.log = self.logger.log @@ -721,6 +723,39 @@ class InteractiveShell(object,Magic): if batchrun and not self.rc.interact: self.ask_exit() + def init_namespaces(self): + """Initialize all user-visible namespaces to their minimum defaults. + + Certain history lists are also initialized here, as they effectively + act as user namespaces. + + Note + ---- + All data structures here are only filled in, they are NOT reset by this + method. If they were not empty before, data will simply be added to + therm. + """ + # The user namespace MUST have a pointer to the shell itself. + self.user_ns[self.name] = self + + # Store the public api instance + self.user_ns['_ip'] = self.api + + # make global variables for user access to the histories + self.user_ns['_ih'] = self.input_hist + self.user_ns['_oh'] = self.output_hist + self.user_ns['_dh'] = self.dir_hist + + # user aliases to input and output histories + self.user_ns['In'] = self.input_hist + self.user_ns['Out'] = self.output_hist + + self.user_ns['_sh'] = IPython.shadowns + + # Fill the history zero entry, user counter starts at 1 + self.input_hist.append('\n') + self.input_hist_raw.append('\n') + def add_builtins(self): """Store ipython references into the builtin namespace. @@ -1249,7 +1284,27 @@ want to merge them back into the new files.""" % locals() except OSError: pass + # Clear all user namespaces to release all references cleanly. + self.reset() + + # Run user hooks self.hooks.shutdown_hook() + + def reset(self): + """Clear all internal namespaces. + + Note that this is much more aggressive than %reset, since it clears + fully all namespaces, as well as all input/output lists. + """ + for ns in self.ns_refs_table: + ns.clear() + + # Clear input and output histories + self.input_hist[:] = [] + self.input_hist_raw[:] = [] + self.output_hist.clear() + # Restore the user namespaces to minimal usability + self.init_namespaces() def savehist(self): """Save input history to a file (via readline library).""" @@ -2012,7 +2067,6 @@ want to merge them back into the new files.""" % locals() # NOT skip even a blank line if we are in a code block (more is # true) - if line or more: # push to raw history, so hist line numbers stay in sync self.input_hist_raw.append("# " + line + "\n") diff --git a/IPython/ipmaker.py b/IPython/ipmaker.py index 9ed6408..a1c1be8 100644 --- a/IPython/ipmaker.py +++ b/IPython/ipmaker.py @@ -107,8 +107,6 @@ def make_IPython(argv=None,user_ns=None,user_global_ns=None,debug=1, IP.user_ns['help'] = _Helper() except ImportError: warn('help() not available - check site.py') - IP.user_config_ns = {} - if DEVDEBUG: # For developer debugging only (global flag) diff --git a/IPython/testing/plugin/ipdoctest.py b/IPython/testing/plugin/ipdoctest.py index 577386c..d6c9996 100644 --- a/IPython/testing/plugin/ipdoctest.py +++ b/IPython/testing/plugin/ipdoctest.py @@ -72,7 +72,7 @@ class py_file_finder(object): def __call__(self,name): from IPython.genutils import get_py_filename try: - get_py_filename(name) + return get_py_filename(name) except IOError: test_dir = os.path.dirname(self.test_filename) new_path = os.path.join(test_dir,name) diff --git a/IPython/tests/obj_del.py b/IPython/tests/obj_del.py new file mode 100644 index 0000000..089182e --- /dev/null +++ b/IPython/tests/obj_del.py @@ -0,0 +1,34 @@ +"""Test code for https://bugs.launchpad.net/ipython/+bug/239054 + +WARNING: this script exits IPython! It MUST be run in a subprocess. + +When you run the following script from CPython it prints: +__init__ is here +__del__ is here + +and creates the __del__.txt file + +When you run it from IPython it prints: +__init__ is here + +When you exit() or Exit from IPython neothing is printed and no file is created +(the file thing is to make sure __del__ is really never called and not that +just the output is eaten). + +Note that if you call %reset in IPython then everything is Ok. + +IPython should do the equivalent of %reset and release all the references it +holds before exit. This behavior is important when working with binding objects +that rely on __del__. If the current behavior has some use case then I suggest +to add a configuration option to IPython to control it. +""" +import sys + +class A(object): + def __del__(self): + print 'object A deleted' + +a = A() + +# Now, we force an exit, the caller will check that the del printout was given +_ip.IP.ask_exit() diff --git a/IPython/tests/test_iplib.py b/IPython/tests/test_iplib.py new file mode 100644 index 0000000..ad9bb15 --- /dev/null +++ b/IPython/tests/test_iplib.py @@ -0,0 +1,16 @@ +"""Tests for the key iplib module, where the main ipython class is defined. +""" + +import nose.tools as nt + +def test_reset(): + """reset must clear most namespaces.""" + ip = _ip.IP + ip.reset() # first, it should run without error + # Then, check that most namespaces end up empty + for ns in ip.ns_refs_table: + if ns is ip.user_ns: + # The user namespace is reset with some data, so we can't check for + # it being empty + continue + nt.assert_equals(len(ns),0) diff --git a/IPython/tests/test_magic.py b/IPython/tests/test_magic.py index 7208416..eeb175c 100644 --- a/IPython/tests/test_magic.py +++ b/IPython/tests/test_magic.py @@ -80,6 +80,14 @@ def doctest_hist_r(): """ +def test_obj_del(): + """Test that object's __del__ methods are called on exit.""" + test_dir = os.path.dirname(__file__) + del_file = os.path.join(test_dir,'obj_del.py') + out = _ip.IP.getoutput('ipython %s' % del_file) + nt.assert_equals(out,'object A deleted') + + def test_shist(): # Simple tests of ShadowHist class - test generator. import os, shutil, tempfile