Merge pull request #648 from takluyver/usermod

Clean up handling of global namespaces with the proper semantics.

A global namespace should always be tied to a module: pickle accesses classes via the module in which they're defined. So I've changed the arguments for instantiating an InteractiveShell to include `user_module` in place of `user_global_ns`. The global namespace simply becomes a reference to `user_module.__dict__`.

For instantiating InteractiveShell, there are four possibilities:

* Neither `user_ns` nor `user_module` is given. A new (real) module is created named `__main__`, and its `__dict__` becomes the global and local namespace. This is what happens when starting IPython normally.
* Only `user_module` is given. Its `__dict__` becomes the global and local namespace.
* Both `user_ns` and `user_module` are given. `user_module.__dict__` is the global namespace, and `user_ns` is the local namespace. Note that we can't interactively define closures over variables in the local namespace (this seems to be a limitation of Python).
* Only `user_ns` is given. It is treated as the global and local namespace, and a `DummyMod` object is created to refer to it. This is intended as a convenience, especially for the test suite. The recommended way to pass in a single global namespace is as a reference to the module.

`embed()` digs out the locals and the module from the frame in which it's called.

Closes gh-29, closes gh-693.

IPython/core/displayhook.py

@@ -265,16 +265,16 b' class DisplayHook(Configurable):'
                 self.___ = self.__
                 self.__ = self._
                 self._ = result
-                self.shell.user_ns.update({'_':self._,
-                                           '__':self.__,
-                                           '___':self.___})
+                self.shell.push({'_':self._,
+                                 '__':self.__,
+                                '___':self.___}, interactive=False)
 
             # hackish access to top-level  namespace to create _1,_2... dynamically
             to_main = {}
             if self.do_full_cache:
                 new_result = '_'+`self.prompt_count`
                 to_main[new_result] = result
-                self.shell.user_ns.update(to_main)
+                self.shell.push(to_main, interactive=False)
                 self.shell.user_ns['_oh'][self.prompt_count] = result
 
     def log_output(self, format_dict):

IPython/core/history.py

@@ -559,7 +559,8 b' class HistoryManager(HistoryAccessor):'
                    '_ii': self._ii,
                    '_iii': self._iii,
                    new_i : self._i00 }
-        self.shell.user_ns.update(to_main)
+
+        self.shell.push(to_main, interactive=False)
 
     def store_output(self, line_num):
         """If database output logging is enabled, this saves all the

IPython/core/interactiveshell.py

@@ -29,6 +29,7 b' import re'
 import sys
 import tempfile
 import types
+
 try:
     from contextlib import nested
 except:
@@ -372,7 +373,7 b' class InteractiveShell(SingletonConfigurable, Magic):'
     _post_execute = Instance(dict)
 
     def __init__(self, config=None, ipython_dir=None, profile_dir=None,
-                 user_ns=None, user_global_ns=None,
+                 user_module=None, user_ns=None,
                  custom_exceptions=((), None)):
 
         # This is where traits with a config_key argument are updated
@@ -387,7 +388,7 b' class InteractiveShell(SingletonConfigurable, Magic):'
         self.init_environment()
 
         # Create namespaces (user_ns, user_global_ns, etc.)
-        self.init_create_namespaces(user_ns, user_global_ns)
+        self.init_create_namespaces(user_module, user_ns)
         # This has to be done after init_create_namespaces because it uses
         # something in self.user_ns, but before init_sys_modules, which
         # is the first thing to modify sys.
@@ -639,17 +640,14 b' class InteractiveShell(SingletonConfigurable, Magic):'
     def save_sys_module_state(self):
         """Save the state of hooks in the sys module.
 
-        This has to be called after self.user_ns is created.
+        This has to be called after self.user_module is created.
         """
         self._orig_sys_module_state = {}
         self._orig_sys_module_state['stdin'] = sys.stdin
         self._orig_sys_module_state['stdout'] = sys.stdout
         self._orig_sys_module_state['stderr'] = sys.stderr
         self._orig_sys_module_state['excepthook'] = sys.excepthook
-        try:
-            self._orig_sys_modules_main_name = self.user_ns['__name__']
-        except KeyError:
-            pass
+        self._orig_sys_modules_main_name = self.user_module.__name__
 
     def restore_sys_module_state(self):
         """Restore the state of the sys module."""
@@ -659,10 +657,7 b' class InteractiveShell(SingletonConfigurable, Magic):'
         except AttributeError:
             pass
         # Reset what what done in self.init_sys_modules
-        try:
-            sys.modules[self.user_ns['__name__']] = self._orig_sys_modules_main_name
-        except (AttributeError, KeyError):
-            pass
+        sys.modules[self.user_module.__name__] = self._orig_sys_modules_main_name
 
     #-------------------------------------------------------------------------
     # Things related to hooks
@@ -860,7 +855,7 b' class InteractiveShell(SingletonConfigurable, Magic):'
     # Things related to IPython's various namespaces
     #-------------------------------------------------------------------------
 
-    def init_create_namespaces(self, user_ns=None, user_global_ns=None):
+    def init_create_namespaces(self, user_module=None, user_ns=None):
         # Create the namespace where the user will operate.  user_ns is
         # normally the only one used, and it is passed to the exec calls as
         # the locals argument.  But we do carry a user_global_ns namespace
@@ -894,26 +889,14 b' class InteractiveShell(SingletonConfigurable, Magic):'
         # should start with "import __builtin__" (note, no 's') which will
         # definitely give you a module. Yeah, it's somewhat confusing:-(.
 
-        # These routines return properly built dicts as needed by the rest of
-        # the code, and can also be used by extension writers to generate
-        # properly initialized namespaces.
-        user_ns, user_global_ns = self.make_user_namespaces(user_ns,
-                                                            user_global_ns)
+        # These routines return a properly built module and dict as needed by
+        # the rest of the code, and can also be used by extension writers to
+        # generate properly initialized namespaces.
+        self.user_module, self.user_ns = self.prepare_user_module(user_module, user_ns)
 
-        # Assign namespaces
-        # This is the namespace where all normal user variables live
-        self.user_ns = user_ns
-        self.user_global_ns = user_global_ns
-
-        # An auxiliary namespace that checks what parts of the user_ns were
-        # loaded at startup, so we can list later only variables defined in
-        # actual interactive use.  Since it is always a subset of user_ns, it
-        # doesn't need to be separately tracked in the ns_table.
-        self.user_ns_hidden = {}
-
-        # A namespace to keep track of internal data structures to prevent
-        # them from cluttering user-visible stuff.  Will be updated later
-        self.internal_ns = {}
+        # A record of hidden variables we have added to the user namespace, so
+        # we can list later only variables defined in actual interactive use.
+        self.user_ns_hidden = set()
 
         # Now that FakeModule produces a real module, we've run into a nasty
         # problem: after script execution (via %run), the module where the user
@@ -946,78 +929,61 b' class InteractiveShell(SingletonConfigurable, Magic):'
 
         # A table holding all the namespaces IPython deals with, so that
         # introspection facilities can search easily.
-        self.ns_table = {'user':user_ns,
-                         'user_global':user_global_ns,
-                         'internal':self.internal_ns,
+        self.ns_table = {'user_global':self.user_module.__dict__,
+                         'user_local':user_ns,
                          'builtin':builtin_mod.__dict__
                          }
+    
+    @property
+    def user_global_ns(self):
+        return self.user_module.__dict__
 
-        # Similarly, track all namespaces where references can be held and that
-        # we can safely clear (so it can NOT include builtin).  This one can be
-        # a simple list.  Note that the main execution namespaces, user_ns and
-        # user_global_ns, can NOT be listed here, as clearing them blindly
-        # causes errors in object __del__ methods.  Instead, the reset() method
-        # clears them manually and carefully.
-        self.ns_refs_table = [ self.user_ns_hidden,
-                               self.internal_ns, self._main_ns_cache ]
-
-    def make_user_namespaces(self, user_ns=None, user_global_ns=None):
-        """Return a valid local and global user interactive namespaces.
-
-        This builds a dict with the minimal information needed to operate as a
-        valid IPython user namespace, which you can pass to the various
-        embedding classes in ipython. The default implementation returns the
-        same dict for both the locals and the globals to allow functions to
-        refer to variables in the namespace. Customized implementations can
-        return different dicts. The locals dictionary can actually be anything
-        following the basic mapping protocol of a dict, but the globals dict
-        must be a true dict, not even a subclass. It is recommended that any
-        custom object for the locals namespace synchronize with the globals
-        dict somehow.
-
-        Raises TypeError if the provided globals namespace is not a true dict.
+    def prepare_user_module(self, user_module=None, user_ns=None):
+        """Prepare the module and namespace in which user code will be run.
+        
+        When IPython is started normally, both parameters are None: a new module
+        is created automatically, and its __dict__ used as the namespace.
+        
+        If only user_module is provided, its __dict__ is used as the namespace.
+        If only user_ns is provided, a dummy module is created, and user_ns
+        becomes the global namespace. If both are provided (as they may be
+        when embedding), user_ns is the local namespace, and user_module
+        provides the global namespace.
 
         Parameters
         ----------
-        user_ns : dict-like, optional
-            The current user namespace. The items in this namespace should
-            be included in the output. If None, an appropriate blank
-            namespace should be created.
-        user_global_ns : dict, optional
-            The current user global namespace. The items in this namespace
-            should be included in the output. If None, an appropriate
-            blank namespace should be created.
+        user_module : module, optional
+            The current user module in which IPython is being run. If None,
+            a clean module will be created.
+        user_ns : dict, optional
+            A namespace in which to run interactive commands.
 
         Returns
         -------
-            A pair of dictionary-like object to be used as the local namespace
-            of the interpreter and a dict to be used as the global namespace.
+        A tuple of user_module and user_ns, each properly initialised.
         """
-
-
+        if user_module is None and user_ns is not None:
+            user_ns.setdefault("__name__", "__main__")
+            class DummyMod(object):
+                "A dummy module used for IPython's interactive namespace."
+                pass
+            user_module = DummyMod()
+            user_module.__dict__ = user_ns
+            
+        if user_module is None:
+            user_module = types.ModuleType("__main__",
+                doc="Automatically created module for IPython interactive environment")
+        
         # We must ensure that __builtin__ (without the final 's') is always
         # available and pointing to the __builtin__ *module*.  For more details:
         # http://mail.python.org/pipermail/python-dev/2001-April/014068.html
-
+        user_module.__dict__.setdefault('__builtin__', builtin_mod)
+        user_module.__dict__.setdefault('__builtins__', builtin_mod)
+        
         if user_ns is None:
-            # Set __name__ to __main__ to better match the behavior of the
-            # normal interpreter.
-            user_ns = {'__name__'     :'__main__',
-                       py3compat.builtin_mod_name: builtin_mod,
-                       '__builtins__' : builtin_mod,
-                      }
-        else:
-            user_ns.setdefault('__name__','__main__')
-            user_ns.setdefault(py3compat.builtin_mod_name,builtin_mod)
-            user_ns.setdefault('__builtins__',builtin_mod)
-
-        if user_global_ns is None:
-            user_global_ns = user_ns
-        if type(user_global_ns) is not dict:
-            raise TypeError("user_global_ns must be a true dict; got %r"
-                % type(user_global_ns))
+            user_ns = user_module.__dict__
 
-        return user_ns, user_global_ns
+        return user_module, user_ns
 
     def init_sys_modules(self):
         # We need to insert into sys.modules something that looks like a
@@ -1036,13 +1002,8 b' class InteractiveShell(SingletonConfigurable, Magic):'
         # embedded in).
 
         # This is overridden in the InteractiveShellEmbed subclass to a no-op.
-
-        try:
-            main_name = self.user_ns['__name__']
-        except KeyError:
-            raise KeyError('user_ns dictionary MUST have a "__name__" key')
-        else:
-            sys.modules[main_name] = FakeModule(self.user_ns)
+        main_name = self.user_module.__name__
+        sys.modules[main_name] = self.user_module
 
     def init_user_ns(self):
         """Initialize all user-visible namespaces to their minimum defaults.
@@ -1073,8 +1034,8 b' class InteractiveShell(SingletonConfigurable, Magic):'
 
         # For more details:
         # http://mail.python.org/pipermail/python-dev/2001-April/014068.html
-        ns = dict(__builtin__ = builtin_mod)
-
+        ns = dict()
+        
         # Put 'help' in the user namespace
         try:
             from site import _Helper
@@ -1096,7 +1057,7 b' class InteractiveShell(SingletonConfigurable, Magic):'
 
         # Store myself as the public api!!!
         ns['get_ipython'] = self.get_ipython
-
+        
         ns['exit'] = self.exiter
         ns['quit'] = self.exiter
 
@@ -1110,6 +1071,16 b' class InteractiveShell(SingletonConfigurable, Magic):'
 
         # Finally, update the real user's namespace
         self.user_ns.update(ns)
+    
+    @property
+    def all_ns_refs(self):
+        """Get a list of references to all the namespace dictionaries in which
+        IPython might store a user-created object.
+        
+        Note that this does not include the displayhook, which also caches
+        objects from the output."""
+        return [self.user_ns, self.user_global_ns,
+                self._user_main_module.__dict__] + self._main_ns_cache.values()
 
     def reset(self, new_session=True):
         """Clear all internal namespaces, and attempt to release references to
@@ -1127,20 +1098,21 b' class InteractiveShell(SingletonConfigurable, Magic):'
         if self.displayhook.do_full_cache:
             self.displayhook.flush()
 
-        # Restore the user namespaces to minimal usability
-        for ns in self.ns_refs_table:
-            ns.clear()
-
         # The main execution namespaces must be cleared very carefully,
         # skipping the deletion of the builtin-related keys, because doing so
         # would cause errors in many object's __del__ methods.
-        for ns in [self.user_ns, self.user_global_ns]:
-            drop_keys = set(ns.keys())
-            drop_keys.discard('__builtin__')
-            drop_keys.discard('__builtins__')
-            for k in drop_keys:
-                del ns[k]
-
+        if self.user_ns is not self.user_global_ns:
+            self.user_ns.clear()
+        ns = self.user_global_ns
+        drop_keys = set(ns.keys())
+        drop_keys.discard('__builtin__')
+        drop_keys.discard('__builtins__')
+        drop_keys.discard('__name__')
+        for k in drop_keys:
+            del ns[k]
+        
+        self.user_ns_hidden.clear()
+        
         # Restore the user namespaces to minimal usability
         self.init_user_ns()
 
@@ -1170,10 +1142,9 b' class InteractiveShell(SingletonConfigurable, Magic):'
         """
         if varname in ('__builtin__', '__builtins__'):
             raise ValueError("Refusing to delete %s" % varname)
-        ns_refs = self.ns_refs_table + [self.user_ns,
-                self.user_global_ns, self._user_main_module.__dict__] +\
-                self._main_ns_cache.values()
 
+        ns_refs = self.all_ns_refs
+        
         if by_name:                    # Delete by name
             for ns in ns_refs:
                 try:
@@ -1214,7 +1185,7 b' class InteractiveShell(SingletonConfigurable, Magic):'
                 raise TypeError('regex must be a string or compiled pattern')
             # Search for keys in each namespace that match the given regex
             # If a match is found, delete the key/value pair.
-            for ns in self.ns_refs_table:
+            for ns in self.all_ns_refs:
                 for var in ns:
                     if m.search(var):
                         del ns[var]
@@ -1260,13 +1231,11 b' class InteractiveShell(SingletonConfigurable, Magic):'
         self.user_ns.update(vdict)
 
         # And configure interactive visibility
-        config_ns = self.user_ns_hidden
+        user_ns_hidden = self.user_ns_hidden
         if interactive:
-            for name, val in vdict.iteritems():
-                config_ns.pop(name, None)
+            user_ns_hidden.difference_update(vdict)
         else:
-            for name,val in vdict.iteritems():
-                config_ns[name] = val
+            user_ns_hidden.update(vdict)
     
     def drop_by_id(self, variables):
         """Remove a dict of variables from the user namespace, if they are the
@@ -1284,7 +1253,7 b' class InteractiveShell(SingletonConfigurable, Magic):'
         for name, obj in variables.iteritems():
             if name in self.user_ns and self.user_ns[name] is obj:
                 del self.user_ns[name]
-                self.user_ns_hidden.pop(name, None)
+                self.user_ns_hidden.discard(name)
 
     #-------------------------------------------------------------------------
     # Things related to object introspection
@@ -1308,7 +1277,7 b' class InteractiveShell(SingletonConfigurable, Magic):'
             # Put them in a list. The order is important so that we
             # find things in the same order that Python finds them.
             namespaces = [ ('Interactive', self.user_ns),
-                           ('IPython internal', self.internal_ns),
+                           ('Interactive (global)', self.user_global_ns),
                            ('Python builtin', builtin_mod.__dict__),
                            ('Alias', self.alias_manager.alias_table),
                            ]

IPython/core/magic.py

@@ -748,11 +748,10 b' Currently the magic system has the following functions:\\n"""'
         """
 
         user_ns = self.shell.user_ns
-        internal_ns = self.shell.internal_ns
         user_ns_hidden = self.shell.user_ns_hidden
         out = [ i for i in user_ns
                 if not i.startswith('_') \
-                and not (i in internal_ns or i in user_ns_hidden) ]
+                and not i in user_ns_hidden ]
 
         typelist = parameter_s.split()
         if typelist:

IPython/core/prefilter.py

@@ -86,7 +86,7 b' def is_shadowed(identifier, ip):'
     than ifun, because it can not contain a '.' character."""
     # This is much safer than calling ofind, which can change state
     return (identifier in ip.user_ns \
-            or identifier in ip.internal_ns \
+            or identifier in ip.user_global_ns \
             or identifier in ip.ns_table['builtin'])
 
 

IPython/core/tests/test_interactiveshell.py

@@ -25,6 +25,7 b' import shutil'
 import tempfile
 import unittest
 from os.path import join
+import sys
 from StringIO import StringIO
 
 from IPython.testing import decorators as dec
@@ -128,6 +129,7 b' class InteractiveShellTestCase(unittest.TestCase):'
         f = IPython.core.formatters.PlainTextFormatter()
         f([Spam(),Spam()])
     
+
     def test_future_flags(self):
         """Check that future flags are used for parsing code (gh-777)"""
         ip = get_ipython()
@@ -151,6 +153,37 b' class InteractiveShellTestCase(unittest.TestCase):'
         finally:
             # Reset compiler flags so we don't mess up other tests.
             ip.compile.reset_compiler_flags()
+    
+    def test_can_pickle(self):
+        "Can we pickle objects defined interactively (GH-29)"
+        ip = get_ipython()
+        ip.reset()
+        ip.run_cell(("class Mylist(list):\n"
+                     "    def __init__(self,x=[]):\n"
+                     "        list.__init__(self,x)"))
+        ip.run_cell("w=Mylist([1,2,3])")
+        
+        from cPickle import dumps
+        
+        # We need to swap in our main module - this is only necessary
+        # inside the test framework, because IPython puts the interactive module
+        # in place (but the test framework undoes this).
+        _main = sys.modules['__main__']
+        sys.modules['__main__'] = ip.user_module
+        try:
+            res = dumps(ip.user_ns["w"])
+        finally:
+            sys.modules['__main__'] = _main
+        self.assertTrue(isinstance(res, bytes))
+        
+    def test_global_ns(self):
+        "Code in functions must be able to access variables outside them."
+        ip = get_ipython()
+        ip.run_cell("a = 10")
+        ip.run_cell(("def f(x):\n"
+                     "    return x + a"))
+        ip.run_cell("b = f(12)")
+        self.assertEqual(ip.user_ns["b"], 22)
 
     def test_bad_custom_tb(self):
         """Check that InteractiveShell is protected from bad custom exception handlers"""

IPython/core/tests/test_iplib.py

@@ -28,12 +28,8 b' ip = get_ipython()'
 # Test functions
 #-----------------------------------------------------------------------------
 
[e-mail]
 def test_reset():
     """reset must clear most namespaces."""
-    # The number of variables in the private user_ns_hidden is not zero, but it
-    # should be constant regardless of what we do
-    nvars_config_ns = len(ip.user_ns_hidden)
 
     # Check that reset runs without error
     ip.reset()
@@ -41,6 +37,7 b' def test_reset():'
     # Once we've reset it (to clear of any junk that might have been there from
     # other tests, we can count how many variables are in the user's namespace
     nvars_user_ns = len(ip.user_ns)
+    nvars_hidden = len(ip.user_ns_hidden)
 
     # Now add a few variables to user_ns, and check that reset clears them
     ip.user_ns['x'] = 1
@@ -49,15 +46,8 b' def test_reset():'
     
     # Finally, check that all namespaces have only as many variables as we
     # expect to find in them:
-    for ns in ip.ns_refs_table:
-        if ns is ip.user_ns:
-            nvars_expected = nvars_user_ns
-        elif ns is ip.user_ns_hidden:
-            nvars_expected = nvars_config_ns
-        else:
-            nvars_expected = 0
-            
-        yield nt.assert_equals(len(ns), nvars_expected)
+    nt.assert_equals(len(ip.user_ns), nvars_user_ns)
+    nt.assert_equals(len(ip.user_ns_hidden), nvars_hidden)
 
 
 # Tests for reporting of exceptions in various modes, handling of SystemExit,

IPython/core/tests/test_run.py

@@ -228,3 +228,15 b' tclass.py: deleting object: C-third'
         else:
             err = None
         tt.ipexec_validate(self.fname, out, err)
+
+    def test_run_i_after_reset(self):
+        """Check that %run -i still works after %reset (gh-693)"""
+        src = "yy = zz\n"
+        self.mktmp(src)
+        _ip.run_cell("zz = 23")
+        _ip.magic('run -i %s' % self.fname)
+        tt.assert_equals(_ip.user_ns['yy'], 23)
+        _ip.magic('reset -f')
+        _ip.run_cell("zz = 23")
+        _ip.magic('run -i %s' % self.fname)
+        tt.assert_equals(_ip.user_ns['yy'], 23)

IPython/frontend/terminal/embed.py

@@ -72,13 +72,13 b' class InteractiveShellEmbed(TerminalInteractiveShell):'
     display_banner = CBool(True)
 
     def __init__(self, config=None, ipython_dir=None, user_ns=None,
-                 user_global_ns=None, custom_exceptions=((),None),
+                 user_module=None, custom_exceptions=((),None),
                  usage=None, banner1=None, banner2=None,
                  display_banner=None, exit_msg=u''):
 
         super(InteractiveShellEmbed,self).__init__(
             config=config, ipython_dir=ipython_dir, user_ns=user_ns,
-            user_global_ns=user_global_ns, custom_exceptions=custom_exceptions,
+            user_module=user_module, custom_exceptions=custom_exceptions,
             usage=usage, banner1=banner1, banner2=banner2,
             display_banner=display_banner
         )
@@ -95,7 +95,7 b' class InteractiveShellEmbed(TerminalInteractiveShell):'
     def init_sys_modules(self):
         pass
 
-    def __call__(self, header='', local_ns=None, global_ns=None, dummy=None,
+    def __call__(self, header='', local_ns=None, module=None, dummy=None,
                  stack_depth=1):
         """Activate the interactive interpreter.
 
@@ -140,14 +140,14 b' class InteractiveShellEmbed(TerminalInteractiveShell):'
 
         # Call the embedding code with a stack depth of 1 so it can skip over
         # our call and get the original caller's namespaces.
-        self.mainloop(local_ns, global_ns, stack_depth=stack_depth)
+        self.mainloop(local_ns, module, stack_depth=stack_depth)
 
         self.banner2 = self.old_banner2
 
         if self.exit_msg is not None:
             print self.exit_msg
 
-    def mainloop(self, local_ns=None, global_ns=None, stack_depth=0,
+    def mainloop(self, local_ns=None, module=None, stack_depth=0,
                  display_banner=None):
         """Embeds IPython into a running python program.
 
@@ -172,32 +172,37 b' class InteractiveShellEmbed(TerminalInteractiveShell):'
         there is no fundamental reason why it can't work perfectly."""
 
         # Get locals and globals from caller
-        if local_ns is None or global_ns is None:
+        if local_ns is None or module is None:
             call_frame = sys._getframe(stack_depth).f_back
 
             if local_ns is None:
                 local_ns = call_frame.f_locals
-            if global_ns is None:
+            if module is None:
                 global_ns = call_frame.f_globals
-
+                module = sys.modules[global_ns['__name__']]
+        
+        # Save original namespace and module so we can restore them after 
+        # embedding; otherwise the shell doesn't shut down correctly.
+        orig_user_module = self.user_module
+        orig_user_ns = self.user_ns
+        
         # Update namespaces and fire up interpreter
-
+        
         # The global one is easy, we can just throw it in
-        self.user_global_ns = global_ns
+        self.user_module = module
 
-        # but the user/local one is tricky: ipython needs it to store internal
-        # data, but we also need the locals.  We'll copy locals in the user
-        # one, but will track what got copied so we can delete them at exit.
-        # This is so that a later embedded call doesn't see locals from a
-        # previous call (which most likely existed in a separate scope).
-        local_varnames = local_ns.keys()
-        self.user_ns.update(local_ns)
-        #self.user_ns['local_ns'] = local_ns  # dbg
+        # But the user/local one is tricky: ipython needs it to store internal
+        # data, but we also need the locals. We'll throw our hidden variables
+        # like _ih and get_ipython() into the local namespace, but delete them
+        # later.
+        self.user_ns = local_ns
+        self.init_user_ns()
 
         # Patch for global embedding to make sure that things don't overwrite
         # user globals accidentally. Thanks to Richard <[e-mail]>
         # FIXME. Test this a bit more carefully (the if.. is new)
-        if local_ns is None and global_ns is None:
+        # N.B. This can't now ever be called. Not sure what it was for.
+        if local_ns is None and module is None:
             self.user_global_ns.update(__main__.__dict__)
 
         # make sure the tab-completer has the correct frame information, so it
@@ -206,13 +211,14 b' class InteractiveShellEmbed(TerminalInteractiveShell):'
 
         with nested(self.builtin_trap, self.display_trap):
             self.interact(display_banner=display_banner)
-
-            # now, purge out the user namespace from anything we might have added
-            # from the caller's local namespace
-            delvar = self.user_ns.pop
-            for var in local_varnames:
-                delvar(var,None)
-
+        
+        # now, purge out the local namespace of IPython's hidden variables.
+        for name in self.user_ns_hidden:
+            local_ns.pop(name, None)
+        
+        # Restore original namespace so shell can shut down when we exit.
+        self.user_module = orig_user_module
+        self.user_ns = orig_user_ns
 
 _embedded_shell = None
 

IPython/frontend/terminal/interactiveshell.py

@@ -172,13 +172,13 b' class TerminalInteractiveShell(InteractiveShell):'
     )
 
     def __init__(self, config=None, ipython_dir=None, profile_dir=None, user_ns=None,
-                 user_global_ns=None, custom_exceptions=((),None),
+                 user_module=None, custom_exceptions=((),None),
                  usage=None, banner1=None, banner2=None,
                  display_banner=None):
 
         super(TerminalInteractiveShell, self).__init__(
             config=config, profile_dir=profile_dir, user_ns=user_ns,
-            user_global_ns=user_global_ns, custom_exceptions=custom_exceptions
+            user_module=user_module, custom_exceptions=custom_exceptions
         )
         # use os.system instead of utils.process.system by default,
         # because piped system doesn't make sense in the Terminal:

IPython/testing/globalipapp.py

@@ -190,7 +190,6 b' def start_ipython():'
     # Create and initialize our test-friendly IPython instance.
     shell = TerminalInteractiveShell.instance(config=config,
                                               user_ns=ipnsdict(),
-                                              user_global_ns={}
                                               )
 
     # A few more tweaks needed for playing nicely with doctests...
@@ -206,8 +205,7 b' def start_ipython():'
     # can capture subcommands and print them to Python's stdout, otherwise the
     # doctest machinery would miss them.
     shell.system = py3compat.MethodType(xsys, shell)
-
-
+    
     shell._showtraceback = py3compat.MethodType(_showtraceback, shell)
 
     # IPython is ready, now clean up some global state...

IPython/testing/plugin/ipdoctest.py

@@ -271,6 +271,8 b' class DocTestCase(doctests.DocTestCase):'
             # for IPython examples *only*, we swap the globals with the ipython
             # namespace, after updating it with the globals (which doctest
             # fills with the necessary info from the module being tested).
+            self.user_ns_orig = {}
+            self.user_ns_orig.update(_ip.user_ns)
             _ip.user_ns.update(self._dt_test.globs)
             self._dt_test.globs = _ip.user_ns
             # IPython must protect the _ key in the namespace (it can't exist)
@@ -286,6 +288,8 b' class DocTestCase(doctests.DocTestCase):'
         # teardown doesn't destroy the ipython namespace
         if isinstance(self._dt_test.examples[0],IPExample):
             self._dt_test.globs = self._dt_test_globs_ori
+            _ip.user_ns.clear()
+            _ip.user_ns.update(self.user_ns_orig)
             # Restore the behavior of the '_' key in the user namespace to
             # normal after each doctest, so that unittests behave normally
             _ip.user_ns.protect_underscore = False

docs/source/whatsnew/development.txt

@@ -97,6 +97,10 b' Major Bugs fixed'
 * IPython no longer crashes when started on recent versions of Python 3 in
   Windows (:ghissue:`737`).
 
+* Instances of classes defined interactively can now be pickled (:ghissue:`29`;
+  :ghpull:`648`). Note that pickling saves a reference to the class definition,
+  so unpickling the instances will only work where the class has been defined.
+
 .. * use bullet list
 
 Backwards incompatible changes
@@ -132,4 +136,10 b' Backwards incompatible changes'
   The full path will still work, and is necessary for using custom launchers not in
   IPython's launcher module.
 
+* For embedding a shell, note that the parameter ``user_global_ns`` has been
+  replaced by ``user_module``, and expects a module-like object, rather than
+  a namespace dict. The ``user_ns`` parameter works the same way as before, and
+  calling :func:`~IPython.frontend.terminal.embed.embed` with no arguments still
+  works the same way.
+
 .. * use bullet list