Cleaned up embedded shell and added cleanup method to InteractiveShell.

* Added explicit code in InteractiveShell to make sure it cleans itself
  up.  This is now done by the single method .cleanup, that can be called
  to have InteractiveShell cleanup.  We can't use __del__ because of
  the many cycles in our object graph.
* The embedded shell is refactored to no embedding logic is in the base
  class.  Thus, InteractiveShell no longer takes an ``embedded``
  argument.  Just use InteractiveShellEmbed.
* Created a super simple top-level :func:`embed` function that creates
  an InteractiveShellEmbed and calls it.

IPython/__init__.py

@@ -38,7 +38,9 b' sys.path.append(os.path.join(os.path.dirname(__file__), "extensions"))'
 # Setup the top level names
 #-----------------------------------------------------------------------------
 
+# In some cases, these are causing circular imports.
 from IPython.core.iplib import InteractiveShell
+from IPython.core.embed import embed
 from IPython.core.error import TryNext
 
 from IPython.lib import (

IPython/core/crashhandler.py

@@ -215,7 +215,7 b' class IPythonCrashHandler(CrashHandler):'
         rpt_add('Platform info  : os.name -> %s, sys.platform -> %s' %
                      (os.name,sys.platform) )
         rpt_add(sec_sep+'Current user configuration structure:\n\n')
-        rpt_add(pformat(self.IP.dict()))
+        # rpt_add(pformat(self.IP.dict()))
         rpt_add(sec_sep+'Crash traceback:\n\n' + traceback)
         try:
             rpt_add(sec_sep+"History of session input:")

IPython/core/embed.py

@@ -28,7 +28,7 b' import sys'
 from IPython.core import ultratb
 from IPython.core.iplib import InteractiveShell
 
-from IPython.utils.traitlets import Bool, Str
+from IPython.utils.traitlets import Bool, Str, CBool
 from IPython.utils.genutils import ask_yes_no
 
 #-----------------------------------------------------------------------------
@@ -57,11 +57,13 b' class InteractiveShellEmbed(InteractiveShell):'
 
     dummy_mode = Bool(False)
     exit_msg = Str('')
+    embedded = CBool(True)
+    embedded_active = CBool(True)
 
-    def __init__(self, parent=None, config=None, usage=None,
+    def __init__(self, parent=None, config=None, ipythondir=None, usage=None,
                  user_ns=None, user_global_ns=None,
-                 banner1='', banner2='',
-                 custom_exceptions=((),None), exit_msg=''):
+                 banner1=None, banner2=None,
+                 custom_exceptions=((),None), exit_msg=None):
 
         # First we need to save the state of sys.displayhook and
         # sys.ipcompleter so we can restore it when we are done.
@@ -69,10 +71,10 b' class InteractiveShellEmbed(InteractiveShell):'
         self.save_sys_ipcompleter()
 
         super(InteractiveShellEmbed,self).__init__(
-            parent=parent, config=config, usage=usage, 
+            parent=parent, config=config, ipythondir=ipythondir, usage=usage, 
             user_ns=user_ns, user_global_ns=user_global_ns,
             banner1=banner1, banner2=banner2, 
-            custom_exceptions=custom_exceptions, embedded=True)
+            custom_exceptions=custom_exceptions)
 
         self.save_sys_displayhook_embed()
         self.exit_msg = exit_msg
@@ -87,6 +89,9 b' class InteractiveShellEmbed(InteractiveShell):'
         self.restore_sys_displayhook()
         self.restore_sys_ipcompleter()
 
+    def init_sys_modules(self):
+        pass
+
     def save_sys_displayhook(self):
         # sys.displayhook is a global, we need to save the user's original
         # Don't rely on __displayhook__, as the user may have changed that.
@@ -121,7 +126,8 b' class InteractiveShellEmbed(InteractiveShell):'
     def restore_sys_displayhook_embed(self):
         sys.displayhook = self.sys_displayhook_embed
 
-    def __call__(self, header='', local_ns=None, global_ns=None, dummy=None):
+    def __call__(self, header='', local_ns=None, global_ns=None, dummy=None,
+                 stack_depth=1):
         """Activate the interactive interpreter.
 
         __call__(self,header='',local_ns=None,global_ns,dummy=None) -> Start
@@ -166,11 +172,44 b' class InteractiveShellEmbed(InteractiveShell):'
 
         # Call the embedding code with a stack depth of 1 so it can skip over
         # our call and get the original caller's namespaces.
-        self.embed_mainloop(banner, local_ns, global_ns, stack_depth=1)
+        self.embed_mainloop(banner, local_ns, global_ns, 
+                            stack_depth=stack_depth)
 
-        if self.exit_msg:
+        if self.exit_msg is not None:
             print self.exit_msg
             
         # Restore global systems (display, completion)
         self.restore_sys_displayhook()
         self.restore_sys_ipcompleter()
+
+
+_embedded_shell = None
+
+
+def embed(header='', config=None, usage=None, banner1=None, banner2=None,
+          exit_msg=''):
+    """Call this to embed IPython at the current point in your program.
+
+    The first invocation of this will create an :class:`InteractiveShellEmbed`
+    instance and then call it.  Consecutive calls just call the already
+    created instance.
+
+    Here is a simple example::
+
+        from IPython import embed
+        a = 10
+        b = 20
+        embed('First time')
+        c = 30
+        d = 40
+        embed
+
+    Full customization can be done by passing a :class:`Struct` in as the 
+    config argument.
+    """
+    global _embedded_shell
+    if _embedded_shell is None:
+        _embedded_shell = InteractiveShellEmbed(config=config,
+        usage=usage, banner1=banner1, banner2=banner2, exit_msg=exit_msg)
+    _embedded_shell(header=header, stack_depth=2)
+

IPython/core/hooks.py

@@ -50,13 +50,12 b' from IPython.core.error import TryNext'
 
 # List here all the default hooks.  For now it's just the editor functions
 # but over time we'll move here all the public API for user-accessible things.
-# vds: >>
+
 __all__ = ['editor', 'fix_error_editor', 'synchronize_with_editor', 'result_display',
            'input_prefilter', 'shutdown_hook', 'late_startup_hook',
            'generate_prompt', 'generate_output_prompt','shell_hook',
            'show_in_pager','pre_prompt_hook', 'pre_runcode_hook',
            'clipboard_get']
-# vds: <<
 
 pformat = PrettyPrinter().pformat
 
@@ -109,10 +108,10 b' def fix_error_editor(self,filename,linenum,column,msg):'
     finally:
         t.close()
 
-# vds: >>
+
 def synchronize_with_editor(self, filename, linenum, column):
         pass
-# vds: <<
+
 
 class CommandChainDispatcher:
     """ Dispatch calls to a chain of commands until some func can handle it
@@ -161,6 +160,7 b' class CommandChainDispatcher:'
         """
         return iter(self.chain)
     
+
 def result_display(self,arg):
     """ Default display hook.
     
@@ -183,6 +183,7 b' def result_display(self,arg):'
     # the default display hook doesn't manipulate the value to put in history
     return None 
 
+
 def input_prefilter(self,line):     
     """ Default input prefilter
     
@@ -197,6 +198,7 b' def input_prefilter(self,line):'
     #print "attempt to rewrite",line #dbg
     return line
 
+
 def shutdown_hook(self):
     """ default shutdown hook
     
@@ -206,31 +208,37 b' def shutdown_hook(self):'
     #print "default shutdown hook ok" # dbg
     return
 
+
 def late_startup_hook(self):
     """ Executed after ipython has been constructed and configured 
     
     """
     #print "default startup hook ok" # dbg
 
+
 def generate_prompt(self, is_continuation):
     """ calculate and return a string with the prompt to display """
     if is_continuation:
         return str(self.outputcache.prompt2)
     return str(self.outputcache.prompt1)
 
+
 def generate_output_prompt(self):
     return str(self.outputcache.prompt_out)
 
+
 def shell_hook(self,cmd):
     """ Run system/shell command a'la os.system() """
 
     shell(cmd, header=self.system_header, verbose=self.system_verbose)
 
+
 def show_in_pager(self,s):
     """ Run a string through pager """
     # raising TryNext here will use the default paging functionality
     raise TryNext
 
+
 def pre_prompt_hook(self):
     """ Run before displaying the next prompt
     
@@ -240,10 +248,12 b' def pre_prompt_hook(self):'
     
     return None
 
+
 def pre_runcode_hook(self):
     """ Executed before running the (prefiltered) code in IPython """
     return None
 
+
 def clipboard_get(self):
     """ Get text from the clipboard.
     """

IPython/core/ipapi.py

@@ -28,8 +28,6 b' Authors:'
 #-----------------------------------------------------------------------------
 
 from IPython.core.error import TryNext, UsageError
-from IPython.core.component import Component
-from IPython.core.iplib import InteractiveShell
 
 #-----------------------------------------------------------------------------
 # Classes and functions
@@ -37,6 +35,7 b' from IPython.core.iplib import InteractiveShell'
 
 def get():
     """Get the most recently created InteractiveShell instance."""
+    from IPython.core.iplib import InteractiveShell
     insts = InteractiveShell.get_instances()
     most_recent = insts[0]
     for inst in insts[1:]:

IPython/core/iplib.py

@@ -112,7 +112,9 b' class SpaceInInput(exceptions.Exception): pass'
 
 class Bunch: pass
 
-class Undefined: pass
+class BuiltinUndefined: pass
+BuiltinUndefined = BuiltinUndefined()
+
 
 class Quitter(object):
     """Simple class to handle exit, similar to Python 2.5's.
@@ -131,6 +133,7 b' class Quitter(object):'
     def __call__(self):
         self.shell.exit()
 
+
 class InputList(list):
     """Class to store user input.
 
@@ -146,6 +149,7 b' class InputList(list):'
     def __getslice__(self,i,j):
         return ''.join(list.__getslice__(self,i,j))
 
+
 class SyntaxTB(ultratb.ListTB):
     """Extension which holds some state: the last exception value"""
 
@@ -163,6 +167,7 b' class SyntaxTB(ultratb.ListTB):'
         self.last_syntax_error = None
         return e
 
+
 def get_default_editor():
     try:
         ed = os.environ['EDITOR']
@@ -190,26 +195,9 b' class SeparateStr(Str):'
 # Main IPython class
 #-----------------------------------------------------------------------------
 
-# FIXME: the Magic class is a mixin for now, and will unfortunately remain so
-# until a full rewrite is made.  I've cleaned all cross-class uses of
-# attributes and methods, but too much user code out there relies on the
-# equlity %foo == __IP.magic_foo, so I can't actually remove the mixin usage.
-#
-# But at least now, all the pieces have been separated and we could, in
-# principle, stop using the mixin.  This will ease the transition to the
-# chainsaw branch.
-
-# For reference, the following is the list of 'self.foo' uses in the Magic
-# class as of 2005-12-28.  These are names we CAN'T use in the main ipython
-# class, to prevent clashes.
-
-# ['self.__class__', 'self.__dict__', 'self._inspect', 'self._ofind',
-#  'self.arg_err', 'self.extract_input', 'self.format_', 'self.lsmagic',
-#  'self.magic_', 'self.options_table', 'self.parse', 'self.shell',
-#  'self.value']
 
 class InteractiveShell(Component, Magic):
-    """An enhanced console for Python."""
+    """An enhanced, interactive shell for Python."""
 
     autocall = Enum((0,1,2), config_key='AUTOCALL')
     autoedit_syntax = CBool(False, config_key='AUTOEDIT_SYNTAX')
@@ -229,6 +217,7 b' class InteractiveShell(Component, Magic):'
     debug = CBool(False, config_key='DEBUG')
     deep_reload = CBool(False, config_key='DEEP_RELOAD')
     embedded = CBool(False)
+    embedded_active = CBool(False)
     editor = Str(get_default_editor(), config_key='EDITOR')
     filename = Str("<ipython console>")
     interactive = CBool(False, config_key='INTERACTIVE')
@@ -295,24 +284,30 b' class InteractiveShell(Component, Magic):'
     # Subclasses with thread support should override this as needed.
     isthreaded = False
 
-    def __init__(self, parent=None, ipythondir=None, config=None, usage=None,
+    def __init__(self, parent=None, config=None, ipythondir=None, usage=None,
                  user_ns=None, user_global_ns=None,
                  banner1=None, banner2=None,
-                 custom_exceptions=((),None), embedded=False):
+                 custom_exceptions=((),None)):
 
         # This is where traitlets with a config_key argument are updated
         # from the values on config.
-        # Ideally, from here on out, the config should only be used when
-        # passing it to children components.
         super(InteractiveShell, self).__init__(parent, config=config, name='__IP')
 
+        # These are relatively independent and stateless
         self.init_ipythondir(ipythondir)
         self.init_instance_attrs()
         self.init_term_title()
         self.init_usage(usage)
         self.init_banner(banner1, banner2)
-        self.init_embedded(embedded)
+
+        # Create namespaces (user_ns, user_global_ns, alias_table, etc.)
         self.init_create_namespaces(user_ns, user_global_ns)
+        # This has to be done after init_create_namespaces because it uses
+        # something in self.user_ns, but before init_sys_modules, which
+        # is the first thing to modify sys.
+        self.save_sys_module_state()
+        self.init_sys_modules()
+
         self.init_history()
         self.init_encoding()
         self.init_handlers()
@@ -323,7 +318,7 b' class InteractiveShell(Component, Magic):'
         self.init_hooks()
         self.init_pushd_popd_magic()
         self.init_traceback_handlers(custom_exceptions)
-        self.init_namespaces()
+        self.init_user_ns()
         self.init_logger()
         self.init_aliases()
         self.init_builtins()
@@ -344,6 +339,10 b' class InteractiveShell(Component, Magic):'
         self.init_pdb()
         self.hooks.late_startup_hook()
 
+    def cleanup(self):
+        self.remove_builtins()
+        self.restore_sys_module_state()
+
     #-------------------------------------------------------------------------
     # Traitlet changed handlers
     #-------------------------------------------------------------------------
@@ -372,12 +371,22 b' class InteractiveShell(Component, Magic):'
     def init_ipythondir(self, ipythondir):
         if ipythondir is not None:
             self.ipythondir = ipythondir
+            self.config.IPYTHONDIR = self.ipythondir
             return
 
+        if hasattr(self.config, 'IPYTHONDIR'):
+            self.ipythondir = self.config.IPYTHONDIR
         if not hasattr(self.config, 'IPYTHONDIR'):
             # cdw is always defined
             self.ipythondir = os.getcwd()
-            return
+
+        # The caller must make sure that ipythondir exists.  We should
+        # probably handle this using a Dir traitlet.
+        if not os.path.isdir(self.ipythondir):
+            raise IOError('IPython dir does not exist: %s' % self.ipythondir)
+
+        # All children can just read this
+        self.config.IPYTHONDIR = self.ipythondir
 
     def init_instance_attrs(self):
         self.jobs = BackgroundJobManager()
@@ -448,15 +457,6 b' class InteractiveShell(Component, Magic):'
         if self.banner2:
             self.banner += '\n' + self.banner2 + '\n'        
 
-    def init_embedded(self, embedded):
-        # We need to know whether the instance is meant for embedding, since
-        # global/local namespaces need to be handled differently in that case
-        self.embedded = embedded
-        if embedded:
-            # Control variable so users can, from within the embedded instance,
-            # permanently deactivate it.
-            self.embedded_active = True
-
     def init_create_namespaces(self, user_ns=None, user_global_ns=None):
         # Create the namespace where the user will operate.  user_ns is
         # normally the only one used, and it is passed to the exec calls as
@@ -562,6 +562,7 b' class InteractiveShell(Component, Magic):'
                                self.alias_table, self.internal_ns,
                                self._main_ns_cache ]
         
+    def init_sys_modules(self):
         # We need to insert into sys.modules something that looks like a
         # module but which accesses the IPython namespace, for shelve and
         # pickle to work interactively. Normally they rely on getting
@@ -577,11 +578,12 b' class InteractiveShell(Component, Magic):'
         # shouldn't overtake the execution environment of the script they're
         # embedded in).
 
-        if not self.embedded:
+        # This is overridden in the InteractiveShellEmbed subclass to a no-op.
+
             try:
                 main_name = self.user_ns['__name__']
             except KeyError:
-                raise KeyError,'user_ns dictionary MUST have a "__name__" key'
+            raise KeyError('user_ns dictionary MUST have a "__name__" key')
             else:
                 sys.modules[main_name] = FakeModule(self.user_ns)
 
@@ -833,12 +835,8 b' class InteractiveShell(Component, Magic):'
 
     def init_builtins(self):
         # track which builtins we add, so we can clean up later
-        self.builtins_added = {}
-        # This method will add the necessary builtins for operation, but
-        # tracking what it did via the builtins_added dict.
-        
-        #TODO: remove this, redundant.  I don't understand why this is 
-        # redundant?
+        self._orig_builtins = {}
+        self._builtins_added = False
         self.add_builtins()
 
     def init_shadow_hist(self):
@@ -1041,7 +1039,7 b' class InteractiveShell(Component, Magic):'
     #     self.extensions[mod] = m
     #     return m
 
-    def init_namespaces(self):
+    def init_user_ns(self):
         """Initialize all user-visible namespaces to their minimum defaults.
 
         Certain history lists are also initialized here, as they effectively
@@ -1077,26 +1075,40 b' class InteractiveShell(Component, Magic):'
         except ImportError:
             warn('help() not available - check site.py')
 
+    def add_builtin(self, key, value):
+        """Add a builtin and save the original."""
+        orig = __builtin__.__dict__.get(key, BuiltinUndefined)
+        self._orig_builtins[key] = orig
+        __builtin__.__dict__[key] = value
+
+    def remove_builtin(self, key):
+        """Remove an added builtin and re-set the original."""
+        try:
+            orig = self._orig_builtins.pop(key)
+        except KeyError:
+            pass
+        else:
+            if orig is BuiltinUndefined:
+                del __builtin__.__dict__[key]
+            else:
+                __builtin__.__dict__[key] = orig
+
     def add_builtins(self):
         """Store ipython references into the __builtin__ namespace.
 
         We strive to modify the __builtin__ namespace as little as possible.
         """
+        if not self._builtins_added:
+            self.add_builtin('exit', Quitter(self,'exit'))
+            self.add_builtin('quit', Quitter(self,'quit'))
 
-        # Install our own quitter instead of the builtins.
-        # This used to be in the __init__ method, but this is a better
-        # place for it.  These can be incorporated to the logic below
-        # when it is refactored.
-        __builtin__.exit = Quitter(self,'exit')
-        __builtin__.quit = Quitter(self,'quit')
-
-        # Recursive reload
+            # Recursive reload function
         try:
             from IPython.lib import deepreload
             if self.deep_reload:
-                __builtin__.reload = deepreload.reload
+                    self.add_builtin('reload', deepreload.reload)
             else:
-                __builtin__.dreload = deepreload.reload
+                    self.add_builtin('dreload', deepreload.reload)
             del deepreload
         except ImportError:
             pass
@@ -1106,16 +1118,49 b' class InteractiveShell(Component, Magic):'
         # another.  Each will increase its value by one upon being activated,
         # which also gives us a way to determine the nesting level.
         __builtin__.__dict__.setdefault('__IPYTHON__active',0)
+            self._builtins_added = True
 
-    def clean_builtins(self):
+    def remove_builtins(self):
         """Remove any builtins which might have been added by add_builtins, or
         restore overwritten ones to their previous values."""
-        for biname,bival in self.builtins_added.items():
-            if bival is Undefined:
-                del __builtin__.__dict__[biname]
-            else:
-                __builtin__.__dict__[biname] = bival
-        self.builtins_added.clear()
+        if self._builtins_added:
+            for key in self._orig_builtins.keys():
+                self.remove_builtin(key)
+            self._orig_builtins.clear()
+            self._builtins_added = False
+
+    def save_sys_module_state(self):
+        """Save the state of hooks in the sys module.
+
+        This has to be called after self.user_ns is created.
+        """
+        self._orig_sys_module_state = {}
+        self._orig_sys_module_state['stdin'] = sys.stdin
+        self._orig_sys_module_state['stdout'] = sys.stdout
+        self._orig_sys_module_state['stderr'] = sys.stderr
+        self._orig_sys_module_state['displayhook'] = sys.displayhook
+        self._orig_sys_module_state['excepthook'] = sys.excepthook
+        try:
+            self._orig_sys_modules_main_name = self.user_ns['__name__']
+        except KeyError:
+            pass
+
+    def restore_sys_module_state(self):
+        """Restore the state of the sys module."""
+        try:
+            for k, v in self._orig_sys_module_state.items():
+                setattr(sys, k, v)
+        except AttributeError:
+            pass
+        try:
+            delattr(sys, 'ipcompleter')
+        except AttributeError:
+            pass
+        # Reset what what done in self.init_sys_modules
+        try:
+            sys.modules[self.user_ns['__name__']] = self._orig_sys_modules_main_name
+        except (AttributeError, KeyError):
+            pass
     
     def set_hook(self,name,hook, priority = 50, str_key = None, re_key = None):
         """set_hook(name,hook) -> sets an internal IPython hook.
@@ -1156,9 +1201,6 b' class InteractiveShell(Component, Magic):'
 
         setattr(self.hooks,name, dp)
         
-        
-        #setattr(self.hooks,name,new.instancemethod(hook,self,self.__class__))
-
     def set_crash_handler(self,crashHandler):
         """Set the IPython crash handler.
 
@@ -1174,7 +1216,6 b' class InteractiveShell(Component, Magic):'
         # frameworks).
         self.sys_excepthook = sys.excepthook
 
-
     def set_custom_exc(self,exc_tuple,handler):
         """set_custom_exc(exc_tuple,handler)
 
@@ -1382,14 +1423,14 b' class InteractiveShell(Component, Magic):'
 
     def ex(self, cmd):
         """Execute a normal python statement in user namespace."""
-        exec cmd in self.user_ns
+        exec cmd in self.user_global_ns, self.user_ns
 
     def ev(self, expr):
         """Evaluate python expression expr in user namespace.
 
         Returns the result of evaluation
         """
-        return eval(expr,self.user_ns)
+        return eval(expr, self.user_global_ns, self.user_ns)
 
     def getoutput(self, cmd):
         return getoutput(self.var_expand(cmd,depth=2),
@@ -1538,7 +1579,7 b' class InteractiveShell(Component, Magic):'
         self.input_hist_raw[:] = []
         self.output_hist.clear()
         # Restore the user namespaces to minimal usability
-        self.init_namespaces()
+        self.init_user_ns()
         
     def savehist(self):
         """Save input history to a file (via readline library)."""
@@ -1937,7 +1978,7 b' class InteractiveShell(Component, Magic):'
         for var in local_varnames:
             delvar(var,None)
         # and clean builtins we may have overridden
-        self.clean_builtins()
+        self.remove_builtins()
 
     def interact_prompt(self):
         """ Print the prompt (in read-eval-print loop)