More re-organization of InteractiveShell.

IPython/core/embed.py

@@ -227,6 +227,14 b' class InteractiveShellEmbed(InteractiveShell):'
             for var in local_varnames:
                 delvar(var,None)
 
+    def set_completer_frame(self, frame=None):
+        if frame:
+            self.Completer.namespace = frame.f_locals
+            self.Completer.global_namespace = frame.f_globals
+        else:
+            self.Completer.namespace = self.user_ns
+            self.Completer.global_namespace = self.user_global_ns
+
 
 _embedded_shell = None
 

IPython/core/iplib.py

@@ -322,9 +322,6 b' class InteractiveShell(Component, Magic):'
         self.init_pdb()
         self.hooks.late_startup_hook()
 
-    def cleanup(self):
-        self.restore_sys_module_state()
-
     #-------------------------------------------------------------------------
     # Traitlet changed handlers
     #-------------------------------------------------------------------------
@@ -346,6 +343,21 b' class InteractiveShell(Component, Magic):'
     def _term_title_changed(self, name, new_value):
         self.init_term_title()
 
+    def set_autoindent(self,value=None):
+        """Set the autoindent flag, checking for readline support.
+
+        If called with no arguments, it acts as a toggle."""
+
+        if not self.has_readline:
+            if os.name == 'posix':
+                warn("The auto-indent feature requires the readline library")
+            self.autoindent = 0
+            return
+        if value is None:
+            self.autoindent = not self.autoindent
+        else:
+            self.autoindent = value
+
     #-------------------------------------------------------------------------
     # init_* methods called by __init__
     #-------------------------------------------------------------------------
@@ -437,580 +449,500 b' class InteractiveShell(Component, Magic):'
         if self.profile:
             self.banner += '\nIPython profile: %s\n' % self.profile
         if self.banner2:
-            self.banner += '\n' + self.banner2 + '\n'        
-
-    def init_create_namespaces(self, user_ns=None, user_global_ns=None):
-        # Create the namespace where the user will operate.  user_ns is
-        # normally the only one used, and it is passed to the exec calls as
-        # the locals argument.  But we do carry a user_global_ns namespace
-        # given as the exec 'globals' argument,  This is useful in embedding
-        # situations where the ipython shell opens in a context where the
-        # distinction between locals and globals is meaningful.  For
-        # non-embedded contexts, it is just the same object as the user_ns dict.
-
-        # FIXME. For some strange reason, __builtins__ is showing up at user
-        # level as a dict instead of a module. This is a manual fix, but I
-        # should really track down where the problem is coming from. Alex
-        # Schmolck reported this problem first.
+            self.banner += '\n' + self.banner2 + '\n'
 
-        # A useful post by Alex Martelli on this topic:
-        # Re: inconsistent value from __builtins__
-        # Von: Alex Martelli <[e-mail]>
-        # Datum: Freitag 01 Oktober 2004 04:45:34 nachmittags/abends
-        # Gruppen: comp.lang.python
+    def init_encoding(self):
+        # Get system encoding at startup time.  Certain terminals (like Emacs
+        # under Win32 have it set to None, and we need to have a known valid
+        # encoding to use in the raw_input() method
+        try:
+            self.stdin_encoding = sys.stdin.encoding or 'ascii'
+        except AttributeError:
+            self.stdin_encoding = 'ascii'
 
-        # Michael Hohn <[e-mail]> wrote:
-        # > >>> print type(builtin_check.get_global_binding('__builtins__'))
-        # > <type 'dict'>
-        # > >>> print type(__builtins__)
-        # > <type 'module'>
-        # > Is this difference in return value intentional?
+    def init_syntax_highlighting(self):
+        # Python source parser/formatter for syntax highlighting
+        pyformat = PyColorize.Parser().format
+        self.pycolorize = lambda src: pyformat(src,'str',self.colors)
 
-        # Well, it's documented that '__builtins__' can be either a dictionary
-        # or a module, and it's been that way for a long time. Whether it's
-        # intentional (or sensible), I don't know. In any case, the idea is
-        # that if you need to access the built-in namespace directly, you
-        # should start with "import __builtin__" (note, no 's') which will
-        # definitely give you a module. Yeah, it's somewhat confusing:-(.
+    def init_pushd_popd_magic(self):
+        # for pushd/popd management
+        try:
+            self.home_dir = get_home_dir()
+        except HomeDirError, msg:
+            fatal(msg)
 
-        # These routines return properly built dicts as needed by the rest of
-        # the code, and can also be used by extension writers to generate
-        # properly initialized namespaces.
-        user_ns, user_global_ns = self.make_user_namespaces(user_ns,
-            user_global_ns)
+        self.dir_stack = []
 
-        # Assign namespaces
-        # This is the namespace where all normal user variables live
-        self.user_ns = user_ns
-        self.user_global_ns = user_global_ns
+    def init_logger(self):
+        self.logger = Logger(self, logfname='ipython_log.py', logmode='rotate')
+        # local shortcut, this is used a LOT
+        self.log = self.logger.log
+        # template for logfile headers.  It gets resolved at runtime by the
+        # logstart method.
+        self.loghead_tpl = \
+"""#log# Automatic Logger file. *** THIS MUST BE THE FIRST LINE ***
+#log# DO NOT CHANGE THIS LINE OR THE TWO BELOW
+#log# opts = %s
+#log# args = %s
+#log# It is safe to make manual edits below here.
+#log#-----------------------------------------------------------------------
+"""
 
-        # An auxiliary namespace that checks what parts of the user_ns were
-        # loaded at startup, so we can list later only variables defined in
-        # actual interactive use.  Since it is always a subset of user_ns, it
-        # doesn't need to be seaparately tracked in the ns_table
-        self.user_config_ns = {}
+    def init_logstart(self):
+        if self.logplay:
+            self.magic_logstart(self.logplay + ' append')
+        elif  self.logfile:
+            self.magic_logstart(self.logfile)
+        elif self.logstart:
+            self.magic_logstart()
 
-        # A namespace to keep track of internal data structures to prevent
-        # them from cluttering user-visible stuff.  Will be updated later
-        self.internal_ns = {}
+    def init_builtins(self):
+        self.builtin_trap = BuiltinTrap(self)
 
-        # Namespace of system aliases.  Each entry in the alias
-        # table must be a 2-tuple of the form (N,name), where N is the number
-        # of positional arguments of the alias.
-        self.alias_table = {}
+    def init_inspector(self):
+        # Object inspector
+        self.inspector = oinspect.Inspector(oinspect.InspectColors,
+                                            PyColorize.ANSICodeColors,
+                                            'NoColor',
+                                            self.object_info_string_level)
 
-        # Now that FakeModule produces a real module, we've run into a nasty
-        # problem: after script execution (via %run), the module where the user
-        # code ran is deleted.  Now that this object is a true module (needed
-        # so docetst and other tools work correctly), the Python module
-        # teardown mechanism runs over it, and sets to None every variable
-        # present in that module.  Top-level references to objects from the
-        # script survive, because the user_ns is updated with them.  However,
-        # calling functions defined in the script that use other things from
-        # the script will fail, because the function's closure had references
-        # to the original objects, which are now all None.  So we must protect
-        # these modules from deletion by keeping a cache.
-        # 
-        # To avoid keeping stale modules around (we only need the one from the
-        # last run), we use a dict keyed with the full path to the script, so
-        # only the last version of the module is held in the cache.  Note,
-        # however, that we must cache the module *namespace contents* (their
-        # __dict__).  Because if we try to cache the actual modules, old ones
-        # (uncached) could be destroyed while still holding references (such as
-        # those held by GUI objects that tend to be long-lived)>
-        # 
-        # The %reset command will flush this cache.  See the cache_main_mod()
-        # and clear_main_mod_cache() methods for details on use.
+    def init_prompts(self):
+        # Initialize cache, set in/out prompts and printing system
+        self.outputcache = CachedOutput(self,
+                                        self.cache_size,
+                                        self.pprint,
+                                        input_sep = self.separate_in,
+                                        output_sep = self.separate_out,
+                                        output_sep2 = self.separate_out2,
+                                        ps1 = self.prompt_in1,
+                                        ps2 = self.prompt_in2,
+                                        ps_out = self.prompt_out,
+                                        pad_left = self.prompts_pad_left)
 
-        # This is the cache used for 'main' namespaces
-        self._main_ns_cache = {}
-        # And this is the single instance of FakeModule whose __dict__ we keep
-        # copying and clearing for reuse on each %run
-        self._user_main_module = FakeModule()
+        # user may have over-ridden the default print hook:
+        try:
+            self.outputcache.__class__.display = self.hooks.display
+        except AttributeError:
+            pass
 
-        # A table holding all the namespaces IPython deals with, so that
-        # introspection facilities can search easily.
-        self.ns_table = {'user':user_ns,
-                         'user_global':user_global_ns,
-                         'alias':self.alias_table,
-                         'internal':self.internal_ns,
-                         'builtin':__builtin__.__dict__
-                         }
+    def init_displayhook(self):
+        self.display_trap = DisplayTrap(self, self.outputcache)
 
-        # Similarly, track all namespaces where references can be held and that
-        # we can safely clear (so it can NOT include builtin).  This one can be
-        # a simple list.
-        self.ns_refs_table = [ user_ns, user_global_ns, self.user_config_ns,
-                               self.alias_table, self.internal_ns,
-                               self._main_ns_cache ]
-        
-    def init_sys_modules(self):
-        # We need to insert into sys.modules something that looks like a
-        # module but which accesses the IPython namespace, for shelve and
-        # pickle to work interactively. Normally they rely on getting
-        # everything out of __main__, but for embedding purposes each IPython
-        # instance has its own private namespace, so we can't go shoving
-        # everything into __main__.
+    def init_reload_doctest(self):
+        # Do a proper resetting of doctest, including the necessary displayhook
+        # monkeypatching
+        try:
+            doctest_reload()
+        except ImportError:
+            warn("doctest module does not exist.")
 
-        # note, however, that we should only do this for non-embedded
-        # ipythons, which really mimic the __main__.__dict__ with their own
-        # namespace.  Embedded instances, on the other hand, should not do
-        # this because they need to manage the user local/global namespaces
-        # only, but they live within a 'normal' __main__ (meaning, they
-        # shouldn't overtake the execution environment of the script they're
-        # embedded in).
+    #-------------------------------------------------------------------------
+    # Things related to injections into the sys module
+    #-------------------------------------------------------------------------
 
-        # This is overridden in the InteractiveShellEmbed subclass to a no-op.
+    def save_sys_module_state(self):
+        """Save the state of hooks in the sys module.
 
+        This has to be called after self.user_ns is created.
+        """
+        self._orig_sys_module_state = {}
+        self._orig_sys_module_state['stdin'] = sys.stdin
+        self._orig_sys_module_state['stdout'] = sys.stdout
+        self._orig_sys_module_state['stderr'] = sys.stderr
+        self._orig_sys_module_state['excepthook'] = sys.excepthook
         try:
-            main_name = self.user_ns['__name__']
+            self._orig_sys_modules_main_name = self.user_ns['__name__']
         except KeyError:
-            raise KeyError('user_ns dictionary MUST have a "__name__" key')
-        else:
-            sys.modules[main_name] = FakeModule(self.user_ns)
+            pass
 
-    def make_user_namespaces(self, user_ns=None, user_global_ns=None):
-        """Return a valid local and global user interactive namespaces.
+    def restore_sys_module_state(self):
+        """Restore the state of the sys module."""
+        try:
+            for k, v in self._orig_sys_module_state.items():
+                setattr(sys, k, v)
+        except AttributeError:
+            pass
+        try:
+            delattr(sys, 'ipcompleter')
+        except AttributeError:
+            pass
+        # Reset what what done in self.init_sys_modules
+        try:
+            sys.modules[self.user_ns['__name__']] = self._orig_sys_modules_main_name
+        except (AttributeError, KeyError):
+            pass
 
-        This builds a dict with the minimal information needed to operate as a
-        valid IPython user namespace, which you can pass to the various
-        embedding classes in ipython. The default implementation returns the
-        same dict for both the locals and the globals to allow functions to
-        refer to variables in the namespace. Customized implementations can
-        return different dicts. The locals dictionary can actually be anything
-        following the basic mapping protocol of a dict, but the globals dict
-        must be a true dict, not even a subclass. It is recommended that any
-        custom object for the locals namespace synchronize with the globals
-        dict somehow.
+    #-------------------------------------------------------------------------
+    # Things related to hooks
+    #-------------------------------------------------------------------------
 
-        Raises TypeError if the provided globals namespace is not a true dict.
+    def init_hooks(self):
+        # hooks holds pointers used for user-side customizations
+        self.hooks = Struct()
 
-        :Parameters:
-            user_ns : dict-like, optional
-                The current user namespace. The items in this namespace should
-                be included in the output. If None, an appropriate blank
-                namespace should be created.
-            user_global_ns : dict, optional
-                The current user global namespace. The items in this namespace
-                should be included in the output. If None, an appropriate
-                blank namespace should be created.
-
-        :Returns:
-            A tuple pair of dictionary-like object to be used as the local namespace
-            of the interpreter and a dict to be used as the global namespace.
-        """
+        self.strdispatchers = {}
 
-        if user_ns is None:
-            # Set __name__ to __main__ to better match the behavior of the
-            # normal interpreter.
-            user_ns = {'__name__'     :'__main__',
-                       '__builtins__' : __builtin__,
-                      }
-        else:
-            user_ns.setdefault('__name__','__main__')
-            user_ns.setdefault('__builtins__',__builtin__)
+        # Set all default hooks, defined in the IPython.hooks module.
+        import IPython.core.hooks
+        hooks = IPython.core.hooks
+        for hook_name in hooks.__all__:
+            # default hooks have priority 100, i.e. low; user hooks should have
+            # 0-100 priority
+            self.set_hook(hook_name,getattr(hooks,hook_name), 100)
 
-        if user_global_ns is None:
-            user_global_ns = user_ns
-        if type(user_global_ns) is not dict:
-            raise TypeError("user_global_ns must be a true dict; got %r"
-                % type(user_global_ns))
+    def set_hook(self,name,hook, priority = 50, str_key = None, re_key = None):
+        """set_hook(name,hook) -> sets an internal IPython hook.
 
-        return user_ns, user_global_ns
+        IPython exposes some of its internal API as user-modifiable hooks.  By
+        adding your function to one of these hooks, you can modify IPython's 
+        behavior to call at runtime your own routines."""
 
-    def init_history(self):
-        # List of input with multi-line handling.
-        self.input_hist = InputList()
-        # This one will hold the 'raw' input history, without any
-        # pre-processing.  This will allow users to retrieve the input just as
-        # it was exactly typed in by the user, with %hist -r.
-        self.input_hist_raw = InputList()
+        # At some point in the future, this should validate the hook before it
+        # accepts it.  Probably at least check that the hook takes the number
+        # of args it's supposed to.
+        
+        f = new.instancemethod(hook,self,self.__class__)
 
-        # list of visited directories
+        # check if the hook is for strdispatcher first
+        if str_key is not None:
+            sdp = self.strdispatchers.get(name, StrDispatch())
+            sdp.add_s(str_key, f, priority )
+            self.strdispatchers[name] = sdp
+            return
+        if re_key is not None:
+            sdp = self.strdispatchers.get(name, StrDispatch())
+            sdp.add_re(re.compile(re_key), f, priority )
+            self.strdispatchers[name] = sdp
+            return
+            
+        dp = getattr(self.hooks, name, None)
+        if name not in IPython.core.hooks.__all__:
+            print "Warning! Hook '%s' is not one of %s" % (name, IPython.core.hooks.__all__ )
+        if not dp:
+            dp = IPython.core.hooks.CommandChainDispatcher()
+        
         try:
-            self.dir_hist = [os.getcwd()]
-        except OSError:
-            self.dir_hist = []
+            dp.add(f,priority)
+        except AttributeError:
+            # it was not commandchain, plain old func - replace
+            dp = f
 
-        # dict of output history
-        self.output_hist = {}
+        setattr(self.hooks,name, dp)
 
-        # Now the history file
-        try:
-            histfname = 'history-%s' % self.profile
-        except AttributeError:
-            histfname = 'history'
-        self.histfile = os.path.join(self.config.IPYTHONDIR, histfname)
+    #-------------------------------------------------------------------------
+    # Things related to the "main" module
+    #-------------------------------------------------------------------------
 
-        # Fill the history zero entry, user counter starts at 1
-        self.input_hist.append('\n')
-        self.input_hist_raw.append('\n')
+    def new_main_mod(self,ns=None):
+        """Return a new 'main' module object for user code execution.
+        """
+        main_mod = self._user_main_module
+        init_fakemod_dict(main_mod,ns)
+        return main_mod
 
-    def init_encoding(self):
-        # Get system encoding at startup time.  Certain terminals (like Emacs
-        # under Win32 have it set to None, and we need to have a known valid
-        # encoding to use in the raw_input() method
-        try:
-            self.stdin_encoding = sys.stdin.encoding or 'ascii'
-        except AttributeError:
-            self.stdin_encoding = 'ascii'
+    def cache_main_mod(self,ns,fname):
+        """Cache a main module's namespace.
 
-    def init_handlers(self):
-        # escapes for automatic behavior on the command line
-        self.ESC_SHELL  = '!'
-        self.ESC_SH_CAP = '!!'
-        self.ESC_HELP   = '?'
-        self.ESC_MAGIC  = '%'
-        self.ESC_QUOTE  = ','
-        self.ESC_QUOTE2 = ';'
-        self.ESC_PAREN  = '/'
+        When scripts are executed via %run, we must keep a reference to the
+        namespace of their __main__ module (a FakeModule instance) around so
+        that Python doesn't clear it, rendering objects defined therein
+        useless.
 
-        # And their associated handlers
-        self.esc_handlers = {self.ESC_PAREN  : self.handle_auto,
-                             self.ESC_QUOTE  : self.handle_auto,
-                             self.ESC_QUOTE2 : self.handle_auto,
-                             self.ESC_MAGIC  : self.handle_magic,
-                             self.ESC_HELP   : self.handle_help,
-                             self.ESC_SHELL  : self.handle_shell_escape,
-                             self.ESC_SH_CAP : self.handle_shell_escape,
-                             }
+        This method keeps said reference in a private dict, keyed by the
+        absolute path of the module object (which corresponds to the script
+        path).  This way, for multiple executions of the same script we only
+        keep one copy of the namespace (the last one), thus preventing memory
+        leaks from old references while allowing the objects from the last
+        execution to be accessible.
 
-    def init_syntax_highlighting(self):
-        # Python source parser/formatter for syntax highlighting
-        pyformat = PyColorize.Parser().format
-        self.pycolorize = lambda src: pyformat(src,'str',self.colors)
+        Note: we can not allow the actual FakeModule instances to be deleted,
+        because of how Python tears down modules (it hard-sets all their
+        references to None without regard for reference counts).  This method
+        must therefore make a *copy* of the given namespace, to allow the
+        original module's __dict__ to be cleared and reused.
 
-    def init_hooks(self):
-        # hooks holds pointers used for user-side customizations
-        self.hooks = Struct()
-        
-        self.strdispatchers = {}
         
-        # Set all default hooks, defined in the IPython.hooks module.
-        import IPython.core.hooks
-        hooks = IPython.core.hooks
-        for hook_name in hooks.__all__:
-            # default hooks have priority 100, i.e. low; user hooks should have
-            # 0-100 priority
-            self.set_hook(hook_name,getattr(hooks,hook_name), 100)
+        Parameters
+        ----------
+          ns : a namespace (a dict, typically)
 
-    def init_pushd_popd_magic(self):
-        # for pushd/popd management
-        try:
-            self.home_dir = get_home_dir()
-        except HomeDirError, msg:
-            fatal(msg)
+          fname : str
+            Filename associated with the namespace.
 
-        self.dir_stack = []
+        Examples
+        --------
 
-    def init_traceback_handlers(self, custom_exceptions):
-        # Syntax error handler.
-        self.SyntaxTB = SyntaxTB(color_scheme='NoColor')
-        
-        # The interactive one is initialized with an offset, meaning we always
-        # want to remove the topmost item in the traceback, which is our own
-        # internal code. Valid modes: ['Plain','Context','Verbose']
-        self.InteractiveTB = ultratb.AutoFormattedTB(mode = 'Plain',
-                                                     color_scheme='NoColor',
-                                                     tb_offset = 1)
+        In [10]: import IPython
 
-        # IPython itself shouldn't crash. This will produce a detailed
-        # post-mortem if it does.  But we only install the crash handler for
-        # non-threaded shells, the threaded ones use a normal verbose reporter
-        # and lose the crash handler.  This is because exceptions in the main
-        # thread (such as in GUI code) propagate directly to sys.excepthook,
-        # and there's no point in printing crash dumps for every user exception.
-        if self.isthreaded:
-            ipCrashHandler = ultratb.FormattedTB()
-        else:
-            from IPython.core import crashhandler
-            ipCrashHandler = crashhandler.IPythonCrashHandler(self)
-        self.set_crash_handler(ipCrashHandler)
+        In [11]: _ip.cache_main_mod(IPython.__dict__,IPython.__file__)
 
-        # and add any custom exception handlers the user may have specified
-        self.set_custom_exc(*custom_exceptions)
+        In [12]: IPython.__file__ in _ip._main_ns_cache
+        Out[12]: True
+        """
+        self._main_ns_cache[os.path.abspath(fname)] = ns.copy()
 
-    def init_logger(self):
-        self.logger = Logger(self, logfname='ipython_log.py', logmode='rotate')
-        # local shortcut, this is used a LOT
-        self.log = self.logger.log
-        # template for logfile headers.  It gets resolved at runtime by the
-        # logstart method.
-        self.loghead_tpl = \
-"""#log# Automatic Logger file. *** THIS MUST BE THE FIRST LINE ***
-#log# DO NOT CHANGE THIS LINE OR THE TWO BELOW
-#log# opts = %s
-#log# args = %s
-#log# It is safe to make manual edits below here.
-#log#-----------------------------------------------------------------------
-"""
+    def clear_main_mod_cache(self):
+        """Clear the cache of main modules.
 
-    def init_logstart(self):
-        if self.logplay:
-            self.magic_logstart(self.logplay + ' append')
-        elif  self.logfile:
-            self.magic_logstart(self.logfile)
-        elif self.logstart:
-            self.magic_logstart()
+        Mainly for use by utilities like %reset.
 
-    def init_aliases(self):
-        # dict of things NOT to alias (keywords, builtins and some magics)
-        no_alias = {}
-        no_alias_magics = ['cd','popd','pushd','dhist','alias','unalias']
-        for key in keyword.kwlist + no_alias_magics:
-            no_alias[key] = 1
-        no_alias.update(__builtin__.__dict__)
-        self.no_alias = no_alias
+        Examples
+        --------
 
-        # Make some aliases automatically
-        # Prepare list of shell aliases to auto-define
-        if os.name == 'posix':
-            auto_alias = ('mkdir mkdir', 'rmdir rmdir',
-                          'mv mv -i','rm rm -i','cp cp -i',
-                          'cat cat','less less','clear clear',
-                          # a better ls
-                          'ls ls -F',
-                          # long ls
-                          'll ls -lF')
-            # Extra ls aliases with color, which need special treatment on BSD
-            # variants
-            ls_extra = ( # color ls
-                         'lc ls -F -o --color',
-                         # ls normal files only
-                         'lf ls -F -o --color %l | grep ^-',
-                         # ls symbolic links
-                         'lk ls -F -o --color %l | grep ^l',
-                         # directories or links to directories,
-                         'ldir ls -F -o --color %l | grep /$',
-                         # things which are executable
-                         'lx ls -F -o --color %l | grep ^-..x',
-                         )
-            # The BSDs don't ship GNU ls, so they don't understand the
-            # --color switch out of the box
-            if 'bsd' in sys.platform:
-                ls_extra = ( # ls normal files only
-                             'lf ls -lF | grep ^-',
-                             # ls symbolic links
-                             'lk ls -lF | grep ^l',
-                             # directories or links to directories,
-                             'ldir ls -lF | grep /$',
-                             # things which are executable
-                             'lx ls -lF | grep ^-..x',
-                             )
-            auto_alias = auto_alias + ls_extra
-        elif os.name in ['nt','dos']:
-            auto_alias = ('ls dir /on',
-                          'ddir dir /ad /on', 'ldir dir /ad /on',
-                          'mkdir mkdir','rmdir rmdir','echo echo',
-                          'ren ren','cls cls','copy copy')
-        else:
-            auto_alias = ()
-        self.auto_alias = [s.split(None,1) for s in auto_alias]
-        
-        # Load default aliases
-        for alias, cmd in self.auto_alias:
-            self.define_alias(alias,cmd)
+        In [15]: import IPython
 
-        # Load user aliases
-        for alias in self.alias:
-            self.magic_alias(alias)
+        In [16]: _ip.cache_main_mod(IPython.__dict__,IPython.__file__)
 
-    def init_builtins(self):
-        self.builtin_trap = BuiltinTrap(self)
+        In [17]: len(_ip._main_ns_cache) > 0
+        Out[17]: True
 
-    def init_shadow_hist(self):
-        try:
-            self.db = pickleshare.PickleShareDB(self.config.IPYTHONDIR + "/db")
-        except exceptions.UnicodeDecodeError:
-            print "Your ipythondir can't be decoded to unicode!"
-            print "Please set HOME environment variable to something that"
-            print r"only has ASCII characters, e.g. c:\home"
-            print "Now it is", self.config.IPYTHONDIR
-            sys.exit()
-        self.shadowhist = ipcorehist.ShadowHist(self.db)
+        In [18]: _ip.clear_main_mod_cache()
 
-    def init_inspector(self):
-        # Object inspector
-        self.inspector = oinspect.Inspector(oinspect.InspectColors,
-                                            PyColorize.ANSICodeColors,
-                                            'NoColor',
-                                            self.object_info_string_level)
+        In [19]: len(_ip._main_ns_cache) == 0
+        Out[19]: True
+        """
+        self._main_ns_cache.clear()
 
-    def init_readline(self):
-        """Command history completion/saving/reloading."""
+    #-------------------------------------------------------------------------
+    # Things related to debugging
+    #-------------------------------------------------------------------------
 
-        self.rl_next_input = None
-        self.rl_do_indent = False
+    def init_pdb(self):
+        # Set calling of pdb on exceptions
+        # self.call_pdb is a property
+        self.call_pdb = self.pdb
 
-        if not self.readline_use:
+    def _get_call_pdb(self):
+        return self._call_pdb
+
+    def _set_call_pdb(self,val):
+
+        if val not in (0,1,False,True):
+            raise ValueError,'new call_pdb value must be boolean'
+
+        # store value in instance
+        self._call_pdb = val
+
+        # notify the actual exception handlers
+        self.InteractiveTB.call_pdb = val
+        if self.isthreaded:
+            try:
+                self.sys_excepthook.call_pdb = val
+            except:
+                warn('Failed to activate pdb for threaded exception handler')
+
+    call_pdb = property(_get_call_pdb,_set_call_pdb,None,
+                        'Control auto-activation of pdb at exceptions')
+
+    def debugger(self,force=False):
+        """Call the pydb/pdb debugger.
+
+        Keywords:
+
+          - force(False): by default, this routine checks the instance call_pdb
+          flag and does not actually invoke the debugger if the flag is false.
+          The 'force' option forces the debugger to activate even if the flag
+          is false.
+        """
+
+        if not (force or self.call_pdb):
             return
 
-        import IPython.utils.rlineimpl as readline
-                  
-        if not readline.have_readline:
-            self.has_readline = 0
-            self.readline = None
-            # no point in bugging windows users with this every time:
-            warn('Readline services not available on this platform.')
+        if not hasattr(sys,'last_traceback'):
+            error('No traceback has been produced, nothing to debug.')
+            return
+
+        # use pydb if available
+        if debugger.has_pydb:
+            from pydb import pm
         else:
-            sys.modules['readline'] = readline
-            import atexit
-            from IPython.core.completer import IPCompleter
-            self.Completer = IPCompleter(self,
-                                            self.user_ns,
-                                            self.user_global_ns,
-                                            self.readline_omit__names,
-                                            self.alias_table)
-            sdisp = self.strdispatchers.get('complete_command', StrDispatch())
-            self.strdispatchers['complete_command'] = sdisp
-            self.Completer.custom_completers = sdisp
-            # Platform-specific configuration
-            if os.name == 'nt':
-                self.readline_startup_hook = readline.set_pre_input_hook
-            else:
-                self.readline_startup_hook = readline.set_startup_hook
+            # fallback to our internal debugger
+            pm = lambda : self.InteractiveTB.debugger(force=True)
+        self.history_saving_wrapper(pm)()
 
-            # Load user's initrc file (readline config)
-            # Or if libedit is used, load editrc.
-            inputrc_name = os.environ.get('INPUTRC')
-            if inputrc_name is None:
-                home_dir = get_home_dir()
-                if home_dir is not None:
-                    inputrc_name = '.inputrc'
-                    if readline.uses_libedit:
-                        inputrc_name = '.editrc'
-                    inputrc_name = os.path.join(home_dir, inputrc_name)
-            if os.path.isfile(inputrc_name):
-                try:
-                    readline.read_init_file(inputrc_name)
-                except:
-                    warn('Problems reading readline initialization file <%s>'
-                         % inputrc_name)
-            
-            self.has_readline = 1
-            self.readline = readline
-            # save this in sys so embedded copies can restore it properly
-            sys.ipcompleter = self.Completer.complete
-            self.set_completer()
+    #-------------------------------------------------------------------------
+    # Things related to IPython's various namespaces
+    #-------------------------------------------------------------------------
 
-            # Configure readline according to user's prefs
-            # This is only done if GNU readline is being used.  If libedit
-            # is being used (as on Leopard) the readline config is
-            # not run as the syntax for libedit is different.
-            if not readline.uses_libedit:
-                for rlcommand in self.readline_parse_and_bind:
-                    #print "loading rl:",rlcommand  # dbg
-                    readline.parse_and_bind(rlcommand)
+    def init_create_namespaces(self, user_ns=None, user_global_ns=None):
+        # Create the namespace where the user will operate.  user_ns is
+        # normally the only one used, and it is passed to the exec calls as
+        # the locals argument.  But we do carry a user_global_ns namespace
+        # given as the exec 'globals' argument,  This is useful in embedding
+        # situations where the ipython shell opens in a context where the
+        # distinction between locals and globals is meaningful.  For
+        # non-embedded contexts, it is just the same object as the user_ns dict.
 
-            # Remove some chars from the delimiters list.  If we encounter
-            # unicode chars, discard them.
-            delims = readline.get_completer_delims().encode("ascii", "ignore")
-            delims = delims.translate(string._idmap,
-                                      self.readline_remove_delims)
-            readline.set_completer_delims(delims)
-            # otherwise we end up with a monster history after a while:
-            readline.set_history_length(1000)
-            try:
-                #print '*** Reading readline history'  # dbg
-                readline.read_history_file(self.histfile)
-            except IOError:
-                pass  # It doesn't exist yet.
+        # FIXME. For some strange reason, __builtins__ is showing up at user
+        # level as a dict instead of a module. This is a manual fix, but I
+        # should really track down where the problem is coming from. Alex
+        # Schmolck reported this problem first.
 
-            atexit.register(self.atexit_operations)
-            del atexit
+        # A useful post by Alex Martelli on this topic:
+        # Re: inconsistent value from __builtins__
+        # Von: Alex Martelli <[e-mail]>
+        # Datum: Freitag 01 Oktober 2004 04:45:34 nachmittags/abends
+        # Gruppen: comp.lang.python
 
-        # Configure auto-indent for all platforms
-        self.set_autoindent(self.autoindent)
+        # Michael Hohn <[e-mail]> wrote:
+        # > >>> print type(builtin_check.get_global_binding('__builtins__'))
+        # > <type 'dict'>
+        # > >>> print type(__builtins__)
+        # > <type 'module'>
+        # > Is this difference in return value intentional?
+
+        # Well, it's documented that '__builtins__' can be either a dictionary
+        # or a module, and it's been that way for a long time. Whether it's
+        # intentional (or sensible), I don't know. In any case, the idea is
+        # that if you need to access the built-in namespace directly, you
+        # should start with "import __builtin__" (note, no 's') which will
+        # definitely give you a module. Yeah, it's somewhat confusing:-(.
+
+        # These routines return properly built dicts as needed by the rest of
+        # the code, and can also be used by extension writers to generate
+        # properly initialized namespaces.
+        user_ns, user_global_ns = self.make_user_namespaces(user_ns,
+            user_global_ns)
+
+        # Assign namespaces
+        # This is the namespace where all normal user variables live
+        self.user_ns = user_ns
+        self.user_global_ns = user_global_ns
+
+        # An auxiliary namespace that checks what parts of the user_ns were
+        # loaded at startup, so we can list later only variables defined in
+        # actual interactive use.  Since it is always a subset of user_ns, it
+        # doesn't need to be seaparately tracked in the ns_table
+        self.user_config_ns = {}
+
+        # A namespace to keep track of internal data structures to prevent
+        # them from cluttering user-visible stuff.  Will be updated later
+        self.internal_ns = {}
+
+        # Namespace of system aliases.  Each entry in the alias
+        # table must be a 2-tuple of the form (N,name), where N is the number
+        # of positional arguments of the alias.
+        self.alias_table = {}
+
+        # Now that FakeModule produces a real module, we've run into a nasty
+        # problem: after script execution (via %run), the module where the user
+        # code ran is deleted.  Now that this object is a true module (needed
+        # so docetst and other tools work correctly), the Python module
+        # teardown mechanism runs over it, and sets to None every variable
+        # present in that module.  Top-level references to objects from the
+        # script survive, because the user_ns is updated with them.  However,
+        # calling functions defined in the script that use other things from
+        # the script will fail, because the function's closure had references
+        # to the original objects, which are now all None.  So we must protect
+        # these modules from deletion by keeping a cache.
+        # 
+        # To avoid keeping stale modules around (we only need the one from the
+        # last run), we use a dict keyed with the full path to the script, so
+        # only the last version of the module is held in the cache.  Note,
+        # however, that we must cache the module *namespace contents* (their
+        # __dict__).  Because if we try to cache the actual modules, old ones
+        # (uncached) could be destroyed while still holding references (such as
+        # those held by GUI objects that tend to be long-lived)>
+        # 
+        # The %reset command will flush this cache.  See the cache_main_mod()
+        # and clear_main_mod_cache() methods for details on use.
+
+        # This is the cache used for 'main' namespaces
+        self._main_ns_cache = {}
+        # And this is the single instance of FakeModule whose __dict__ we keep
+        # copying and clearing for reuse on each %run
+        self._user_main_module = FakeModule()
+
+        # A table holding all the namespaces IPython deals with, so that
+        # introspection facilities can search easily.
+        self.ns_table = {'user':user_ns,
+                         'user_global':user_global_ns,
+                         'alias':self.alias_table,
+                         'internal':self.internal_ns,
+                         'builtin':__builtin__.__dict__
+                         }
+
+        # Similarly, track all namespaces where references can be held and that
+        # we can safely clear (so it can NOT include builtin).  This one can be
+        # a simple list.
+        self.ns_refs_table = [ user_ns, user_global_ns, self.user_config_ns,
+                               self.alias_table, self.internal_ns,
+                               self._main_ns_cache ]
+
+    def init_sys_modules(self):
+        # We need to insert into sys.modules something that looks like a
+        # module but which accesses the IPython namespace, for shelve and
+        # pickle to work interactively. Normally they rely on getting
+        # everything out of __main__, but for embedding purposes each IPython
+        # instance has its own private namespace, so we can't go shoving
+        # everything into __main__.
+
+        # note, however, that we should only do this for non-embedded
+        # ipythons, which really mimic the __main__.__dict__ with their own
+        # namespace.  Embedded instances, on the other hand, should not do
+        # this because they need to manage the user local/global namespaces
+        # only, but they live within a 'normal' __main__ (meaning, they
+        # shouldn't overtake the execution environment of the script they're
+        # embedded in).
 
-    def init_prompts(self):
-        # Initialize cache, set in/out prompts and printing system
-        self.outputcache = CachedOutput(self,
-                                        self.cache_size,
-                                        self.pprint,
-                                        input_sep = self.separate_in,
-                                        output_sep = self.separate_out,
-                                        output_sep2 = self.separate_out2,
-                                        ps1 = self.prompt_in1,
-                                        ps2 = self.prompt_in2,
-                                        ps_out = self.prompt_out,
-                                        pad_left = self.prompts_pad_left)
+        # This is overridden in the InteractiveShellEmbed subclass to a no-op.
 
-        # user may have over-ridden the default print hook:
         try:
-            self.outputcache.__class__.display = self.hooks.display
-        except AttributeError:
-            pass
+            main_name = self.user_ns['__name__']
+        except KeyError:
+            raise KeyError('user_ns dictionary MUST have a "__name__" key')
+        else:
+            sys.modules[main_name] = FakeModule(self.user_ns)
 
-    def init_displayhook(self):
-        self.display_trap = DisplayTrap(self, self.outputcache)
+    def make_user_namespaces(self, user_ns=None, user_global_ns=None):
+        """Return a valid local and global user interactive namespaces.
 
-    def init_reload_doctest(self):
-        # Do a proper resetting of doctest, including the necessary displayhook
-        # monkeypatching
-        try:
-            doctest_reload()
-        except ImportError:
-            warn("doctest module does not exist.")
+        This builds a dict with the minimal information needed to operate as a
+        valid IPython user namespace, which you can pass to the various
+        embedding classes in ipython. The default implementation returns the
+        same dict for both the locals and the globals to allow functions to
+        refer to variables in the namespace. Customized implementations can
+        return different dicts. The locals dictionary can actually be anything
+        following the basic mapping protocol of a dict, but the globals dict
+        must be a true dict, not even a subclass. It is recommended that any
+        custom object for the locals namespace synchronize with the globals
+        dict somehow.
 
-    def init_magics(self):
-        # Set user colors (don't do it in the constructor above so that it
-        # doesn't crash if colors option is invalid)
-        self.magic_colors(self.colors)
+        Raises TypeError if the provided globals namespace is not a true dict.
 
-    def init_pdb(self):
-        # Set calling of pdb on exceptions
-        # self.call_pdb is a property
-        self.call_pdb = self.pdb
+        :Parameters:
+            user_ns : dict-like, optional
+                The current user namespace. The items in this namespace should
+                be included in the output. If None, an appropriate blank
+                namespace should be created.
+            user_global_ns : dict, optional
+                The current user global namespace. The items in this namespace
+                should be included in the output. If None, an appropriate
+                blank namespace should be created.
 
-    # def init_exec_commands(self):
-    #     for cmd in self.config.EXECUTE:
-    #         print "execute:", cmd
-    #         self.api.runlines(cmd)
-    #         
-    #     batchrun = False
-    #     if self.config.has_key('EXECFILE'):
-    #         for batchfile in [path(arg) for arg in self.config.EXECFILE
-    #             if arg.lower().endswith('.ipy')]:
-    #             if not batchfile.isfile():
-    #                 print "No such batch file:", batchfile
-    #                 continue
-    #             self.api.runlines(batchfile.text())
-    #             batchrun = True
-    #     # without -i option, exit after running the batch file
-    #     if batchrun and not self.interactive:
-    #         self.ask_exit()            
+        :Returns:
+            A tuple pair of dictionary-like object to be used as the local namespace
+            of the interpreter and a dict to be used as the global namespace.
+        """
 
-    # def load(self, mod):
-    #     """ Load an extension.
-    #     
-    #     Some modules should (or must) be 'load()':ed, rather than just imported.
-    #     
-    #     Loading will do:
-    #     
-    #     - run init_ipython(ip)
-    #     - run ipython_firstrun(ip)
-    #     """
-    # 
-    #     if mod in self.extensions:
-    #         # just to make sure we don't init it twice
-    #         # note that if you 'load' a module that has already been
-    #         # imported, init_ipython gets run anyway
-    #         
-    #         return self.extensions[mod]
-    #     __import__(mod)
-    #     m = sys.modules[mod]
-    #     if hasattr(m,'init_ipython'):
-    #         m.init_ipython(self)
-    #         
-    #     if hasattr(m,'ipython_firstrun'):
-    #         already_loaded = self.db.get('firstrun_done', set())
-    #         if mod not in already_loaded:
-    #             m.ipython_firstrun(self)
-    #             already_loaded.add(mod)
-    #             self.db['firstrun_done'] = already_loaded
-    #         
-    #     self.extensions[mod] = m
-    #     return m
+        if user_ns is None:
+            # Set __name__ to __main__ to better match the behavior of the
+            # normal interpreter.
+            user_ns = {'__name__'     :'__main__',
+                       '__builtins__' : __builtin__,
+                      }
+        else:
+            user_ns.setdefault('__name__','__main__')
+            user_ns.setdefault('__builtins__',__builtin__)
+
+        if user_global_ns is None:
+            user_global_ns = user_ns
+        if type(user_global_ns) is not dict:
+            raise TypeError("user_global_ns must be a true dict; got %r"
+                % type(user_global_ns))
+
+        return user_ns, user_global_ns
 
     def init_user_ns(self):
         """Initialize all user-visible namespaces to their minimum defaults.
@@ -1048,76 +980,183 b' class InteractiveShell(Component, Magic):'
         except ImportError:
             warn('help() not available - check site.py')
 
-    def save_sys_module_state(self):
-        """Save the state of hooks in the sys module.
+    def reset(self):
+        """Clear all internal namespaces.
+
+        Note that this is much more aggressive than %reset, since it clears
+        fully all namespaces, as well as all input/output lists.
+        """
+        for ns in self.ns_refs_table:
+            ns.clear()
+
+        # Clear input and output histories
+        self.input_hist[:] = []
+        self.input_hist_raw[:] = []
+        self.output_hist.clear()
+        # Restore the user namespaces to minimal usability
+        self.init_user_ns()
+
+    def push(self, variables, interactive=True):
+        """Inject a group of variables into the IPython user namespace.
+
+        Parameters
+        ----------
+        variables : dict, str or list/tuple of str
+            The variables to inject into the user's namespace.  If a dict,
+            a simple update is done.  If a str, the string is assumed to 
+            have variable names separated by spaces.  A list/tuple of str
+            can also be used to give the variable names.  If just the variable
+            names are give (list/tuple/str) then the variable values looked
+            up in the callers frame.
+        interactive : bool
+            If True (default), the variables will be listed with the ``who``
+            magic.
+        """
+        vdict = None
+
+        # We need a dict of name/value pairs to do namespace updates.
+        if isinstance(variables, dict):
+            vdict = variables
+        elif isinstance(variables, (basestring, list, tuple)):
+            if isinstance(variables, basestring):
+                vlist = variables.split()
+            else:
+                vlist = variables
+            vdict = {}
+            cf = sys._getframe(1)
+            for name in vlist:
+                try:
+                    vdict[name] = eval(name, cf.f_globals, cf.f_locals)
+                except:
+                    print ('Could not get variable %s from %s' %
+                           (name,cf.f_code.co_name))
+        else:
+            raise ValueError('variables must be a dict/str/list/tuple')
+            
+        # Propagate variables to user namespace
+        self.user_ns.update(vdict)
+
+        # And configure interactive visibility
+        config_ns = self.user_config_ns
+        if interactive:
+            for name, val in vdict.iteritems():
+                config_ns.pop(name, None)
+        else:
+            for name,val in vdict.iteritems():
+                config_ns[name] = val
+
+    #-------------------------------------------------------------------------
+    # Things related to history management
+    #-------------------------------------------------------------------------
+
+    def init_history(self):
+        # List of input with multi-line handling.
+        self.input_hist = InputList()
+        # This one will hold the 'raw' input history, without any
+        # pre-processing.  This will allow users to retrieve the input just as
+        # it was exactly typed in by the user, with %hist -r.
+        self.input_hist_raw = InputList()
+
+        # list of visited directories
+        try:
+            self.dir_hist = [os.getcwd()]
+        except OSError:
+            self.dir_hist = []
+
+        # dict of output history
+        self.output_hist = {}
+
+        # Now the history file
+        try:
+            histfname = 'history-%s' % self.profile
+        except AttributeError:
+            histfname = 'history'
+        self.histfile = os.path.join(self.config.IPYTHONDIR, histfname)
+
+        # Fill the history zero entry, user counter starts at 1
+        self.input_hist.append('\n')
+        self.input_hist_raw.append('\n')
+
+    def init_shadow_hist(self):
+        try:
+            self.db = pickleshare.PickleShareDB(self.config.IPYTHONDIR + "/db")
+        except exceptions.UnicodeDecodeError:
+            print "Your ipythondir can't be decoded to unicode!"
+            print "Please set HOME environment variable to something that"
+            print r"only has ASCII characters, e.g. c:\home"
+            print "Now it is", self.config.IPYTHONDIR
+            sys.exit()
+        self.shadowhist = ipcorehist.ShadowHist(self.db)
+
+    def savehist(self):
+        """Save input history to a file (via readline library)."""
+
+        if not self.has_readline:
+            return
+        
+        try:
+            self.readline.write_history_file(self.histfile)
+        except:
+            print 'Unable to save IPython command history to file: ' + \
+                  `self.histfile`
+
+    def reloadhist(self):
+        """Reload the input history from disk file."""
+
+        if self.has_readline:
+            try:
+                self.readline.clear_history()
+                self.readline.read_history_file(self.shell.histfile)
+            except AttributeError:
+                pass
+
+    def history_saving_wrapper(self, func):
+        """ Wrap func for readline history saving
 
-        This has to be called after self.user_ns is created.
-        """
-        self._orig_sys_module_state = {}
-        self._orig_sys_module_state['stdin'] = sys.stdin
-        self._orig_sys_module_state['stdout'] = sys.stdout
-        self._orig_sys_module_state['stderr'] = sys.stderr
-        self._orig_sys_module_state['excepthook'] = sys.excepthook
-        try:
-            self._orig_sys_modules_main_name = self.user_ns['__name__']
-        except KeyError:
-            pass
+        Convert func into callable that saves & restores
+        history around the call """
 
-    def restore_sys_module_state(self):
-        """Restore the state of the sys module."""
-        try:
-            for k, v in self._orig_sys_module_state.items():
-                setattr(sys, k, v)
-        except AttributeError:
-            pass
-        try:
-            delattr(sys, 'ipcompleter')
-        except AttributeError:
-            pass
-        # Reset what what done in self.init_sys_modules
-        try:
-            sys.modules[self.user_ns['__name__']] = self._orig_sys_modules_main_name
-        except (AttributeError, KeyError):
-            pass
+        if not self.has_readline:
+            return func
 
-    def set_hook(self,name,hook, priority = 50, str_key = None, re_key = None):
-        """set_hook(name,hook) -> sets an internal IPython hook.
+        def wrapper():
+            self.savehist()
+            try:
+                func()
+            finally:
+                readline.read_history_file(self.histfile)
+        return wrapper
 
-        IPython exposes some of its internal API as user-modifiable hooks.  By
-        adding your function to one of these hooks, you can modify IPython's 
-        behavior to call at runtime your own routines."""
+    #-------------------------------------------------------------------------
+    # Things related to exception handling and tracebacks (not debugging)
+    #-------------------------------------------------------------------------
 
-        # At some point in the future, this should validate the hook before it
-        # accepts it.  Probably at least check that the hook takes the number
-        # of args it's supposed to.
+    def init_traceback_handlers(self, custom_exceptions):
+        # Syntax error handler.
+        self.SyntaxTB = SyntaxTB(color_scheme='NoColor')
         
-        f = new.instancemethod(hook,self,self.__class__)
+        # The interactive one is initialized with an offset, meaning we always
+        # want to remove the topmost item in the traceback, which is our own
+        # internal code. Valid modes: ['Plain','Context','Verbose']
+        self.InteractiveTB = ultratb.AutoFormattedTB(mode = 'Plain',
+                                                     color_scheme='NoColor',
+                                                     tb_offset = 1)
 
-        # check if the hook is for strdispatcher first
-        if str_key is not None:
-            sdp = self.strdispatchers.get(name, StrDispatch())
-            sdp.add_s(str_key, f, priority )
-            self.strdispatchers[name] = sdp
-            return
-        if re_key is not None:
-            sdp = self.strdispatchers.get(name, StrDispatch())
-            sdp.add_re(re.compile(re_key), f, priority )
-            self.strdispatchers[name] = sdp
-            return
-            
-        dp = getattr(self.hooks, name, None)
-        if name not in IPython.core.hooks.__all__:
-            print "Warning! Hook '%s' is not one of %s" % (name, IPython.core.hooks.__all__ )
-        if not dp:
-            dp = IPython.core.hooks.CommandChainDispatcher()
-        
-        try:
-            dp.add(f,priority)
-        except AttributeError:
-            # it was not commandchain, plain old func - replace
-            dp = f
+        # IPython itself shouldn't crash. This will produce a detailed
+        # post-mortem if it does.  But we only install the crash handler for
+        # non-threaded shells, the threaded ones use a normal verbose reporter
+        # and lose the crash handler.  This is because exceptions in the main
+        # thread (such as in GUI code) propagate directly to sys.excepthook,
+        # and there's no point in printing crash dumps for every user exception.
+        if self.isthreaded:
+            ipCrashHandler = ultratb.FormattedTB()
+        else:
+            from IPython.core import crashhandler
+            ipCrashHandler = crashhandler.IPythonCrashHandler(self)
+        self.set_crash_handler(ipCrashHandler)
 
-        setattr(self.hooks,name, dp)
+        # and add any custom exception handlers the user may have specified
+        self.set_custom_exc(*custom_exceptions)
 
     def set_crash_handler(self, crashHandler):
         """Set the IPython crash handler.
@@ -1177,192 +1216,169 b' class InteractiveShell(Component, Magic):'
         self.CustomTB = new.instancemethod(handler,self,self.__class__)
         self.custom_exceptions = exc_tuple
 
-    def set_custom_completer(self,completer,pos=0):
-        """set_custom_completer(completer,pos=0)
-
-        Adds a new custom completer function.
-
-        The position argument (defaults to 0) is the index in the completers
-        list where you want the completer to be inserted."""
-
-        newcomp = new.instancemethod(completer,self.Completer,
-                                     self.Completer.__class__)
-        self.Completer.matchers.insert(pos,newcomp)
-
-    def set_completer(self):
-        """reset readline's completer to be our own."""
-        self.readline.set_completer(self.Completer.complete)
-        
-    def _get_call_pdb(self):
-        return self._call_pdb
-
-    def _set_call_pdb(self,val):
-
-        if val not in (0,1,False,True):
-            raise ValueError,'new call_pdb value must be boolean'
-
-        # store value in instance
-        self._call_pdb = val
-
-        # notify the actual exception handlers
-        self.InteractiveTB.call_pdb = val
-        if self.isthreaded:
-            try:
-                self.sys_excepthook.call_pdb = val
-            except:
-                warn('Failed to activate pdb for threaded exception handler')
-
-    call_pdb = property(_get_call_pdb,_set_call_pdb,None,
-                        'Control auto-activation of pdb at exceptions')
-
-    def magic(self,arg_s):
-        """Call a magic function by name.
+    def excepthook(self, etype, value, tb):
+      """One more defense for GUI apps that call sys.excepthook.
 
-        Input: a string containing the name of the magic function to call and any
-        additional arguments to be passed to the magic.
+      GUI frameworks like wxPython trap exceptions and call
+      sys.excepthook themselves.  I guess this is a feature that
+      enables them to keep running after exceptions that would
+      otherwise kill their mainloop. This is a bother for IPython
+      which excepts to catch all of the program exceptions with a try:
+      except: statement.
 
-        magic('name -opt foo bar') is equivalent to typing at the ipython
-        prompt:
+      Normally, IPython sets sys.excepthook to a CrashHandler instance, so if
+      any app directly invokes sys.excepthook, it will look to the user like
+      IPython crashed.  In order to work around this, we can disable the
+      CrashHandler and replace it with this excepthook instead, which prints a
+      regular traceback using our InteractiveTB.  In this fashion, apps which
+      call sys.excepthook will generate a regular-looking exception from
+      IPython, and the CrashHandler will only be triggered by real IPython
+      crashes.
 
-        In[1]: %name -opt foo bar
+      This hook should be used sparingly, only in places which are not likely
+      to be true IPython errors.
+      """
+      self.showtraceback((etype,value,tb),tb_offset=0)
 
-        To call a magic without arguments, simply use magic('name').
+    def showtraceback(self,exc_tuple = None,filename=None,tb_offset=None):
+        """Display the exception that just occurred.
 
-        This provides a proper Python function to call IPython's magics in any
-        valid Python code you can type at the interpreter, including loops and
-        compound statements.
-        """
+        If nothing is known about the exception, this is the method which
+        should be used throughout the code for presenting user tracebacks,
+        rather than directly invoking the InteractiveTB object.
 
-        args = arg_s.split(' ',1)
-        magic_name = args[0]
-        magic_name = magic_name.lstrip(self.ESC_MAGIC)
+        A specific showsyntaxerror() also exists, but this method can take
+        care of calling it if needed, so unless you are explicitly catching a
+        SyntaxError exception, don't try to analyze the stack manually and
+        simply call this method."""
 
+        
+        # Though this won't be called by syntax errors in the input line,
+        # there may be SyntaxError cases whith imported code.
+        
         try:
-            magic_args = args[1]
-        except IndexError:
-            magic_args = ''
-        fn = getattr(self,'magic_'+magic_name,None)
-        if fn is None:
-            error("Magic function `%s` not found." % magic_name)
-        else:
-            magic_args = self.var_expand(magic_args,1)
-            with nested(self.builtin_trap, self.display_trap):
-                return fn(magic_args)
-            # return result
-
-    def define_magic(self, magicname, func):
-        """Expose own function as magic function for ipython 
+            if exc_tuple is None:
+                etype, value, tb = sys.exc_info()
+            else:
+                etype, value, tb = exc_tuple
     
-        def foo_impl(self,parameter_s=''):
-            'My very own magic!. (Use docstrings, IPython reads them).'
-            print 'Magic function. Passed parameter is between < >:'
-            print '<%s>' % parameter_s
-            print 'The self object is:',self
+            if etype is SyntaxError:
+                self.showsyntaxerror(filename)
+            elif etype is UsageError:
+                print "UsageError:", value
+            else:
+                # WARNING: these variables are somewhat deprecated and not
+                # necessarily safe to use in a threaded environment, but tools
+                # like pdb depend on their existence, so let's set them.  If we
+                # find problems in the field, we'll need to revisit their use.
+                sys.last_type = etype
+                sys.last_value = value
+                sys.last_traceback = tb
     
-        self.define_magic('foo',foo_impl)
-        """
-        
-        import new
-        im = new.instancemethod(func,self, self.__class__)
-        old = getattr(self, "magic_" + magicname, None)
-        setattr(self, "magic_" + magicname, im)
-        return old
-
-    def define_macro(self, name, themacro):
-        """Define a new macro
-
-        Parameters
-        ----------
-        name : str
-            The name of the macro.
-        themacro : str or Macro
-            The action to do upon invoking the macro.  If a string, a new 
-            Macro object is created by passing the string to it.
-        """
-        
-        from IPython.core import macro
+                if etype in self.custom_exceptions:
+                    self.CustomTB(etype,value,tb)
+                else:
+                    self.InteractiveTB(etype,value,tb,tb_offset=tb_offset)
+                    if self.InteractiveTB.call_pdb and self.has_readline:
+                        # pdb mucks up readline, fix it back
+                        self.set_completer()
+        except KeyboardInterrupt:
+            self.write("\nKeyboardInterrupt\n")
 
-        if isinstance(themacro, basestring):
-            themacro = macro.Macro(themacro)
-        if not isinstance(themacro, macro.Macro):
-            raise ValueError('A macro must be a string or a Macro instance.')
-        self.user_ns[name] = themacro
+    def showsyntaxerror(self, filename=None):
+        """Display the syntax error that just occurred.
 
-    def define_alias(self, name, cmd):
-        """ Define a new alias."""
+        This doesn't display a stack trace because there isn't one.
 
-        if callable(cmd):
-            self.alias_table[name] = cmd
-            from IPython.core import shadowns
-            setattr(shadowns, name, cmd)
-            return
+        If a filename is given, it is stuffed in the exception instead
+        of what was there before (because Python's parser always uses
+        "<string>" when reading from a string).
+        """
+        etype, value, last_traceback = sys.exc_info()
 
-        if isinstance(cmd, basestring):
-            nargs = cmd.count('%s')
-            if nargs>0 and cmd.find('%l')>=0:
-                raise Exception('The %s and %l specifiers are mutually '
-                                'exclusive in alias definitions.')
-                  
-            self.alias_table[name] = (nargs,cmd)
-            return
+        # See note about these variables in showtraceback() below
+        sys.last_type = etype
+        sys.last_value = value
+        sys.last_traceback = last_traceback
         
-        self.alias_table[name] = cmd
-
-    def ipalias(self,arg_s):
-        """Call an alias by name.
-
-        Input: a string containing the name of the alias to call and any
-        additional arguments to be passed to the magic.
+        if filename and etype is SyntaxError:
+            # Work hard to stuff the correct filename in the exception
+            try:
+                msg, (dummy_filename, lineno, offset, line) = value
+            except:
+                # Not the format we expect; leave it alone
+                pass
+            else:
+                # Stuff in the right filename
+                try:
+                    # Assume SyntaxError is a class exception
+                    value = SyntaxError(msg, (filename, lineno, offset, line))
+                except:
+                    # If that failed, assume SyntaxError is a string
+                    value = msg, (filename, lineno, offset, line)
+        self.SyntaxTB(etype,value,[])
 
-        ipalias('name -opt foo bar') is equivalent to typing at the ipython
-        prompt:
+    def edit_syntax_error(self):
+        """The bottom half of the syntax error handler called in the main loop.
 
-        In[1]: name -opt foo bar
+        Loop until syntax error is fixed or user cancels.
+        """
 
-        To call an alias without arguments, simply use ipalias('name').
+        while self.SyntaxTB.last_syntax_error:
+            # copy and clear last_syntax_error
+            err = self.SyntaxTB.clear_err_state()
+            if not self._should_recompile(err):
+                return
+            try:
+                # may set last_syntax_error again if a SyntaxError is raised
+                self.safe_execfile(err.filename,self.user_ns)
+            except:
+                self.showtraceback()
+            else:
+                try:
+                    f = file(err.filename)
+                    try:
+                        # This should be inside a display_trap block and I 
+                        # think it is.
+                        sys.displayhook(f.read())
+                    finally:
+                        f.close()
+                except:
+                    self.showtraceback()
 
-        This provides a proper Python function to call IPython's aliases in any
-        valid Python code you can type at the interpreter, including loops and
-        compound statements.  It is added by IPython to the Python builtin
-        namespace upon initialization."""
+    def _should_recompile(self,e):
+        """Utility routine for edit_syntax_error"""
 
-        args = arg_s.split(' ',1)
-        alias_name = args[0]
+        if e.filename in ('<ipython console>','<input>','<string>',
+                          '<console>','<BackgroundJob compilation>',
+                          None):
+                              
+            return False
         try:
-            alias_args = args[1]
-        except IndexError:
-            alias_args = ''
-        if alias_name in self.alias_table:
-            self.call_alias(alias_name,alias_args)
-        else:
-            error("Alias `%s` not found." % alias_name)
-
-    def system(self, cmd):
-        """Make a system call, using IPython."""
-        return self.hooks.shell_hook(self.var_expand(cmd, depth=2))
-
-    def ex(self, cmd):
-        """Execute a normal python statement in user namespace."""
-        with nested(self.builtin_trap, self.display_trap):
-            exec cmd in self.user_global_ns, self.user_ns
-
-    def ev(self, expr):
-        """Evaluate python expression expr in user namespace.
-
-        Returns the result of evaluation
-        """
-        with nested(self.builtin_trap, self.display_trap):
-            return eval(expr, self.user_global_ns, self.user_ns)
+            if (self.autoedit_syntax and 
+                not self.ask_yes_no('Return to editor to correct syntax error? '
+                              '[Y/n] ','y')):
+                return False
+        except EOFError:
+            return False
 
-    def getoutput(self, cmd):
-        return getoutput(self.var_expand(cmd,depth=2),
-                         header=self.system_header,
-                         verbose=self.system_verbose)
+        def int0(x):
+            try:
+                return int(x)
+            except TypeError:
+                return 0
+        # always pass integer line and offset values to editor hook
+        try:
+            self.hooks.fix_error_editor(e.filename,
+                int0(e.lineno),int0(e.offset),e.msg)
+        except TryNext:
+            warn('Could not open editor')
+            return False
+        return True
 
-    def getoutputerror(self, cmd):
-        return getoutputerror(self.var_expand(cmd,depth=2),
-                              header=self.system_header,
-                              verbose=self.system_verbose)
+    #-------------------------------------------------------------------------
+    # Things related to tab completion
+    #-------------------------------------------------------------------------
 
     def complete(self, text):
         """Return a sorted list of all possible completions on text.
@@ -1409,35 +1425,112 b' class InteractiveShell(Component, Magic):'
             #print "T:",text,"OC:",outcomps  # dbg
             #print "vars:",self.user_ns.keys()
             return outcomps
-        
-    def set_completer_frame(self, frame=None):
-        if frame:
-            self.Completer.namespace = frame.f_locals
-            self.Completer.global_namespace = frame.f_globals
-        else:
-            self.Completer.namespace = self.user_ns
-            self.Completer.global_namespace = self.user_global_ns
 
-    def init_auto_alias(self):
-        """Define some aliases automatically.
+    def set_custom_completer(self,completer,pos=0):
+        """set_custom_completer(completer,pos=0)
 
-        These are ALL parameter-less aliases"""
+        Adds a new custom completer function.
 
-        for alias,cmd in self.auto_alias:
-            self.define_alias(alias,cmd)
+        The position argument (defaults to 0) is the index in the completers
+        list where you want the completer to be inserted."""
 
-    def alias_table_validate(self,verbose=0):
-        """Update information about the alias table.
+        newcomp = new.instancemethod(completer,self.Completer,
+                                     self.Completer.__class__)
+        self.Completer.matchers.insert(pos,newcomp)
 
-        In particular, make sure no Python keywords/builtins are in it."""
+    def set_completer(self):
+        """reset readline's completer to be our own."""
+        self.readline.set_completer(self.Completer.complete)
+
+    #-------------------------------------------------------------------------
+    # Things related to readline
+    #-------------------------------------------------------------------------
+
+    def init_readline(self):
+        """Command history completion/saving/reloading."""
+
+        self.rl_next_input = None
+        self.rl_do_indent = False
+
+        if not self.readline_use:
+            return
+
+        import IPython.utils.rlineimpl as readline
+                  
+        if not readline.have_readline:
+            self.has_readline = 0
+            self.readline = None
+            # no point in bugging windows users with this every time:
+            warn('Readline services not available on this platform.')
+        else:
+            sys.modules['readline'] = readline
+            import atexit
+            from IPython.core.completer import IPCompleter
+            self.Completer = IPCompleter(self,
+                                            self.user_ns,
+                                            self.user_global_ns,
+                                            self.readline_omit__names,
+                                            self.alias_table)
+            sdisp = self.strdispatchers.get('complete_command', StrDispatch())
+            self.strdispatchers['complete_command'] = sdisp
+            self.Completer.custom_completers = sdisp
+            # Platform-specific configuration
+            if os.name == 'nt':
+                self.readline_startup_hook = readline.set_pre_input_hook
+            else:
+                self.readline_startup_hook = readline.set_startup_hook
+
+            # Load user's initrc file (readline config)
+            # Or if libedit is used, load editrc.
+            inputrc_name = os.environ.get('INPUTRC')
+            if inputrc_name is None:
+                home_dir = get_home_dir()
+                if home_dir is not None:
+                    inputrc_name = '.inputrc'
+                    if readline.uses_libedit:
+                        inputrc_name = '.editrc'
+                    inputrc_name = os.path.join(home_dir, inputrc_name)
+            if os.path.isfile(inputrc_name):
+                try:
+                    readline.read_init_file(inputrc_name)
+                except:
+                    warn('Problems reading readline initialization file <%s>'
+                         % inputrc_name)
+            
+            self.has_readline = 1
+            self.readline = readline
+            # save this in sys so embedded copies can restore it properly
+            sys.ipcompleter = self.Completer.complete
+            self.set_completer()
+
+            # Configure readline according to user's prefs
+            # This is only done if GNU readline is being used.  If libedit
+            # is being used (as on Leopard) the readline config is
+            # not run as the syntax for libedit is different.
+            if not readline.uses_libedit:
+                for rlcommand in self.readline_parse_and_bind:
+                    #print "loading rl:",rlcommand  # dbg
+                    readline.parse_and_bind(rlcommand)
+
+            # Remove some chars from the delimiters list.  If we encounter
+            # unicode chars, discard them.
+            delims = readline.get_completer_delims().encode("ascii", "ignore")
+            delims = delims.translate(string._idmap,
+                                      self.readline_remove_delims)
+            readline.set_completer_delims(delims)
+            # otherwise we end up with a monster history after a while:
+            readline.set_history_length(1000)
+            try:
+                #print '*** Reading readline history'  # dbg
+                readline.read_history_file(self.histfile)
+            except IOError:
+                pass  # It doesn't exist yet.
+
+            atexit.register(self.atexit_operations)
+            del atexit
 
-        no_alias = self.no_alias
-        for k in self.alias_table.keys():
-            if k in no_alias:
-                del self.alias_table[k]
-                if verbose:
-                    print ("Deleting alias <%s>, it's a Python "
-                           "keyword or builtin." % k)
+        # Configure auto-indent for all platforms
+        self.set_autoindent(self.autoindent)
 
     def set_next_input(self, s):
         """ Sets the 'default' input string for the next command line.
@@ -1452,348 +1545,369 b' class InteractiveShell(Component, Magic):'
 
         self.rl_next_input = s
 
-    def set_autoindent(self,value=None):
-        """Set the autoindent flag, checking for readline support.
+    def pre_readline(self):
+        """readline hook to be used at the start of each line.
 
-        If called with no arguments, it acts as a toggle."""
+        Currently it handles auto-indent only."""
 
-        if not self.has_readline:
-            if os.name == 'posix':
-                warn("The auto-indent feature requires the readline library")
-            self.autoindent = 0
-            return
-        if value is None:
-            self.autoindent = not self.autoindent
-        else:
-            self.autoindent = value
+        #debugx('self.indent_current_nsp','pre_readline:')
 
-    def atexit_operations(self):
-        """This will be executed at the time of exit.
+        if self.rl_do_indent:
+            self.readline.insert_text(self._indent_current_str())
+        if self.rl_next_input is not None:
+            self.readline.insert_text(self.rl_next_input)
+            self.rl_next_input = None
 
-        Saving of persistent data should be performed here. """
+    def _indent_current_str(self):
+        """return the current level of indentation as a string"""
+        return self.indent_current_nsp * ' '
 
-        #print '*** IPython exit cleanup ***' # dbg
-        # input history
-        self.savehist()
+    #-------------------------------------------------------------------------
+    # Things related to magics
+    #-------------------------------------------------------------------------
 
-        # Cleanup all tempfiles left around
-        for tfile in self.tempfiles:
-            try:
-                os.unlink(tfile)
-            except OSError:
-                pass
+    def init_magics(self):
+        # Set user colors (don't do it in the constructor above so that it
+        # doesn't crash if colors option is invalid)
+        self.magic_colors(self.colors)
 
-        # Clear all user namespaces to release all references cleanly.
-        self.reset()
+    def magic(self,arg_s):
+        """Call a magic function by name.
 
-        # Run user hooks
-        self.hooks.shutdown_hook()
+        Input: a string containing the name of the magic function to call and any
+        additional arguments to be passed to the magic.
 
-    def reset(self):
-        """Clear all internal namespaces.
+        magic('name -opt foo bar') is equivalent to typing at the ipython
+        prompt:
 
-        Note that this is much more aggressive than %reset, since it clears
-        fully all namespaces, as well as all input/output lists.
-        """
-        for ns in self.ns_refs_table:
-            ns.clear()
+        In[1]: %name -opt foo bar
 
-        # Clear input and output histories
-        self.input_hist[:] = []
-        self.input_hist_raw[:] = []
-        self.output_hist.clear()
-        # Restore the user namespaces to minimal usability
-        self.init_user_ns()
-        
-    def savehist(self):
-        """Save input history to a file (via readline library)."""
+        To call a magic without arguments, simply use magic('name').
 
-        if not self.has_readline:
-            return
-        
-        try:
-            self.readline.write_history_file(self.histfile)
-        except:
-            print 'Unable to save IPython command history to file: ' + \
-                  `self.histfile`
+        This provides a proper Python function to call IPython's magics in any
+        valid Python code you can type at the interpreter, including loops and
+        compound statements.
+        """
 
-    def reloadhist(self):
-        """Reload the input history from disk file."""
+        args = arg_s.split(' ',1)
+        magic_name = args[0]
+        magic_name = magic_name.lstrip(self.ESC_MAGIC)
 
-        if self.has_readline:
-            try:
-                self.readline.clear_history()
-                self.readline.read_history_file(self.shell.histfile)
-            except AttributeError:
-                pass
-            
+        try:
+            magic_args = args[1]
+        except IndexError:
+            magic_args = ''
+        fn = getattr(self,'magic_'+magic_name,None)
+        if fn is None:
+            error("Magic function `%s` not found." % magic_name)
+        else:
+            magic_args = self.var_expand(magic_args,1)
+            with nested(self.builtin_trap, self.display_trap):
+                return fn(magic_args)
+            # return result
 
-    def history_saving_wrapper(self, func):
-        """ Wrap func for readline history saving
-        
-        Convert func into callable that saves & restores
-        history around the call """
-        
-        if not self.has_readline:
-            return func
+    def define_magic(self, magicname, func):
+        """Expose own function as magic function for ipython 
+    
+        def foo_impl(self,parameter_s=''):
+            'My very own magic!. (Use docstrings, IPython reads them).'
+            print 'Magic function. Passed parameter is between < >:'
+            print '<%s>' % parameter_s
+            print 'The self object is:',self
+    
+        self.define_magic('foo',foo_impl)
+        """
         
-        def wrapper():
-            self.savehist()
-            try:
-                func()
-            finally:
-                readline.read_history_file(self.histfile)
-        return wrapper
-            
-    def pre_readline(self):
-        """readline hook to be used at the start of each line.
-
-        Currently it handles auto-indent only."""
-
-        #debugx('self.indent_current_nsp','pre_readline:')
+        import new
+        im = new.instancemethod(func,self, self.__class__)
+        old = getattr(self, "magic_" + magicname, None)
+        setattr(self, "magic_" + magicname, im)
+        return old
 
-        if self.rl_do_indent:
-            self.readline.insert_text(self.indent_current_str())
-        if self.rl_next_input is not None:
-            self.readline.insert_text(self.rl_next_input)
-            self.rl_next_input = None
+    #-------------------------------------------------------------------------
+    # Things related to macros
+    #-------------------------------------------------------------------------
 
-    def ask_yes_no(self,prompt,default=True):
-        if self.quiet:
-            return True
-        return ask_yes_no(prompt,default)
+    def define_macro(self, name, themacro):
+        """Define a new macro
 
-    def new_main_mod(self,ns=None):
-        """Return a new 'main' module object for user code execution.
+        Parameters
+        ----------
+        name : str
+            The name of the macro.
+        themacro : str or Macro
+            The action to do upon invoking the macro.  If a string, a new 
+            Macro object is created by passing the string to it.
         """
-        main_mod = self._user_main_module
-        init_fakemod_dict(main_mod,ns)
-        return main_mod
+        
+        from IPython.core import macro
 
-    def cache_main_mod(self,ns,fname):
-        """Cache a main module's namespace.
+        if isinstance(themacro, basestring):
+            themacro = macro.Macro(themacro)
+        if not isinstance(themacro, macro.Macro):
+            raise ValueError('A macro must be a string or a Macro instance.')
+        self.user_ns[name] = themacro
 
-        When scripts are executed via %run, we must keep a reference to the
-        namespace of their __main__ module (a FakeModule instance) around so
-        that Python doesn't clear it, rendering objects defined therein
-        useless.
+    #-------------------------------------------------------------------------
+    # Things related to the running of system commands
+    #-------------------------------------------------------------------------
 
-        This method keeps said reference in a private dict, keyed by the
-        absolute path of the module object (which corresponds to the script
-        path).  This way, for multiple executions of the same script we only
-        keep one copy of the namespace (the last one), thus preventing memory
-        leaks from old references while allowing the objects from the last
-        execution to be accessible.
+    def system(self, cmd):
+        """Make a system call, using IPython."""
+        return self.hooks.shell_hook(self.var_expand(cmd, depth=2))
 
-        Note: we can not allow the actual FakeModule instances to be deleted,
-        because of how Python tears down modules (it hard-sets all their
-        references to None without regard for reference counts).  This method
-        must therefore make a *copy* of the given namespace, to allow the
-        original module's __dict__ to be cleared and reused.
+    #-------------------------------------------------------------------------
+    # Things related to aliases
+    #-------------------------------------------------------------------------
+
+    def init_aliases(self):
+        # dict of things NOT to alias (keywords, builtins and some magics)
+        no_alias = {}
+        no_alias_magics = ['cd','popd','pushd','dhist','alias','unalias']
+        for key in keyword.kwlist + no_alias_magics:
+            no_alias[key] = 1
+        no_alias.update(__builtin__.__dict__)
+        self.no_alias = no_alias
 
+        # Make some aliases automatically
+        # Prepare list of shell aliases to auto-define
+        if os.name == 'posix':
+            auto_alias = ('mkdir mkdir', 'rmdir rmdir',
+                          'mv mv -i','rm rm -i','cp cp -i',
+                          'cat cat','less less','clear clear',
+                          # a better ls
+                          'ls ls -F',
+                          # long ls
+                          'll ls -lF')
+            # Extra ls aliases with color, which need special treatment on BSD
+            # variants
+            ls_extra = ( # color ls
+                         'lc ls -F -o --color',
+                         # ls normal files only
+                         'lf ls -F -o --color %l | grep ^-',
+                         # ls symbolic links
+                         'lk ls -F -o --color %l | grep ^l',
+                         # directories or links to directories,
+                         'ldir ls -F -o --color %l | grep /$',
+                         # things which are executable
+                         'lx ls -F -o --color %l | grep ^-..x',
+                         )
+            # The BSDs don't ship GNU ls, so they don't understand the
+            # --color switch out of the box
+            if 'bsd' in sys.platform:
+                ls_extra = ( # ls normal files only
+                             'lf ls -lF | grep ^-',
+                             # ls symbolic links
+                             'lk ls -lF | grep ^l',
+                             # directories or links to directories,
+                             'ldir ls -lF | grep /$',
+                             # things which are executable
+                             'lx ls -lF | grep ^-..x',
+                             )
+            auto_alias = auto_alias + ls_extra
+        elif os.name in ['nt','dos']:
+            auto_alias = ('ls dir /on',
+                          'ddir dir /ad /on', 'ldir dir /ad /on',
+                          'mkdir mkdir','rmdir rmdir','echo echo',
+                          'ren ren','cls cls','copy copy')
+        else:
+            auto_alias = ()
+        self.auto_alias = [s.split(None,1) for s in auto_alias]
         
-        Parameters
-        ----------
-          ns : a namespace (a dict, typically)
-
-          fname : str
-            Filename associated with the namespace.
+        # Load default aliases
+        for alias, cmd in self.auto_alias:
+            self.define_alias(alias,cmd)
 
-        Examples
-        --------
+        # Load user aliases
+        for alias in self.alias:
+            self.magic_alias(alias)
 
-        In [10]: import IPython
+    def call_alias(self,alias,rest=''):
+        """Call an alias given its name and the rest of the line.
 
-        In [11]: _ip.cache_main_mod(IPython.__dict__,IPython.__file__)
+        This is only used to provide backwards compatibility for users of
+        ipalias(), use of which is not recommended for anymore."""
 
-        In [12]: IPython.__file__ in _ip._main_ns_cache
-        Out[12]: True
-        """
-        self._main_ns_cache[os.path.abspath(fname)] = ns.copy()
+        # Now call the macro, evaluating in the user's namespace
+        cmd = self.transform_alias(alias, rest)
+        try:
+            self.system(cmd)
+        except:
+            self.showtraceback()
 
-    def clear_main_mod_cache(self):
-        """Clear the cache of main modules.
+    def define_alias(self, name, cmd):
+        """ Define a new alias."""
 
-        Mainly for use by utilities like %reset.
+        if callable(cmd):
+            self.alias_table[name] = cmd
+            from IPython.core import shadowns
+            setattr(shadowns, name, cmd)
+            return
 
-        Examples
-        --------
+        if isinstance(cmd, basestring):
+            nargs = cmd.count('%s')
+            if nargs>0 and cmd.find('%l')>=0:
+                raise Exception('The %s and %l specifiers are mutually '
+                                'exclusive in alias definitions.')
+                  
+            self.alias_table[name] = (nargs,cmd)
+            return
+        
+        self.alias_table[name] = cmd
 
-        In [15]: import IPython
+    def ipalias(self,arg_s):
+        """Call an alias by name.
 
-        In [16]: _ip.cache_main_mod(IPython.__dict__,IPython.__file__)
+        Input: a string containing the name of the alias to call and any
+        additional arguments to be passed to the magic.
 
-        In [17]: len(_ip._main_ns_cache) > 0
-        Out[17]: True
+        ipalias('name -opt foo bar') is equivalent to typing at the ipython
+        prompt:
 
-        In [18]: _ip.clear_main_mod_cache()
+        In[1]: name -opt foo bar
 
-        In [19]: len(_ip._main_ns_cache) == 0
-        Out[19]: True
-        """
-        self._main_ns_cache.clear()
+        To call an alias without arguments, simply use ipalias('name').
 
-    def _should_recompile(self,e):
-        """Utility routine for edit_syntax_error"""
+        This provides a proper Python function to call IPython's aliases in any
+        valid Python code you can type at the interpreter, including loops and
+        compound statements.  It is added by IPython to the Python builtin
+        namespace upon initialization."""
 
-        if e.filename in ('<ipython console>','<input>','<string>',
-                          '<console>','<BackgroundJob compilation>',
-                          None):
-                              
-            return False
+        args = arg_s.split(' ',1)
+        alias_name = args[0]
         try:
-            if (self.autoedit_syntax and 
-                not self.ask_yes_no('Return to editor to correct syntax error? '
-                              '[Y/n] ','y')):
-                return False
-        except EOFError:
-            return False
+            alias_args = args[1]
+        except IndexError:
+            alias_args = ''
+        if alias_name in self.alias_table:
+            self.call_alias(alias_name,alias_args)
+        else:
+            error("Alias `%s` not found." % alias_name)
 
-        def int0(x):
-            try:
-                return int(x)
-            except TypeError:
-                return 0
-        # always pass integer line and offset values to editor hook
-        try:
-            self.hooks.fix_error_editor(e.filename,
-                int0(e.lineno),int0(e.offset),e.msg)
-        except TryNext:
-            warn('Could not open editor')
-            return False
-        return True
+    def expand_alias(self, line):
+        """ Expand an alias in the command line 
         
-    def edit_syntax_error(self):
-        """The bottom half of the syntax error handler called in the main loop.
+        Returns the provided command line, possibly with the first word 
+        (command) translated according to alias expansion rules.
+        
+        [ipython]|16> _ip.expand_aliases("np myfile.txt")
+                 <16> 'q:/opt/np/notepad++.exe myfile.txt'
+        """
+        
+        pre,fn,rest = self.split_user_input(line)
+        res = pre + self.expand_aliases(fn, rest)
+        return res
 
-        Loop until syntax error is fixed or user cancels.
+    def expand_aliases(self, fn, rest):
+        """Expand multiple levels of aliases:
+        
+        if:
+        
+        alias foo bar /tmp
+        alias baz foo
+        
+        then:
+        
+        baz huhhahhei -> bar /tmp huhhahhei
+        
         """
+        line = fn + " " + rest
+        
+        done = set()
+        while 1:
+            pre,fn,rest = prefilter.splitUserInput(line,
+                                                   prefilter.shell_line_split)
+            if fn in self.alias_table:
+                if fn in done:
+                    warn("Cyclic alias definition, repeated '%s'" % fn)
+                    return ""
+                done.add(fn)
 
-        while self.SyntaxTB.last_syntax_error:
-            # copy and clear last_syntax_error
-            err = self.SyntaxTB.clear_err_state()
-            if not self._should_recompile(err):
-                return
-            try:
-                # may set last_syntax_error again if a SyntaxError is raised
-                self.safe_execfile(err.filename,self.user_ns)
-            except:
-                self.showtraceback()
+                l2 = self.transform_alias(fn,rest)
+                # dir -> dir 
+                # print "alias",line, "->",l2  #dbg
+                if l2 == line:
+                    break
+                # ls -> ls -F should not recurse forever
+                if l2.split(None,1)[0] == line.split(None,1)[0]:
+                    line = l2
+                    break
+                
+                line=l2
+                
+                
+                # print "al expand to",line #dbg
             else:
-                try:
-                    f = file(err.filename)
-                    try:
-                        # This should be inside a display_trap block and I 
-                        # think it is.
-                        sys.displayhook(f.read())
-                    finally:
-                        f.close()
-                except:
-                    self.showtraceback()
-
-    def showsyntaxerror(self, filename=None):
-        """Display the syntax error that just occurred.
-
-        This doesn't display a stack trace because there isn't one.
+                break
+                
+        return line
 
-        If a filename is given, it is stuffed in the exception instead
-        of what was there before (because Python's parser always uses
-        "<string>" when reading from a string).
+    def transform_alias(self, alias,rest=''):
+        """ Transform alias to system command string.
         """
-        etype, value, last_traceback = sys.exc_info()
+        trg = self.alias_table[alias]
 
-        # See note about these variables in showtraceback() below
-        sys.last_type = etype
-        sys.last_value = value
-        sys.last_traceback = last_traceback
-        
-        if filename and etype is SyntaxError:
-            # Work hard to stuff the correct filename in the exception
-            try:
-                msg, (dummy_filename, lineno, offset, line) = value
-            except:
-                # Not the format we expect; leave it alone
-                pass
-            else:
-                # Stuff in the right filename
-                try:
-                    # Assume SyntaxError is a class exception
-                    value = SyntaxError(msg, (filename, lineno, offset, line))
-                except:
-                    # If that failed, assume SyntaxError is a string
-                    value = msg, (filename, lineno, offset, line)
-        self.SyntaxTB(etype,value,[])
+        nargs,cmd = trg
+        # print trg #dbg
+        if ' ' in cmd and os.path.isfile(cmd):
+            cmd = '"%s"' % cmd
 
-    def debugger(self,force=False):
-        """Call the pydb/pdb debugger.
+        # Expand the %l special to be the user's input line
+        if cmd.find('%l') >= 0:
+            cmd = cmd.replace('%l',rest)
+            rest = ''
+        if nargs==0:
+            # Simple, argument-less aliases
+            cmd = '%s %s' % (cmd,rest)
+        else:
+            # Handle aliases with positional arguments
+            args = rest.split(None,nargs)
+            if len(args)< nargs:
+                error('Alias <%s> requires %s arguments, %s given.' %
+                      (alias,nargs,len(args)))
+                return None
+            cmd = '%s %s' % (cmd % tuple(args[:nargs]),' '.join(args[nargs:]))
+        # Now call the macro, evaluating in the user's namespace
+        #print 'new command: <%r>' % cmd  # dbg
+        return cmd
 
-        Keywords:
+    def init_auto_alias(self):
+        """Define some aliases automatically.
 
-          - force(False): by default, this routine checks the instance call_pdb
-          flag and does not actually invoke the debugger if the flag is false.
-          The 'force' option forces the debugger to activate even if the flag
-          is false.
-        """
+        These are ALL parameter-less aliases"""
 
-        if not (force or self.call_pdb):
-            return
+        for alias,cmd in self.auto_alias:
+            self.define_alias(alias,cmd)
+
+    def alias_table_validate(self,verbose=0):
+        """Update information about the alias table.
 
-        if not hasattr(sys,'last_traceback'):
-            error('No traceback has been produced, nothing to debug.')
-            return
+        In particular, make sure no Python keywords/builtins are in it."""
 
-        # use pydb if available
-        if debugger.has_pydb:
-            from pydb import pm
-        else:
-            # fallback to our internal debugger
-            pm = lambda : self.InteractiveTB.debugger(force=True)
-        self.history_saving_wrapper(pm)()
+        no_alias = self.no_alias
+        for k in self.alias_table.keys():
+            if k in no_alias:
+                del self.alias_table[k]
+                if verbose:
+                    print ("Deleting alias <%s>, it's a Python "
+                           "keyword or builtin." % k)
 
-    def showtraceback(self,exc_tuple = None,filename=None,tb_offset=None):
-        """Display the exception that just occurred.
+    #-------------------------------------------------------------------------
+    # Things related to the running of code
+    #-------------------------------------------------------------------------
 
-        If nothing is known about the exception, this is the method which
-        should be used throughout the code for presenting user tracebacks,
-        rather than directly invoking the InteractiveTB object.
+    def ex(self, cmd):
+        """Execute a normal python statement in user namespace."""
+        with nested(self.builtin_trap, self.display_trap):
+            exec cmd in self.user_global_ns, self.user_ns
 
-        A specific showsyntaxerror() also exists, but this method can take
-        care of calling it if needed, so unless you are explicitly catching a
-        SyntaxError exception, don't try to analyze the stack manually and
-        simply call this method."""
+    def ev(self, expr):
+        """Evaluate python expression expr in user namespace.
 
-        
-        # Though this won't be called by syntax errors in the input line,
-        # there may be SyntaxError cases whith imported code.
-        
-        try:
-            if exc_tuple is None:
-                etype, value, tb = sys.exc_info()
-            else:
-                etype, value, tb = exc_tuple
-    
-            if etype is SyntaxError:
-                self.showsyntaxerror(filename)
-            elif etype is UsageError:
-                print "UsageError:", value
-            else:
-                # WARNING: these variables are somewhat deprecated and not
-                # necessarily safe to use in a threaded environment, but tools
-                # like pdb depend on their existence, so let's set them.  If we
-                # find problems in the field, we'll need to revisit their use.
-                sys.last_type = etype
-                sys.last_value = value
-                sys.last_traceback = tb
-    
-                if etype in self.custom_exceptions:
-                    self.CustomTB(etype,value,tb)
-                else:
-                    self.InteractiveTB(etype,value,tb,tb_offset=tb_offset)
-                    if self.InteractiveTB.call_pdb and self.has_readline:
-                        # pdb mucks up readline, fix it back
-                        self.set_completer()
-        except KeyboardInterrupt:
-            self.write("\nKeyboardInterrupt\n")
+        Returns the result of evaluation
+        """
+        with nested(self.builtin_trap, self.display_trap):
+            return eval(expr, self.user_global_ns, self.user_ns)
 
     def mainloop(self, banner=None):
         """Start the mainloop.
@@ -1978,200 +2092,170 b' class InteractiveShell(Component, Magic):'
         # We are off again...
         __builtin__.__dict__['__IPYTHON__active'] -= 1
 
-    def excepthook(self, etype, value, tb):
-      """One more defense for GUI apps that call sys.excepthook.
-
-      GUI frameworks like wxPython trap exceptions and call
-      sys.excepthook themselves.  I guess this is a feature that
-      enables them to keep running after exceptions that would
-      otherwise kill their mainloop. This is a bother for IPython
-      which excepts to catch all of the program exceptions with a try:
-      except: statement.
-
-      Normally, IPython sets sys.excepthook to a CrashHandler instance, so if
-      any app directly invokes sys.excepthook, it will look to the user like
-      IPython crashed.  In order to work around this, we can disable the
-      CrashHandler and replace it with this excepthook instead, which prints a
-      regular traceback using our InteractiveTB.  In this fashion, apps which
-      call sys.excepthook will generate a regular-looking exception from
-      IPython, and the CrashHandler will only be triggered by real IPython
-      crashes.
-
-      This hook should be used sparingly, only in places which are not likely
-      to be true IPython errors.
-      """
-      self.showtraceback((etype,value,tb),tb_offset=0)
+    def safe_execfile(self,fname,*where,**kw):
+        """A safe version of the builtin execfile().
 
-    def expand_alias(self, line):
-        """ Expand an alias in the command line 
-        
-        Returns the provided command line, possibly with the first word 
-        (command) translated according to alias expansion rules.
-        
-        [ipython]|16> _ip.expand_aliases("np myfile.txt")
-                 <16> 'q:/opt/np/notepad++.exe myfile.txt'
-        """
-        
-        pre,fn,rest = self.split_user_input(line)
-        res = pre + self.expand_aliases(fn, rest)
-        return res
+        This version will never throw an exception, and knows how to handle
+        ipython logs as well.
 
-    def expand_aliases(self, fn, rest):
-        """Expand multiple levels of aliases:
-        
-        if:
-        
-        alias foo bar /tmp
-        alias baz foo
-        
-        then:
-        
-        baz huhhahhei -> bar /tmp huhhahhei
-        
-        """
-        line = fn + " " + rest
-        
-        done = set()
-        while 1:
-            pre,fn,rest = prefilter.splitUserInput(line,
-                                                   prefilter.shell_line_split)
-            if fn in self.alias_table:
-                if fn in done:
-                    warn("Cyclic alias definition, repeated '%s'" % fn)
-                    return ""
-                done.add(fn)
+        :Parameters:
+          fname : string
+            Name of the file to be executed.
+            
+          where : tuple
+            One or two namespaces, passed to execfile() as (globals,locals).
+            If only one is given, it is passed as both.
 
-                l2 = self.transform_alias(fn,rest)
-                # dir -> dir 
-                # print "alias",line, "->",l2  #dbg
-                if l2 == line:
-                    break
-                # ls -> ls -F should not recurse forever
-                if l2.split(None,1)[0] == line.split(None,1)[0]:
-                    line = l2
-                    break
-                
-                line=l2
-                
-                
-                # print "al expand to",line #dbg
-            else:
-                break
-                
-        return line
+        :Keywords:
+          islog : boolean (False)
 
-    def transform_alias(self, alias,rest=''):
-        """ Transform alias to system command string.
-        """
-        trg = self.alias_table[alias]
+          quiet : boolean (True)
 
-        nargs,cmd = trg
-        # print trg #dbg
-        if ' ' in cmd and os.path.isfile(cmd):
-            cmd = '"%s"' % cmd
+          exit_ignore : boolean (False)
+          """
 
-        # Expand the %l special to be the user's input line
-        if cmd.find('%l') >= 0:
-            cmd = cmd.replace('%l',rest)
-            rest = ''
-        if nargs==0:
-            # Simple, argument-less aliases
-            cmd = '%s %s' % (cmd,rest)
-        else:
-            # Handle aliases with positional arguments
-            args = rest.split(None,nargs)
-            if len(args)< nargs:
-                error('Alias <%s> requires %s arguments, %s given.' %
-                      (alias,nargs,len(args)))
-                return None
-            cmd = '%s %s' % (cmd % tuple(args[:nargs]),' '.join(args[nargs:]))
-        # Now call the macro, evaluating in the user's namespace
-        #print 'new command: <%r>' % cmd  # dbg
-        return cmd
+        def syspath_cleanup():
+            """Internal cleanup routine for sys.path."""
+            if add_dname:
+                try:
+                    sys.path.remove(dname)
+                except ValueError:
+                    # For some reason the user has already removed it, ignore.
+                    pass
         
-    def call_alias(self,alias,rest=''):
-        """Call an alias given its name and the rest of the line.
-
-        This is only used to provide backwards compatibility for users of
-        ipalias(), use of which is not recommended for anymore."""
-
-        # Now call the macro, evaluating in the user's namespace
-        cmd = self.transform_alias(alias, rest)
-        try:
-            self.system(cmd)
-        except:
-            self.showtraceback()
-
-    def indent_current_str(self):
-        """return the current level of indentation as a string"""
-        return self.indent_current_nsp * ' '
-
-    def autoindent_update(self,line):
-        """Keep track of the indent level."""
-
-        #debugx('line')
-        #debugx('self.indent_current_nsp')
-        if self.autoindent:
-            if line:
-                inisp = num_ini_spaces(line)
-                if inisp < self.indent_current_nsp:
-                    self.indent_current_nsp = inisp
-
-                if line[-1] == ':':
-                    self.indent_current_nsp += 4
-                elif dedent_re.match(line):
-                    self.indent_current_nsp -= 4
-            else:
-                self.indent_current_nsp = 0
-
-    def push(self, variables, interactive=True):
-        """Inject a group of variables into the IPython user namespace.
+        fname = os.path.expanduser(fname)
 
-        Parameters
-        ----------
-        variables : dict, str or list/tuple of str
-            The variables to inject into the user's namespace.  If a dict,
-            a simple update is done.  If a str, the string is assumed to 
-            have variable names separated by spaces.  A list/tuple of str
-            can also be used to give the variable names.  If just the variable
-            names are give (list/tuple/str) then the variable values looked
-            up in the callers frame.
-        interactive : bool
-            If True (default), the variables will be listed with the ``who``
-            magic.
-        """
-        vdict = None
+        # Find things also in current directory.  This is needed to mimic the
+        # behavior of running a script from the system command line, where
+        # Python inserts the script's directory into sys.path
+        dname = os.path.dirname(os.path.abspath(fname))
+        add_dname = False
+        if dname not in sys.path:
+            sys.path.insert(0,dname)
+            add_dname = True
 
-        # We need a dict of name/value pairs to do namespace updates.
-        if isinstance(variables, dict):
-            vdict = variables
-        elif isinstance(variables, (basestring, list, tuple)):
-            if isinstance(variables, basestring):
-                vlist = variables.split()
-            else:
-                vlist = variables
-            vdict = {}
-            cf = sys._getframe(1)
-            for name in vlist:
+        try:
+            xfile = open(fname)
+        except:
+            print >> Term.cerr, \
+                  'Could not open file <%s> for safe execution.' % fname
+            syspath_cleanup()
+            return None
+
+        kw.setdefault('islog',0)
+        kw.setdefault('quiet',1)
+        kw.setdefault('exit_ignore',0)
+        
+        first = xfile.readline()
+        loghead = str(self.loghead_tpl).split('\n',1)[0].strip()
+        xfile.close()
+        # line by line execution
+        if first.startswith(loghead) or kw['islog']:
+            print 'Loading log file <%s> one line at a time...' % fname
+            if kw['quiet']:
+                stdout_save = sys.stdout
+                sys.stdout = StringIO.StringIO()
+            try:
+                globs,locs = where[0:2]
+            except:
                 try:
-                    vdict[name] = eval(name, cf.f_globals, cf.f_locals)
+                    globs = locs = where[0]
                 except:
-                    print ('Could not get variable %s from %s' %
-                           (name,cf.f_code.co_name))
-        else:
-            raise ValueError('variables must be a dict/str/list/tuple')
-            
-        # Propagate variables to user namespace
-        self.user_ns.update(vdict)
+                    globs = locs = globals()
+            badblocks = []
 
-        # And configure interactive visibility
-        config_ns = self.user_config_ns
-        if interactive:
-            for name, val in vdict.iteritems():
-                config_ns.pop(name, None)
-        else:
-            for name,val in vdict.iteritems():
-                config_ns[name] = val
+            # we also need to identify indented blocks of code when replaying
+            # logs and put them together before passing them to an exec
+            # statement. This takes a bit of regexp and look-ahead work in the
+            # file. It's easiest if we swallow the whole thing in memory
+            # first, and manually walk through the lines list moving the
+            # counter ourselves.
+            indent_re = re.compile('\s+\S')
+            xfile = open(fname)
+            filelines = xfile.readlines()
+            xfile.close()
+            nlines = len(filelines)
+            lnum = 0
+            while lnum < nlines:
+                line = filelines[lnum]
+                lnum += 1
+                # don't re-insert logger status info into cache
+                if line.startswith('#log#'):
+                    continue
+                else:
+                    # build a block of code (maybe a single line) for execution
+                    block = line
+                    try:
+                        next = filelines[lnum] # lnum has already incremented
+                    except:
+                        next = None
+                    while next and indent_re.match(next):
+                        block += next
+                        lnum += 1
+                        try:
+                            next = filelines[lnum]
+                        except:
+                            next = None
+                    # now execute the block of one or more lines
+                    try:
+                        exec block in globs,locs
+                    except SystemExit:
+                        pass
+                    except:
+                        badblocks.append(block.rstrip())
+            if kw['quiet']:  # restore stdout
+                sys.stdout.close()
+                sys.stdout = stdout_save
+            print 'Finished replaying log file <%s>' % fname
+            if badblocks:
+                print >> sys.stderr, ('\nThe following lines/blocks in file '
+                                      '<%s> reported errors:' % fname)
+                    
+                for badline in badblocks:
+                    print >> sys.stderr, badline
+        else:  # regular file execution
+            try:
+                if sys.platform == 'win32' and sys.version_info < (2,5,1):
+                    # Work around a bug in Python for Windows.  The bug was
+                    # fixed in in Python 2.5 r54159 and 54158, but that's still
+                    # SVN Python as of March/07.  For details, see:
+                    # http://projects.scipy.org/ipython/ipython/ticket/123
+                    try:
+                        globs,locs = where[0:2]
+                    except:
+                        try:
+                            globs = locs = where[0]
+                        except:
+                            globs = locs = globals()
+                    exec file(fname) in globs,locs
+                else:
+                    execfile(fname,*where)
+            except SyntaxError:
+                self.showsyntaxerror()
+                warn('Failure executing file: <%s>' % fname)
+            except SystemExit,status:
+                # Code that correctly sets the exit status flag to success (0)
+                # shouldn't be bothered with a traceback.  Note that a plain
+                # sys.exit() does NOT set the message to 0 (it's empty) so that
+                # will still get a traceback.  Note that the structure of the
+                # SystemExit exception changed between Python 2.4 and 2.5, so
+                # the checks must be done in a version-dependent way.
+                show = False
+
+                if sys.version_info[:2] > (2,5):
+                    if status.message!=0 and not kw['exit_ignore']:
+                        show = True
+                else:
+                    if status.code and not kw['exit_ignore']:
+                        show = True
+                if show:
+                    self.showtraceback()
+                    warn('Failure executing file: <%s>' % fname)
+            except:
+                self.showtraceback()
+                warn('Failure executing file: <%s>' % fname)
+
+        syspath_cleanup()
 
     def cleanup_ipy_script(self, script):
         """Make a script safe for self.runlines()
@@ -2383,13 +2467,31 b' class InteractiveShell(Component, Magic):'
 
         #print 'push line: <%s>' % line  # dbg
         for subline in line.splitlines():
-            self.autoindent_update(subline)
+            self._autoindent_update(subline)
         self.buffer.append(line)
         more = self.runsource('\n'.join(self.buffer), self.filename)
         if not more:
             self.resetbuffer()
         return more
 
+    def _autoindent_update(self,line):
+        """Keep track of the indent level."""
+
+        #debugx('line')
+        #debugx('self.indent_current_nsp')
+        if self.autoindent:
+            if line:
+                inisp = num_ini_spaces(line)
+                if inisp < self.indent_current_nsp:
+                    self.indent_current_nsp = inisp
+
+                if line[-1] == ':':
+                    self.indent_current_nsp += 4
+                elif dedent_re.match(line):
+                    self.indent_current_nsp -= 4
+            else:
+                self.indent_current_nsp = 0
+
     def split_user_input(self, line):
         # This is really a hold-over to support ipapi and some extensions
         return prefilter.splitUserInput(line)
@@ -2425,47 +2527,121 b' class InteractiveShell(Component, Magic):'
             self.ask_exit()
             return ""
 
-        # Try to be reasonably smart about not re-indenting pasted input more
-        # than necessary.  We do this by trimming out the auto-indent initial
-        # spaces, if the user's actual input started itself with whitespace.
-        #debugx('self.buffer[-1]')
+        # Try to be reasonably smart about not re-indenting pasted input more
+        # than necessary.  We do this by trimming out the auto-indent initial
+        # spaces, if the user's actual input started itself with whitespace.
+        #debugx('self.buffer[-1]')
+
+        if self.autoindent:
+            if num_ini_spaces(line) > self.indent_current_nsp:
+                line = line[self.indent_current_nsp:]
+                self.indent_current_nsp = 0
+            
+        # store the unfiltered input before the user has any chance to modify
+        # it.
+        if line.strip():
+            if continue_prompt:
+                self.input_hist_raw[-1] += '%s\n' % line
+                if self.has_readline: # and some config option is set?
+                    try:
+                        histlen = self.readline.get_current_history_length()
+                        if histlen > 1:
+                            newhist = self.input_hist_raw[-1].rstrip()
+                            self.readline.remove_history_item(histlen-1)
+                            self.readline.replace_history_item(histlen-2,
+                                            newhist.encode(self.stdin_encoding))
+                    except AttributeError:
+                        pass # re{move,place}_history_item are new in 2.4.                
+            else:
+                self.input_hist_raw.append('%s\n' % line)                
+            # only entries starting at first column go to shadow history
+            if line.lstrip() == line:
+                self.shadowhist.add(line.strip())
+        elif not continue_prompt:
+            self.input_hist_raw.append('\n')
+        try:
+            lineout = self.prefilter(line,continue_prompt)
+        except:
+            # blanket except, in case a user-defined prefilter crashes, so it
+            # can't take all of ipython with it.
+            self.showtraceback()
+            return ''
+        else:
+            return lineout
+
+    # def init_exec_commands(self):
+    #     for cmd in self.config.EXECUTE:
+    #         print "execute:", cmd
+    #         self.api.runlines(cmd)
+    #         
+    #     batchrun = False
+    #     if self.config.has_key('EXECFILE'):
+    #         for batchfile in [path(arg) for arg in self.config.EXECFILE
+    #             if arg.lower().endswith('.ipy')]:
+    #             if not batchfile.isfile():
+    #                 print "No such batch file:", batchfile
+    #                 continue
+    #             self.api.runlines(batchfile.text())
+    #             batchrun = True
+    #     # without -i option, exit after running the batch file
+    #     if batchrun and not self.interactive:
+    #         self.ask_exit()            
+
+    # def load(self, mod):
+    #     """ Load an extension.
+    #     
+    #     Some modules should (or must) be 'load()':ed, rather than just imported.
+    #     
+    #     Loading will do:
+    #     
+    #     - run init_ipython(ip)
+    #     - run ipython_firstrun(ip)
+    #     """
+    # 
+    #     if mod in self.extensions:
+    #         # just to make sure we don't init it twice
+    #         # note that if you 'load' a module that has already been
+    #         # imported, init_ipython gets run anyway
+    #         
+    #         return self.extensions[mod]
+    #     __import__(mod)
+    #     m = sys.modules[mod]
+    #     if hasattr(m,'init_ipython'):
+    #         m.init_ipython(self)
+    #         
+    #     if hasattr(m,'ipython_firstrun'):
+    #         already_loaded = self.db.get('firstrun_done', set())
+    #         if mod not in already_loaded:
+    #             m.ipython_firstrun(self)
+    #             already_loaded.add(mod)
+    #             self.db['firstrun_done'] = already_loaded
+    #         
+    #     self.extensions[mod] = m
+    #     return m
+
+    #-------------------------------------------------------------------------
+    # Things related to the prefilter
+    #-------------------------------------------------------------------------
 
-        if self.autoindent:
-            if num_ini_spaces(line) > self.indent_current_nsp:
-                line = line[self.indent_current_nsp:]
-                self.indent_current_nsp = 0
-            
-        # store the unfiltered input before the user has any chance to modify
-        # it.
-        if line.strip():
-            if continue_prompt:
-                self.input_hist_raw[-1] += '%s\n' % line
-                if self.has_readline: # and some config option is set?
-                    try:
-                        histlen = self.readline.get_current_history_length()
-                        if histlen > 1:
-                            newhist = self.input_hist_raw[-1].rstrip()
-                            self.readline.remove_history_item(histlen-1)
-                            self.readline.replace_history_item(histlen-2,
-                                            newhist.encode(self.stdin_encoding))
-                    except AttributeError:
-                        pass # re{move,place}_history_item are new in 2.4.                
-            else:
-                self.input_hist_raw.append('%s\n' % line)                
-            # only entries starting at first column go to shadow history
-            if line.lstrip() == line:
-                self.shadowhist.add(line.strip())
-        elif not continue_prompt:
-            self.input_hist_raw.append('\n')
-        try:
-            lineout = self.prefilter(line,continue_prompt)
-        except:
-            # blanket except, in case a user-defined prefilter crashes, so it
-            # can't take all of ipython with it.
-            self.showtraceback()
-            return ''
-        else:
-            return lineout
+    def init_handlers(self):
+        # escapes for automatic behavior on the command line
+        self.ESC_SHELL  = '!'
+        self.ESC_SH_CAP = '!!'
+        self.ESC_HELP   = '?'
+        self.ESC_MAGIC  = '%'
+        self.ESC_QUOTE  = ','
+        self.ESC_QUOTE2 = ';'
+        self.ESC_PAREN  = '/'
+
+        # And their associated handlers
+        self.esc_handlers = {self.ESC_PAREN  : self.handle_auto,
+                             self.ESC_QUOTE  : self.handle_auto,
+                             self.ESC_QUOTE2 : self.handle_auto,
+                             self.ESC_MAGIC  : self.handle_magic,
+                             self.ESC_HELP   : self.handle_help,
+                             self.ESC_SHELL  : self.handle_shell_escape,
+                             self.ESC_SH_CAP : self.handle_shell_escape,
+                             }
 
     def _prefilter(self, line, continue_prompt):
         """Calls different preprocessors, depending on the form of line."""
@@ -2731,7 +2907,21 b' class InteractiveShell(Component, Magic):'
 
         # The input cache shouldn't be updated
         return line_info.line
-    
+
+    #-------------------------------------------------------------------------
+    # Utilities
+    #-------------------------------------------------------------------------
+
+    def getoutput(self, cmd):
+        return getoutput(self.var_expand(cmd,depth=2),
+                         header=self.system_header,
+                         verbose=self.system_verbose)
+
+    def getoutputerror(self, cmd):
+        return getoutputerror(self.var_expand(cmd,depth=2),
+                              header=self.system_header,
+                              verbose=self.system_verbose)
+
     def var_expand(self,cmd,depth=0):
         """Expand python variables in a string.
 
@@ -2776,6 +2966,15 b' class InteractiveShell(Component, Magic):'
         """Write a string to the default error output"""
         Term.cerr.write(data)
 
+    def ask_yes_no(self,prompt,default=True):
+        if self.quiet:
+            return True
+        return ask_yes_no(prompt,default)
+
+    #-------------------------------------------------------------------------
+    # Things related to IPython exiting
+    #-------------------------------------------------------------------------
+
     def ask_exit(self):
         """ Call for exiting. Can be overiden and used as a callback. """
         self.exit_now = True
@@ -2790,169 +2989,30 b' class InteractiveShell(Component, Magic):'
         else:
             self.ask_exit()
 
-    def safe_execfile(self,fname,*where,**kw):
-        """A safe version of the builtin execfile().
-
-        This version will never throw an exception, and knows how to handle
-        ipython logs as well.
-
-        :Parameters:
-          fname : string
-            Name of the file to be executed.
-            
-          where : tuple
-            One or two namespaces, passed to execfile() as (globals,locals).
-            If only one is given, it is passed as both.
-
-        :Keywords:
-          islog : boolean (False)
+    def atexit_operations(self):
+        """This will be executed at the time of exit.
 
-          quiet : boolean (True)
+        Saving of persistent data should be performed here.
+        """
+        self.savehist()
 
-          exit_ignore : boolean (False)
-          """
+        # Cleanup all tempfiles left around
+        for tfile in self.tempfiles:
+            try:
+                os.unlink(tfile)
+            except OSError:
+                pass
 
-        def syspath_cleanup():
-            """Internal cleanup routine for sys.path."""
-            if add_dname:
-                try:
-                    sys.path.remove(dname)
-                except ValueError:
-                    # For some reason the user has already removed it, ignore.
-                    pass
-        
-        fname = os.path.expanduser(fname)
+        # Clear all user namespaces to release all references cleanly.
+        self.reset()
 
-        # Find things also in current directory.  This is needed to mimic the
-        # behavior of running a script from the system command line, where
-        # Python inserts the script's directory into sys.path
-        dname = os.path.dirname(os.path.abspath(fname))
-        add_dname = False
-        if dname not in sys.path:
-            sys.path.insert(0,dname)
-            add_dname = True
+        # Run user hooks
+        self.hooks.shutdown_hook()
 
-        try:
-            xfile = open(fname)
-        except:
-            print >> Term.cerr, \
-                  'Could not open file <%s> for safe execution.' % fname
-            syspath_cleanup()
-            return None
+    def cleanup(self):
+        self.restore_sys_module_state()
 
-        kw.setdefault('islog',0)
-        kw.setdefault('quiet',1)
-        kw.setdefault('exit_ignore',0)
-        
-        first = xfile.readline()
-        loghead = str(self.loghead_tpl).split('\n',1)[0].strip()
-        xfile.close()
-        # line by line execution
-        if first.startswith(loghead) or kw['islog']:
-            print 'Loading log file <%s> one line at a time...' % fname
-            if kw['quiet']:
-                stdout_save = sys.stdout
-                sys.stdout = StringIO.StringIO()
-            try:
-                globs,locs = where[0:2]
-            except:
-                try:
-                    globs = locs = where[0]
-                except:
-                    globs = locs = globals()
-            badblocks = []
 
-            # we also need to identify indented blocks of code when replaying
-            # logs and put them together before passing them to an exec
-            # statement. This takes a bit of regexp and look-ahead work in the
-            # file. It's easiest if we swallow the whole thing in memory
-            # first, and manually walk through the lines list moving the
-            # counter ourselves.
-            indent_re = re.compile('\s+\S')
-            xfile = open(fname)
-            filelines = xfile.readlines()
-            xfile.close()
-            nlines = len(filelines)
-            lnum = 0
-            while lnum < nlines:
-                line = filelines[lnum]
-                lnum += 1
-                # don't re-insert logger status info into cache
-                if line.startswith('#log#'):
-                    continue
-                else:
-                    # build a block of code (maybe a single line) for execution
-                    block = line
-                    try:
-                        next = filelines[lnum] # lnum has already incremented
-                    except:
-                        next = None
-                    while next and indent_re.match(next):
-                        block += next
-                        lnum += 1
-                        try:
-                            next = filelines[lnum]
-                        except:
-                            next = None
-                    # now execute the block of one or more lines
-                    try:
-                        exec block in globs,locs
-                    except SystemExit:
-                        pass
-                    except:
-                        badblocks.append(block.rstrip())
-            if kw['quiet']:  # restore stdout
-                sys.stdout.close()
-                sys.stdout = stdout_save
-            print 'Finished replaying log file <%s>' % fname
-            if badblocks:
-                print >> sys.stderr, ('\nThe following lines/blocks in file '
-                                      '<%s> reported errors:' % fname)
-                    
-                for badline in badblocks:
-                    print >> sys.stderr, badline
-        else:  # regular file execution
-            try:
-                if sys.platform == 'win32' and sys.version_info < (2,5,1):
-                    # Work around a bug in Python for Windows.  The bug was
-                    # fixed in in Python 2.5 r54159 and 54158, but that's still
-                    # SVN Python as of March/07.  For details, see:
-                    # http://projects.scipy.org/ipython/ipython/ticket/123
-                    try:
-                        globs,locs = where[0:2]
-                    except:
-                        try:
-                            globs = locs = where[0]
-                        except:
-                            globs = locs = globals()
-                    exec file(fname) in globs,locs
-                else:
-                    execfile(fname,*where)
-            except SyntaxError:
-                self.showsyntaxerror()
-                warn('Failure executing file: <%s>' % fname)
-            except SystemExit,status:
-                # Code that correctly sets the exit status flag to success (0)
-                # shouldn't be bothered with a traceback.  Note that a plain
-                # sys.exit() does NOT set the message to 0 (it's empty) so that
-                # will still get a traceback.  Note that the structure of the
-                # SystemExit exception changed between Python 2.4 and 2.5, so
-                # the checks must be done in a version-dependent way.
-                show = False
 
-                if sys.version_info[:2] > (2,5):
-                    if status.message!=0 and not kw['exit_ignore']:
-                        show = True
-                else:
-                    if status.code and not kw['exit_ignore']:
-                        show = True
-                if show:
-                    self.showtraceback()
-                    warn('Failure executing file: <%s>' % fname)
-            except:
-                self.showtraceback()
-                warn('Failure executing file: <%s>' % fname)
 
-        syspath_cleanup()
 
-#************************* end of file <iplib.py> *****************************