Major restructuring of magics, breaking them up into separate classes.

This is the first step to get the new magic architecture in place,
with a new base class for magic functions.

At this point IPython does *not* run, but the changes are extensive
enough to warrant intermediate non-working commits.

IPython/core/interactiveshell.py

@@ -380,6 +380,7 class InteractiveShell(SingletonConfigurable):
     plugin_manager = Instance('IPython.core.plugin.PluginManager')
     payload_manager = Instance('IPython.core.payload.PayloadManager')
     history_manager = Instance('IPython.core.history.HistoryManager')
+    magics_manager = Instance('IPython.core.magic.MagicsManager')
 
     profile_dir = Instance('IPython.core.application.ProfileDir')
     @property

IPython/core/magic.py

@@ -4,8 +4,8
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2001 Janko Hauser <[e-mail]> and
-#  Copyright (C) 2001-2007 Fernando Perez <[e-mail]>
-#  Copyright (C) 2008-2011  The IPython Development Team
+#  Copyright (C) 2001 Fernando Perez <[e-mail]>
+#  Copyright (C) 2008 The IPython Development Team
 
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
@@ -18,14 +18,16
 import __builtin__ as builtin_mod
 import __future__
 import bdb
+import gc
+import imp
 import inspect
 import io
 import json
 import os
-import sys
 import re
+import shutil
+import sys
 import time
-import gc
 from StringIO import StringIO
 from getopt import getopt,GetoptError
 from pprint import pformat
@@ -42,36 +44,48 except ImportError:
     except ImportError:
         profile = pstats = None
 
+import IPython
+from IPython.config.application import Application
+from IPython.config.configurable import Configurable
 from IPython.core import debugger, oinspect
+from IPython.core import magic_arguments, page
+from IPython.core.error import StdinNotImplementedError
 from IPython.core.error import TryNext
 from IPython.core.error import UsageError
-from IPython.core.error import StdinNotImplementedError
+from IPython.core.fakemodule import FakeModule
 from IPython.core.macro import Macro
-from IPython.core import magic_arguments, page
 from IPython.core.prefilter import ESC_MAGIC
+from IPython.core.profiledir import ProfileDir
 from IPython.testing.skipdoctest import skip_doctest
+from IPython.utils import openpy
 from IPython.utils import py3compat
 from IPython.utils.encoding import DEFAULT_ENCODING
 from IPython.utils.io import file_read, nlprint
+from IPython.utils.ipstruct import Struct
 from IPython.utils.module_paths import find_mod
 from IPython.utils.path import get_py_filename, unquote_filename
 from IPython.utils.process import arg_split, abbrev_cwd
 from IPython.utils.terminal import set_term_title
 from IPython.utils.text import format_screen
 from IPython.utils.timing import clock, clock2
+from IPython.utils.traitlets import Bool, Dict, Instance, Integer, List, Unicode
 from IPython.utils.warn import warn, error
-from IPython.utils.ipstruct import Struct
-from IPython.config.application import Application
 
 #-----------------------------------------------------------------------------
-# Utility functions
+# Utility classes and functions
 #-----------------------------------------------------------------------------
 
+class Bunch: pass
+
+
+# Used for exception handling in magic_edit
+class MacroToEdit(ValueError): pass
+
+
 def on_off(tag):
     """Return an ON/OFF string for a 1/0 input. Simple utility function."""
     return ['OFF','ON'][tag]
 
-class Bunch: pass
 
 def compress_dhist(dh):
     head, tail = dh[:-10], dh[-10:]
@@ -86,72 +100,28 def compress_dhist(dh):
 
     return newhead + tail
 
+
 def needs_local_scope(func):
     """Decorator to mark magic functions which need to local scope to run."""
     func.needs_local_scope = True
     return func
 
-    
-# Used for exception handling in magic_edit
-class MacroToEdit(ValueError): pass
-
 #***************************************************************************
-# Main class implementing Magic functionality
-
-# XXX - for some odd reason, if Magic is made a new-style class, we get errors
-# on construction of the main InteractiveShell object.  Something odd is going
-# on with super() calls, Configurable and the MRO... For now leave it as-is, but
-# eventually this needs to be clarified.
-# BG: This is because InteractiveShell inherits from this, but is itself a
-# Configurable. This messes up the MRO in some way. The fix is that we need to
-# make Magic a configurable that InteractiveShell does not subclass.
-
-class Magic(object):
-    """Magic functions for InteractiveShell.
-
-    Shell functions which can be reached as %function_name. All magic
-    functions should accept a string, which they can parse for their own
-    needs. This can make some functions easier to type, eg `%cd ../`
-    vs. `%cd("../")`
-
-    ALL definitions MUST begin with the prefix magic_. The user won't need it
-    at the command line, but it is is needed in the definition. """
 
-    # class globals
-    auto_status = ['Automagic is OFF, % prefix IS needed for magic functions.',
-                   'Automagic is ON, % prefix NOT needed for magic functions.']
+class MagicManager(Configurable):
+    """Object that handles all magic-related functionality for IPython.
+    """
+    # An instance of the IPython shell we are attached to
+    shell = Instance('IPython.core.interactiveshell.InteractiveShellABC')
 
+    auto_status = Enum([
+        'Automagic is OFF, % prefix IS needed for magic functions.',
+        'Automagic is ON, % prefix NOT needed for magic functions.'])
 
-    configurables = None
+    def __init__(self, shell=None, config=None, **traits):
 
-    default_runner = None
-    #......................................................................
-    # some utility functions
+        super(MagicManager, self).__init__(shell=shell, config=config, **traits)
 
-    def __init__(self, shell):
-
-        self.options_table = {}
-        if profile is None:
-            self.magic_prun = self.profile_missing_notice
-        self.shell = shell
-        if self.configurables is None:
-            self.configurables = []
-
-        # namespace for holding state we may need
-        self._magic_state = Bunch()
-
-    def profile_missing_notice(self, *args, **kwargs):
-        error("""\
-The profile module could not be found. It has been removed from the standard
-python packages because of its non-free license. To use profiling, install the
-python-profiler package from non-free.""")
-
-    def default_option(self,fn,optstr):
-        """Make an entry in the options_table for fn, with value optstr"""
-
-        if fn not in self.lsmagic():
-            error("%s is not a magic function" % fn)
-        self.options_table[fn] = optstr
 
     def lsmagic(self):
         """Return a list of currently available magic functions.
@@ -170,15 +140,39 python-profiler package from non-free.""")
         # and bound magics by user (so they can access self):
         inst_bound_magic =  lambda fn: fn.startswith('magic_') and \
                            callable(self.__class__.__dict__[fn])
-        magics = filter(class_magic,Magic.__dict__.keys()) + \
+        magics = filter(class_magic, Magic.__dict__.keys()) + \
                  filter(inst_magic, self.__dict__.keys()) + \
                  filter(inst_bound_magic, self.__class__.__dict__.keys())
         out = []
         for fn in set(magics):
-            out.append(fn.replace('magic_','',1))
+            out.append(fn.replace('magic_', '', 1))
         out.sort()
         return out
 
+
+class MagicFunctions(object):
+    """Base class for implementing magic functions.
+
+    Shell functions which can be reached as %function_name. All magic
+    functions should accept a string, which they can parse for their own
+    needs. This can make some functions easier to type, eg `%cd ../`
+    vs. `%cd("../")`
+    """
+
+    options_table = Dict(config=True,
+        help = """Dict holding all command-line options for each magic.
+        """)
+
+    class __metaclass__(type):
+        def __new__(cls, name, bases, dct):
+            cls.registered = False
+            return type.__new__(cls, name, bases, dct)
+
+    def __init__(self, shell):
+        if not(self.__class__.registered):
+            raise ValueError('unregistered Magics')
+        self.shell = shell
+
     def arg_err(self,func):
         """Print docstring if incorrect arguments were passed"""
         print 'Error in arguments:'
@@ -211,7 +205,7 python-profiler package from non-free.""")
         strng = newline_re.sub(r'\\textbackslash{}n',strng)
         return strng
 
-    def parse_options(self,arg_str,opt_str,*long_opts,**kw):
+    def parse_options(self, arg_str, opt_str, *long_opts, **kw):
         """Parse options passed to an argument string.
 
         The interface is similar to that of getopt(), but it returns back a
@@ -280,10 +274,20 python-profiler package from non-free.""")
 
         return opts,args
 
-    #......................................................................
-    # And now the actual magic functions
+    def default_option(self,fn,optstr):
+        """Make an entry in the options_table for fn, with value optstr"""
+
+        if fn not in self.lsmagic():
+            error("%s is not a magic function" % fn)
+        self.options_table[fn] = optstr
+
+
+class BasicMagics(MagicFunctions):
+    """Magics that provide central IPython functionality.
+
+    These are various magics that don't fit into specific categories but that
+    are all part of the base 'IPython experience'."""
 
-    # Functions for IPython shell work (vars,funcs, config, etc)
     def magic_lsmagic(self, parameter_s = ''):
         """List currently available magic functions."""
         mesc = ESC_MAGIC
@@ -383,99 +387,6 Currently the magic system has the following functions:\n"""
                                      Magic.auto_status[self.shell.automagic] ) )
         page.page(outmsg)
 
-    def magic_automagic(self, parameter_s = ''):
-        """Make magic functions callable without having to type the initial %.
-
-        Without argumentsl toggles on/off (when off, you must call it as
-        %automagic, of course).  With arguments it sets the value, and you can
-        use any of (case insensitive):
-
-         - on,1,True: to activate
-
-         - off,0,False: to deactivate.
-
-        Note that magic functions have lowest priority, so if there's a
-        variable whose name collides with that of a magic fn, automagic won't
-        work for that function (you get the variable instead). However, if you
-        delete the variable (del var), the previously shadowed magic function
-        becomes visible to automagic again."""
-
-        arg = parameter_s.lower()
-        if parameter_s in ('on','1','true'):
-            self.shell.automagic = True
-        elif parameter_s in ('off','0','false'):
-            self.shell.automagic = False
-        else:
-            self.shell.automagic = not self.shell.automagic
-        print '\n' + Magic.auto_status[self.shell.automagic]
-
-    @skip_doctest
-    def magic_autocall(self, parameter_s = ''):
-        """Make functions callable without having to type parentheses.
-
-        Usage:
-
-           %autocall [mode]
-
-        The mode can be one of: 0->Off, 1->Smart, 2->Full.  If not given, the
-        value is toggled on and off (remembering the previous state).
-
-        In more detail, these values mean:
-
-        0 -> fully disabled
-
-        1 -> active, but do not apply if there are no arguments on the line.
-
-        In this mode, you get::
-
-          In [1]: callable
-          Out[1]: <built-in function callable>
-
-          In [2]: callable 'hello'
-          ------> callable('hello')
-          Out[2]: False
-
-        2 -> Active always.  Even if no arguments are present, the callable
-        object is called::
-
-          In [2]: float
-          ------> float()
-          Out[2]: 0.0
-
-        Note that even with autocall off, you can still use '/' at the start of
-        a line to treat the first argument on the command line as a function
-        and add parentheses to it::
-
-          In [8]: /str 43
-          ------> str(43)
-          Out[8]: '43'
-
-        # all-random (note for auto-testing)
-        """
-
-        if parameter_s:
-            arg = int(parameter_s)
-        else:
-            arg = 'toggle'
-
-        if not arg in (0,1,2,'toggle'):
-            error('Valid modes: (0->Off, 1->Smart, 2->Full')
-            return
-
-        if arg in (0,1,2):
-            self.shell.autocall = arg
-        else: # toggle
-            if self.shell.autocall:
-                self._magic_state.autocall_save = self.shell.autocall
-                self.shell.autocall = 0
-            else:
-                try:
-                    self.shell.autocall = self._magic_state.autocall_save
-                except AttributeError:
-                    self.shell.autocall = self._magic_state.autocall_save = 1
-
-        print "Automatic calling is:",['OFF','Smart','Full'][self.shell.autocall]
-
 
     def magic_page(self, parameter_s=''):
         """Pretty print the object and display it through a pager.
@@ -510,2214 +421,2572 Currently the magic system has the following functions:\n"""
         else:
             error("profile is an application-level value, but you don't appear to be in an IPython application")
 
-    def magic_pinfo(self, parameter_s='', namespaces=None):
-        """Provide detailed information about an object.
-
-        '%pinfo object' is just a synonym for object? or ?object."""
-
-        #print 'pinfo par: <%s>' % parameter_s  # dbg
-
-
-        # detail_level: 0 -> obj? , 1 -> obj??
-        detail_level = 0
-        # We need to detect if we got called as 'pinfo pinfo foo', which can
-        # happen if the user types 'pinfo foo?' at the cmd line.
-        pinfo,qmark1,oname,qmark2 = \
-               re.match('(pinfo )?(\?*)(.*?)(\??$)',parameter_s).groups()
-        if pinfo or qmark1 or qmark2:
-            detail_level = 1
-        if "*" in oname:
-            self.magic_psearch(oname)
-        else:
-            self.shell._inspect('pinfo', oname, detail_level=detail_level,
-                                namespaces=namespaces)
-
-    def magic_pinfo2(self, parameter_s='', namespaces=None):
-        """Provide extra detailed information about an object.
+    def magic_pprint(self, parameter_s=''):
+        """Toggle pretty printing on/off."""
+        ptformatter = self.shell.display_formatter.formatters['text/plain']
+        ptformatter.pprint = bool(1 - ptformatter.pprint)
+        print 'Pretty printing has been turned', \
+              ['OFF','ON'][ptformatter.pprint]
 
-        '%pinfo2 object' is just a synonym for object?? or ??object."""
-        self.shell._inspect('pinfo', parameter_s, detail_level=1,
-                            namespaces=namespaces)
+    def magic_colors(self,parameter_s = ''):
+        """Switch color scheme for prompts, info system and exception handlers.
 
-    @skip_doctest
-    def magic_pdef(self, parameter_s='', namespaces=None):
-        """Print the definition header for any callable object.
+        Currently implemented schemes: NoColor, Linux, LightBG.
 
-        If the object is a class, print the constructor information.
+        Color scheme names are not case-sensitive.
 
         Examples
         --------
-        ::
+        To get a plain black and white terminal::
 
-          In [3]: %pdef urllib.urlopen
-          urllib.urlopen(url, data=None, proxies=None)
+          %colors nocolor
         """
-        self._inspect('pdef',parameter_s, namespaces)
 
-    def magic_pdoc(self, parameter_s='', namespaces=None):
-        """Print the docstring for an object.
+        def color_switch_err(name):
+            warn('Error changing %s color schemes.\n%s' %
+                 (name,sys.exc_info()[1]))
 
-        If the given object is a class, it will print both the class and the
-        constructor docstrings."""
-        self._inspect('pdoc',parameter_s, namespaces)
 
-    def magic_psource(self, parameter_s='', namespaces=None):
-        """Print (or run through pager) the source code for an object."""
-        self._inspect('psource',parameter_s, namespaces)
+        new_scheme = parameter_s.strip()
+        if not new_scheme:
+            raise UsageError(
+                "%colors: you must specify a color scheme. See '%colors?'")
+            return
+        # local shortcut
+        shell = self.shell
 
-    def magic_pfile(self, parameter_s=''):
-        """Print (or run through pager) the file where an object is defined.
+        import IPython.utils.rlineimpl as readline
 
-        The file opens at the line where the object definition begins. IPython
-        will honor the environment variable PAGER if set, and otherwise will
-        do its best to print the file in a convenient form.
+        if not shell.colors_force and \
+                not readline.have_readline and sys.platform == "win32":
+            msg = """\
+Proper color support under MS Windows requires the pyreadline library.
+You can find it at:
+http://ipython.org/pyreadline.html
+Gary's readline needs the ctypes module, from:
+http://starship.python.net/crew/theller/ctypes
+(Note that ctypes is already part of Python versions 2.5 and newer).
 
-        If the given argument is not an object currently defined, IPython will
-        try to interpret it as a filename (automatically adding a .py extension
-        if needed). You can thus use %pfile as a syntax highlighting code
-        viewer."""
+Defaulting color scheme to 'NoColor'"""
+            new_scheme = 'NoColor'
+            warn(msg)
 
-        # first interpret argument as an object name
-        out = self._inspect('pfile',parameter_s)
-        # if not, try the input as a filename
-        if out == 'not found':
+        # readline option is 0
+        if not shell.colors_force and not shell.has_readline:
+            new_scheme = 'NoColor'
+
+        # Set prompt colors
+        try:
+            shell.prompt_manager.color_scheme = new_scheme
+        except:
+            color_switch_err('prompt')
+        else:
+            shell.colors = \
+                   shell.prompt_manager.color_scheme_table.active_scheme_name
+        # Set exception colors
+        try:
+            shell.InteractiveTB.set_colors(scheme = new_scheme)
+            shell.SyntaxTB.set_colors(scheme = new_scheme)
+        except:
+            color_switch_err('exception')
+
+        # Set info (for 'object?') colors
+        if shell.color_info:
             try:
-                filename = get_py_filename(parameter_s)
-            except IOError,msg:
-                print msg
-                return
-            page.page(self.shell.inspector.format(open(filename).read()))
+                shell.inspector.set_active_scheme(new_scheme)
+            except:
+                color_switch_err('object inspector')
+        else:
+            shell.inspector.set_active_scheme('NoColor')
 
-    def magic_psearch(self, parameter_s=''):
-        """Search for object in namespaces by wildcard.
+    def magic_xmode(self,parameter_s = ''):
+        """Switch modes for the exception handlers.
 
-        %psearch [options] PATTERN [OBJECT TYPE]
+        Valid modes: Plain, Context and Verbose.
 
-        Note: ? can be used as a synonym for %psearch, at the beginning or at
-        the end: both a*? and ?a* are equivalent to '%psearch a*'.  Still, the
-        rest of the command line must be unchanged (options come first), so
-        for example the following forms are equivalent
+        If called without arguments, acts as a toggle."""
 
-        %psearch -i a* function
-        -i a* function?
-        ?-i a* function
+        def xmode_switch_err(name):
+            warn('Error changing %s exception modes.\n%s' %
+                 (name,sys.exc_info()[1]))
 
-        Arguments:
+        shell = self.shell
+        new_mode = parameter_s.strip().capitalize()
+        try:
+            shell.InteractiveTB.set_mode(mode=new_mode)
+            print 'Exception reporting mode:',shell.InteractiveTB.mode
+        except:
+            xmode_switch_err('user')
 
-          PATTERN
+    def magic_quickref(self,arg):
+        """ Show a quick reference sheet """
+        import IPython.core.usage
+        qr = IPython.core.usage.quick_reference + self.magic_magic('-brief')
+        page.page(qr)
 
-          where PATTERN is a string containing * as a wildcard similar to its
-          use in a shell.  The pattern is matched in all namespaces on the
-          search path. By default objects starting with a single _ are not
-          matched, many IPython generated objects have a single
-          underscore. The default is case insensitive matching. Matching is
-          also done on the attributes of objects and not only on the objects
-          in a module.
+    def magic_doctest_mode(self,parameter_s=''):
+        """Toggle doctest mode on and off.
 
-          [OBJECT TYPE]
+        This mode is intended to make IPython behave as much as possible like a
+        plain Python shell, from the perspective of how its prompts, exceptions
+        and output look.  This makes it easy to copy and paste parts of a
+        session into doctests.  It does so by:
 
-          Is the name of a python type from the types module. The name is
-          given in lowercase without the ending type, ex. StringType is
-          written string. By adding a type here only objects matching the
-          given type are matched. Using all here makes the pattern match all
-          types (this is the default).
+        - Changing the prompts to the classic ``>>>`` ones.
+        - Changing the exception reporting mode to 'Plain'.
+        - Disabling pretty-printing of output.
 
-        Options:
+        Note that IPython also supports the pasting of code snippets that have
+        leading '>>>' and '...' prompts in them.  This means that you can paste
+        doctests from files or docstrings (even if they have leading
+        whitespace), and the code will execute correctly.  You can then use
+        '%history -t' to see the translated history; this will give you the
+        input after removal of all the leading prompts and whitespace, which
+        can be pasted back into an editor.
 
-          -a: makes the pattern match even objects whose names start with a
-          single underscore.  These names are normally omitted from the
-          search.
+        With these features, you can switch into this mode easily whenever you
+        need to do testing and changes to doctests, without having to leave
+        your existing IPython session.
+        """
 
-          -i/-c: make the pattern case insensitive/sensitive.  If neither of
-          these options are given, the default is read from your configuration
-          file, with the option ``InteractiveShell.wildcards_case_sensitive``.
-          If this option is not specified in your configuration file, IPython's
-          internal default is to do a case sensitive search.
+        from IPython.utils.ipstruct import Struct
 
-          -e/-s NAMESPACE: exclude/search a given namespace.  The pattern you
-          specify can be searched in any of the following namespaces:
-          'builtin', 'user', 'user_global','internal', 'alias', where
-          'builtin' and 'user' are the search defaults.  Note that you should
-          not use quotes when specifying namespaces.
+        # Shorthands
+        shell = self.shell
+        pm = shell.prompt_manager
+        meta = shell.meta
+        disp_formatter = self.shell.display_formatter
+        ptformatter = disp_formatter.formatters['text/plain']
+        # dstore is a data store kept in the instance metadata bag to track any
+        # changes we make, so we can undo them later.
+        dstore = meta.setdefault('doctest_mode',Struct())
+        save_dstore = dstore.setdefault
 
-          'Builtin' contains the python module builtin, 'user' contains all
-          user data, 'alias' only contain the shell aliases and no python
-          objects, 'internal' contains objects used by IPython.  The
-          'user_global' namespace is only used by embedded IPython instances,
-          and it contains module-level globals.  You can add namespaces to the
-          search with -s or exclude them with -e (these options can be given
-          more than once).
+        # save a few values we'll need to recover later
+        mode = save_dstore('mode',False)
+        save_dstore('rc_pprint',ptformatter.pprint)
+        save_dstore('xmode',shell.InteractiveTB.mode)
+        save_dstore('rc_separate_out',shell.separate_out)
+        save_dstore('rc_separate_out2',shell.separate_out2)
+        save_dstore('rc_prompts_pad_left',pm.justify)
+        save_dstore('rc_separate_in',shell.separate_in)
+        save_dstore('rc_plain_text_only',disp_formatter.plain_text_only)
+        save_dstore('prompt_templates',(pm.in_template, pm.in2_template, pm.out_template))
 
-        Examples
-        --------
-        ::
+        if mode == False:
+            # turn on
+            pm.in_template = '>>> '
+            pm.in2_template = '... '
+            pm.out_template = ''
 
-          %psearch a*            -> objects beginning with an a
-          %psearch -e builtin a* -> objects NOT in the builtin space starting in a
-          %psearch a* function   -> all functions beginning with an a
-          %psearch re.e*         -> objects beginning with an e in module re
-          %psearch r*.e*         -> objects that start with e in modules starting in r
-          %psearch r*.* string   -> all strings in modules beginning with r
+            # Prompt separators like plain python
+            shell.separate_in = ''
+            shell.separate_out = ''
+            shell.separate_out2 = ''
 
-        Case sensitive search::
+            pm.justify = False
 
-          %psearch -c a*         list all object beginning with lower case a
+            ptformatter.pprint = False
+            disp_formatter.plain_text_only = True
 
-        Show objects beginning with a single _::
+            shell.magic('xmode Plain')
+        else:
+            # turn off
+            pm.in_template, pm.in2_template, pm.out_template = dstore.prompt_templates
 
-          %psearch -a _*         list objects beginning with a single underscore"""
-        try:
-            parameter_s.encode('ascii')
-        except UnicodeEncodeError:
-            print 'Python identifiers can only contain ascii characters.'
-            return
+            shell.separate_in = dstore.rc_separate_in
 
-        # default namespaces to be searched
-        def_search = ['user_local', 'user_global', 'builtin']
+            shell.separate_out = dstore.rc_separate_out
+            shell.separate_out2 = dstore.rc_separate_out2
 
-        # Process options/args
-        opts,args = self.parse_options(parameter_s,'cias:e:',list_all=True)
-        opt = opts.get
-        shell = self.shell
-        psearch = shell.inspector.psearch
+            pm.justify = dstore.rc_prompts_pad_left
 
-        # select case options
-        if opts.has_key('i'):
-            ignore_case = True
-        elif opts.has_key('c'):
-            ignore_case = False
-        else:
-            ignore_case = not shell.wildcards_case_sensitive
+            ptformatter.pprint = dstore.rc_pprint
+            disp_formatter.plain_text_only = dstore.rc_plain_text_only
 
-        # Build list of namespaces to search from user options
-        def_search.extend(opt('s',[]))
-        ns_exclude = ns_exclude=opt('e',[])
-        ns_search = [nm for nm in def_search if nm not in ns_exclude]
+            shell.magic('xmode ' + dstore.xmode)
 
-        # Call the actual search
+        # Store new mode and inform
+        dstore.mode = bool(1-int(mode))
+        mode_label = ['OFF','ON'][dstore.mode]
+        print 'Doctest mode is:', mode_label
+
+    def magic_gui(self, parameter_s=''):
+        """Enable or disable IPython GUI event loop integration.
+
+        %gui [GUINAME]
+
+        This magic replaces IPython's threaded shells that were activated
+        using the (pylab/wthread/etc.) command line flags.  GUI toolkits
+        can now be enabled at runtime and keyboard
+        interrupts should work without any problems.  The following toolkits
+        are supported:  wxPython, PyQt4, PyGTK, Tk and Cocoa (OSX)::
+
+            %gui wx      # enable wxPython event loop integration
+            %gui qt4|qt  # enable PyQt4 event loop integration
+            %gui gtk     # enable PyGTK event loop integration
+            %gui gtk3    # enable Gtk3 event loop integration
+            %gui tk      # enable Tk event loop integration
+            %gui OSX     # enable Cocoa event loop integration
+                         # (requires %matplotlib 1.1)
+            %gui         # disable all event loop integration
+
+        WARNING:  after any of these has been called you can simply create
+        an application object, but DO NOT start the event loop yourself, as
+        we have already handled that.
+        """
+        opts, arg = self.parse_options(parameter_s, '')
+        if arg=='': arg = None
         try:
-            psearch(args,shell.ns_table,ns_search,
-                    show_all=opt('a'),ignore_case=ignore_case)
-        except:
-            shell.showtraceback()
+            return self.enable_gui(arg)
+        except Exception as e:
+            # print simple error message, rather than traceback if we can't
+            # hook up the GUI
+            error(str(e))
 
     @skip_doctest
-    def magic_who_ls(self, parameter_s=''):
-        """Return a sorted list of all interactive variables.
+    def magic_precision(self, s=''):
+        """Set floating point precision for pretty printing.
 
-        If arguments are given, only variables of types matching these
-        arguments are returned.
+        Can set either integer precision or a format string.
+
+        If numpy has been imported and precision is an int,
+        numpy display precision will also be set, via ``numpy.set_printoptions``.
+
+        If no argument is given, defaults will be restored.
 
         Examples
         --------
+        ::
 
-        Define two variables and list them with who_ls::
+            In [1]: from math import pi
 
-          In [1]: alpha = 123
+            In [2]: %precision 3
+            Out[2]: u'%.3f'
 
-          In [2]: beta = 'test'
+            In [3]: pi
+            Out[3]: 3.142
 
-          In [3]: %who_ls
-          Out[3]: ['alpha', 'beta']
+            In [4]: %precision %i
+            Out[4]: u'%i'
 
-          In [4]: %who_ls int
-          Out[4]: ['alpha']
+            In [5]: pi
+            Out[5]: 3
 
-          In [5]: %who_ls str
-          Out[5]: ['beta']
-        """
+            In [6]: %precision %e
+            Out[6]: u'%e'
 
-        user_ns = self.shell.user_ns
-        user_ns_hidden = self.shell.user_ns_hidden
-        out = [ i for i in user_ns
-                if not i.startswith('_') \
-                and not i in user_ns_hidden ]
+            In [7]: pi**10
+            Out[7]: 9.364805e+04
 
-        typelist = parameter_s.split()
-        if typelist:
-            typeset = set(typelist)
-            out = [i for i in out if type(user_ns[i]).__name__ in typeset]
+            In [8]: %precision
+            Out[8]: u'%r'
 
-        out.sort()
-        return out
+            In [9]: pi**10
+            Out[9]: 93648.047476082982
+        """
+        ptformatter = self.shell.display_formatter.formatters['text/plain']
+        ptformatter.float_precision = s
+        return ptformatter.float_format
 
-    @skip_doctest
-    def magic_who(self, parameter_s=''):
-        """Print all interactive variables, with some minimal formatting.
+    @magic_arguments.magic_arguments()
+    @magic_arguments.argument(
+        '-e', '--export', action='store_true', default=False,
+        help='Export IPython history as a notebook. The filename argument '
+             'is used to specify the notebook name and format. For example '
+             'a filename of notebook.ipynb will result in a notebook name '
+             'of "notebook" and a format of "xml". Likewise using a ".json" '
+             'or ".py" file extension will write the notebook in the json '
+             'or py formats.'
+    )
+    @magic_arguments.argument(
+        '-f', '--format',
+        help='Convert an existing IPython notebook to a new format. This option '
+             'specifies the new format and can have the values: xml, json, py. '
+             'The target filename is chosen automatically based on the new '
+             'format. The filename argument gives the name of the source file.'
+    )
+    @magic_arguments.argument(
+        'filename', type=unicode,
+        help='Notebook name or filename'
+    )
+    def magic_notebook(self, s):
+        """Export and convert IPython notebooks.
 
-        If any arguments are given, only variables whose type matches one of
-        these are printed.  For example::
+        This function can export the current IPython history to a notebook file
+        or can convert an existing notebook file into a different format. For
+        example, to export the history to "foo.ipynb" do "%notebook -e foo.ipynb".
+        To export the history to "foo.py" do "%notebook -e foo.py". To convert
+        "foo.ipynb" to "foo.json" do "%notebook -f json foo.ipynb". Possible
+        formats include (json/ipynb, py).
+        """
+        args = magic_arguments.parse_argstring(self.magic_notebook, s)
 
-          %who function str
+        from IPython.nbformat import current
+        args.filename = unquote_filename(args.filename)
+        if args.export:
+            fname, name, format = current.parse_filename(args.filename)
+            cells = []
+            hist = list(self.shell.history_manager.get_range())
+            for session, prompt_number, input in hist[:-1]:
+                cells.append(current.new_code_cell(prompt_number=prompt_number,
+                                                   input=input))
+            worksheet = current.new_worksheet(cells=cells)
+            nb = current.new_notebook(name=name,worksheets=[worksheet])
+            with io.open(fname, 'w', encoding='utf-8') as f:
+                current.write(nb, f, format);
+        elif args.format is not None:
+            old_fname, old_name, old_format = current.parse_filename(args.filename)
+            new_format = args.format
+            if new_format == u'xml':
+                raise ValueError('Notebooks cannot be written as xml.')
+            elif new_format == u'ipynb' or new_format == u'json':
+                new_fname = old_name + u'.ipynb'
+                new_format = u'json'
+            elif new_format == u'py':
+                new_fname = old_name + u'.py'
+            else:
+                raise ValueError('Invalid notebook format: %s' % new_format)
+            with io.open(old_fname, 'r', encoding='utf-8') as f:
+                nb = current.read(f, old_format)
+            with io.open(new_fname, 'w', encoding='utf-8') as f:
+                current.write(nb, f, new_format)
 
-        will only list functions and strings, excluding all other types of
-        variables.  To find the proper type names, simply use type(var) at a
-        command line to see how python prints type names.  For example:
 
-        ::
+class CodeMagics(MagicFunctions):
+    """Magics related to code management (loading, saving, editing, ...)."""
 
-          In [1]: type('hello')\\
-          Out[1]: <type 'str'>
+    def magic_save(self,parameter_s = ''):
+        """Save a set of lines or a macro to a given filename.
 
-        indicates that the type name for strings is 'str'.
+        Usage:\\
+          %save [options] filename n1-n2 n3-n4 ... n5 .. n6 ...
 
-        ``%who`` always excludes executed names loaded through your configuration
-        file and things which are internal to IPython.
+        Options:
 
-        This is deliberate, as typically you may load many modules and the
-        purpose of %who is to show you only what you've manually defined.
+          -r: use 'raw' input.  By default, the 'processed' history is used,
+          so that magics are loaded in their transformed version to valid
+          Python.  If this option is given, the raw input as typed as the
+          command line is used instead.
 
-        Examples
-        --------
+        This function uses the same syntax as %history for input ranges,
+        then saves the lines to the filename you specify.
 
-        Define two variables and list them with who::
+        It adds a '.py' extension to the file if you don't do so yourself, and
+        it asks for confirmation before overwriting existing files."""
 
-          In [1]: alpha = 123
+        opts,args = self.parse_options(parameter_s,'r',mode='list')
+        fname, codefrom = unquote_filename(args[0]), " ".join(args[1:])
+        if not fname.endswith('.py'):
+            fname += '.py'
+        if os.path.isfile(fname):
+            ans = raw_input('File `%s` exists. Overwrite (y/[N])? ' % fname)
+            if ans.lower() not in ['y','yes']:
+                print 'Operation cancelled.'
+                return
+        try:
+            cmds = self.shell.find_user_code(codefrom, 'r' in opts)
+        except (TypeError, ValueError) as e:
+            print e.args[0]
+            return
+        with io.open(fname,'w', encoding="utf-8") as f:
+            f.write(u"# coding: utf-8\n")
+            f.write(py3compat.cast_unicode(cmds))
+        print 'The following commands were written to file `%s`:' % fname
+        print cmds
 
-          In [2]: beta = 'test'
+    def magic_pastebin(self, parameter_s = ''):
+        """Upload code to Github's Gist paste bin, returning the URL.
 
-          In [3]: %who
-          alpha   beta
+        Usage:\\
+          %pastebin [-d "Custom description"] 1-7
 
-          In [4]: %who int
-          alpha
+        The argument can be an input history range, a filename, or the name of a
+        string or macro.
 
-          In [5]: %who str
-          beta
+        Options:
+
+          -d: Pass a custom description for the gist. The default will say
+              "Pasted from IPython".
         """
+        opts, args = self.parse_options(parameter_s, 'd:')
 
-        varlist = self.magic_who_ls(parameter_s)
-        if not varlist:
-            if parameter_s:
-                print 'No variables match your requested type.'
-            else:
-                print 'Interactive namespace is empty.'
+        try:
+            code = self.shell.find_user_code(args)
+        except (ValueError, TypeError) as e:
+            print e.args[0]
             return
 
-        # if we have variables, move on...
-        count = 0
-        for i in varlist:
-            print i+'\t',
-            count += 1
-            if count > 8:
-                count = 0
-                print
-        print
-
-    @skip_doctest
-    def magic_whos(self, parameter_s=''):
-        """Like %who, but gives some extra information about each variable.
+        post_data = json.dumps({
+          "description": opts.get('d', "Pasted from IPython"),
+          "public": True,
+          "files": {
+            "file1.py": {
+              "content": code
+            }
+          }
+        }).encode('utf-8')
 
-        The same type filtering of %who can be applied here.
+        response = urlopen("https://api.github.com/gists", post_data)
+        response_data = json.loads(response.read().decode('utf-8'))
+        return response_data['html_url']
 
-        For all variables, the type is printed. Additionally it prints:
+    def magic_loadpy(self, arg_s):
+        """Alias of `%load`
+        
+        `%loadpy` has gained some flexibility and droped the requirement of a `.py`
+        extension. So it has been renamed simply into %load. You can look at
+        `%load`'s docstring for more info.
+        """
+        self.magic_load(arg_s)
 
-          - For {},[],(): their length.
+    def magic_load(self, arg_s):
+        """Load code into the current frontend.
 
-          - For numpy arrays, a summary with shape, number of
-          elements, typecode and size in memory.
+        Usage:\\
+          %load [options] source
 
-          - Everything else: a string representation, snipping their middle if
-          too long.
+          where source can be a filename, URL, input history range or macro
 
-        Examples
+        Options:
         --------
+          -y : Don't ask confirmation for loading source above 200 000 characters.
 
-        Define two variables and list them with whos::
-
-          In [1]: alpha = 123
-
-          In [2]: beta = 'test'
+        This magic command can either take a local filename, a URL, an history
+        range (see %history) or a macro as argument, it will prompt for
+        confirmation before loading source with more than 200 000 characters, unless
+        -y flag is passed or if the frontend does not support raw_input::
 
-          In [3]: %whos
-          Variable   Type        Data/Info
-          --------------------------------
-          alpha      int         123
-          beta       str         test
+        %load myscript.py
+        %load 7-27
+        %load myMacro
+        %load http://www.example.com/myscript.py
         """
+        opts,args = self.parse_options(arg_s,'y')
 
-        varnames = self.magic_who_ls(parameter_s)
-        if not varnames:
-            if parameter_s:
-                print 'No variables match your requested type.'
-            else:
-                print 'Interactive namespace is empty.'
-            return
-
-        # if we have variables, move on...
+        contents = self.shell.find_user_code(args)
+        l = len(contents)
 
-        # for these types, show len() instead of data:
-        seq_types = ['dict', 'list', 'tuple']
+        # 200 000 is ~ 2500 full 80 caracter lines
+        # so in average, more than 5000 lines
+        if l > 200000 and 'y' not in opts:
+            try:
+                ans = self.shell.ask_yes_no(("The text you're trying to load seems pretty big"\
+                " (%d characters). Continue (y/[N]) ?" % l), default='n' )
+            except StdinNotImplementedError:
+                #asume yes if raw input not implemented
+                ans = True
 
-        # for numpy arrays, display summary info
-        ndarray_type = None
-        if 'numpy' in sys.modules:
+            if ans is False :
+                print 'Operation cancelled.'
+                return
+
+        self.set_next_input(contents)
+
+    def _find_edit_target(self, args, opts, last_call):
+        """Utility method used by magic_edit to find what to edit."""
+
+        def make_filename(arg):
+            "Make a filename from the given args"
+            arg = unquote_filename(arg)
             try:
-                from numpy import ndarray
-            except ImportError:
-                pass
-            else:
-                ndarray_type = ndarray.__name__
+                filename = get_py_filename(arg)
+            except IOError:
+                # If it ends with .py but doesn't already exist, assume we want
+                # a new file.
+                if arg.endswith('.py'):
+                    filename = arg
+                else:
+                    filename = None
+            return filename
 
-        # Find all variable names and types so we can figure out column sizes
-        def get_vars(i):
-            return self.shell.user_ns[i]
+        # Set a few locals from the options for convenience:
+        opts_prev = 'p' in opts
+        opts_raw = 'r' in opts
 
-        # some types are well known and can be shorter
-        abbrevs = {'IPython.core.macro.Macro' : 'Macro'}
-        def type_name(v):
-            tn = type(v).__name__
-            return abbrevs.get(tn,tn)
+        # custom exceptions
+        class DataIsObject(Exception): pass
 
-        varlist = map(get_vars,varnames)
+        # Default line number value
+        lineno = opts.get('n',None)
 
-        typelist = []
-        for vv in varlist:
-            tt = type_name(vv)
+        if opts_prev:
+            args = '_%s' % last_call[0]
+            if not self.shell.user_ns.has_key(args):
+                args = last_call[1]
 
-            if tt=='instance':
-                typelist.append( abbrevs.get(str(vv.__class__),
-                                             str(vv.__class__)))
-            else:
-                typelist.append(tt)
+        # use last_call to remember the state of the previous call, but don't
+        # let it be clobbered by successive '-p' calls.
+        try:
+            last_call[0] = self.shell.displayhook.prompt_count
+            if not opts_prev:
+                last_call[1] = args
+        except:
+            pass
 
-        # column labels and # of spaces as separator
-        varlabel = 'Variable'
-        typelabel = 'Type'
-        datalabel = 'Data/Info'
-        colsep = 3
-        # variable format strings
-        vformat    = "{0:<{varwidth}}{1:<{typewidth}}"
-        aformat    = "%s: %s elems, type `%s`, %s bytes"
-        # find the size of the columns to format the output nicely
-        varwidth = max(max(map(len,varnames)), len(varlabel)) + colsep
-        typewidth = max(max(map(len,typelist)), len(typelabel)) + colsep
-        # table header
-        print varlabel.ljust(varwidth) + typelabel.ljust(typewidth) + \
-              ' '+datalabel+'\n' + '-'*(varwidth+typewidth+len(datalabel)+1)
-        # and the table itself
-        kb = 1024
-        Mb = 1048576  # kb**2
-        for vname,var,vtype in zip(varnames,varlist,typelist):
-            print vformat.format(vname, vtype, varwidth=varwidth, typewidth=typewidth),
-            if vtype in seq_types:
-                print "n="+str(len(var))
-            elif vtype == ndarray_type:
-                vshape = str(var.shape).replace(',','').replace(' ','x')[1:-1]
-                if vtype==ndarray_type:
-                    # numpy
-                    vsize  = var.size
-                    vbytes = vsize*var.itemsize
-                    vdtype = var.dtype
+        # by default this is done with temp files, except when the given
+        # arg is a filename
+        use_temp = True
 
-                if vbytes < 100000:
-                    print aformat % (vshape,vsize,vdtype,vbytes)
-                else:
-                    print aformat % (vshape,vsize,vdtype,vbytes),
-                    if vbytes < Mb:
-                        print '(%s kb)' % (vbytes/kb,)
-                    else:
-                        print '(%s Mb)' % (vbytes/Mb,)
-            else:
+        data = ''
+
+        # First, see if the arguments should be a filename.
+        filename = make_filename(args)
+        if filename:
+            use_temp = False
+        elif args:
+            # Mode where user specifies ranges of lines, like in %macro.
+            data = self.shell.extract_input_lines(args, opts_raw)
+            if not data:
                 try:
-                    vstr = str(var)
-                except UnicodeEncodeError:
-                    vstr = unicode(var).encode(DEFAULT_ENCODING,
-                                               'backslashreplace')
-                except:
-                    vstr = "<object with id %d (str() failed)>" % id(var)
-                vstr = vstr.replace('\n','\\n')
-                if len(vstr) < 50:
-                    print vstr
-                else:
-                    print vstr[:25] + "<...>" + vstr[-25:]
+                    # Load the parameter given as a variable. If not a string,
+                    # process it as an object instead (below)
 
-    def magic_reset(self, parameter_s=''):
-        """Resets the namespace by removing all names defined by the user, if
-        called without arguments, or by removing some types of objects, such
-        as everything currently in IPython's In[] and Out[] containers (see
-        the parameters for details).
+                    #print '*** args',args,'type',type(args)  # dbg
+                    data = eval(args, self.shell.user_ns)
+                    if not isinstance(data, basestring):
+                        raise DataIsObject
 
-        Parameters
-        ----------
-        -f : force reset without asking for confirmation.
+                except (NameError,SyntaxError):
+                    # given argument is not a variable, try as a filename
+                    filename = make_filename(args)
+                    if filename is None:
+                        warn("Argument given (%s) can't be found as a variable "
+                             "or as a filename." % args)
+                        return
+                    use_temp = False
 
-        -s : 'Soft' reset: Only clears your namespace, leaving history intact.
-            References to objects may be kept. By default (without this option),
-            we do a 'hard' reset, giving you a new session and removing all
-            references to objects from the current session.
+                except DataIsObject:
+                    # macros have a special edit function
+                    if isinstance(data, Macro):
+                        raise MacroToEdit(data)
 
-        in : reset input history
-       
-        out : reset output history
-       
-        dhist : reset directory history
-        
-        array : reset only variables that are NumPy arrays
+                    # For objects, try to edit the file where they are defined
+                    try:
+                        filename = inspect.getabsfile(data)
+                        if 'fakemodule' in filename.lower() and inspect.isclass(data):
+                            # class created by %edit? Try to find source
+                            # by looking for method definitions instead, the
+                            # __module__ in those classes is FakeModule.
+                            attrs = [getattr(data, aname) for aname in dir(data)]
+                            for attr in attrs:
+                                if not inspect.ismethod(attr):
+                                    continue
+                                filename = inspect.getabsfile(attr)
+                                if filename and 'fakemodule' not in filename.lower():
+                                    # change the attribute to be the edit target instead
+                                    data = attr
+                                    break
 
-        See Also
-        --------
-        magic_reset_selective : invoked as ``%reset_selective``
+                        datafile = 1
+                    except TypeError:
+                        filename = make_filename(args)
+                        datafile = 1
+                        warn('Could not find file where `%s` is defined.\n'
+                             'Opening a file named `%s`' % (args,filename))
+                    # Now, make sure we can actually read the source (if it was in
+                    # a temp file it's gone by now).
+                    if datafile:
+                        try:
+                            if lineno is None:
+                                lineno = inspect.getsourcelines(data)[1]
+                        except IOError:
+                            filename = make_filename(args)
+                            if filename is None:
+                                warn('The file `%s` where `%s` was defined cannot '
+                                     'be read.' % (filename,data))
+                                return
+                    use_temp = False
 
-        Examples
-        --------
-        ::
+        if use_temp:
+            filename = self.shell.mktempfile(data)
+            print 'IPython will make a temporary file named:',filename
 
-          In [6]: a = 1
+        return filename, lineno, use_temp
 
-          In [7]: a
-          Out[7]: 1
+    def _edit_macro(self,mname,macro):
+        """open an editor with the macro data in a file"""
+        filename = self.shell.mktempfile(macro.value)
+        self.shell.hooks.editor(filename)
 
-          In [8]: 'a' in _ip.user_ns
-          Out[8]: True
+        # and make a new macro object, to replace the old one
+        mfile = open(filename)
+        mvalue = mfile.read()
+        mfile.close()
+        self.shell.user_ns[mname] = Macro(mvalue)
 
-          In [9]: %reset -f
+    def magic_ed(self,parameter_s=''):
+        """Alias to %edit."""
+        return self.magic_edit(parameter_s)
 
-          In [1]: 'a' in _ip.user_ns
-          Out[1]: False
+    @skip_doctest
+    def magic_edit(self,parameter_s='',last_call=['','']):
+        """Bring up an editor and execute the resulting code.
 
-          In [2]: %reset -f in
-          Flushing input history
+        Usage:
+          %edit [options] [args]
 
-          In [3]: %reset -f dhist in
-          Flushing directory history
-          Flushing input history
+        %edit runs IPython's editor hook. The default version of this hook is
+        set to call the editor specified by your $EDITOR environment variable.
+        If this isn't found, it will default to vi under Linux/Unix and to
+        notepad under Windows. See the end of this docstring for how to change
+        the editor hook.
 
-        Notes
-        -----
-        Calling this magic from clients that do not implement standard input,
-        such as the ipython notebook interface, will reset the namespace
-        without confirmation.
-        """
-        opts, args = self.parse_options(parameter_s,'sf', mode='list')
-        if 'f' in opts:
-            ans = True
-        else:
-            try:
-                ans = self.shell.ask_yes_no(
-                "Once deleted, variables cannot be recovered. Proceed (y/[n])? ", default='n')
-            except StdinNotImplementedError:
-                ans = True
-        if not ans:
-            print 'Nothing done.'
-            return
-
-        if 's' in opts:                     # Soft reset
-            user_ns = self.shell.user_ns
-            for i in self.magic_who_ls():
-                del(user_ns[i])
-        elif len(args) == 0:                # Hard reset
-            self.shell.reset(new_session = False)
-        
-        # reset in/out/dhist/array: previously extensinions/clearcmd.py
-        ip = self.shell
-        user_ns = self.shell.user_ns  # local lookup, heavily used
+        You can also set the value of this editor via the
+        ``TerminalInteractiveShell.editor`` option in your configuration file.
+        This is useful if you wish to use a different editor from your typical
+        default with IPython (and for Windows users who typically don't set
+        environment variables).
 
-        for target in args:
-            target = target.lower() # make matches case insensitive
-            if target == 'out':
-                print "Flushing output cache (%d entries)" % len(user_ns['_oh'])
-                self.shell.displayhook.flush()
+        This command allows you to conveniently edit multi-line code right in
+        your IPython session.
 
-            elif target == 'in':
-                print "Flushing input history"
-                pc = self.shell.displayhook.prompt_count + 1
-                for n in range(1, pc):
-                    key = '_i'+repr(n)
-                    user_ns.pop(key,None)
-                user_ns.update(dict(_i=u'',_ii=u'',_iii=u''))
-                hm = ip.history_manager
-                # don't delete these, as %save and %macro depending on the length
-                # of these lists to be preserved
-                hm.input_hist_parsed[:] = [''] * pc
-                hm.input_hist_raw[:] = [''] * pc
-                # hm has internal machinery for _i,_ii,_iii, clear it out
-                hm._i = hm._ii = hm._iii = hm._i00 =  u''
+        If called without arguments, %edit opens up an empty editor with a
+        temporary file and will execute the contents of this file when you
+        close it (don't forget to save it!).
 
-            elif target == 'array':
-                # Support cleaning up numpy arrays
-                try:
-                    from numpy import ndarray
-                    # This must be done with items and not iteritems because we're
-                    # going to modify the dict in-place.
-                    for x,val in user_ns.items():
-                        if isinstance(val,ndarray):
-                            del user_ns[x]
-                except ImportError:
-                    print "reset array only works if Numpy is available."
 
-            elif target == 'dhist':
-                print "Flushing directory history"
-                del user_ns['_dh'][:]
+        Options:
 
-            else:
-                print "Don't know how to reset ",
-                print target + ", please run `%reset?` for details"
+        -n <number>: open the editor at a specified line number.  By default,
+        the IPython editor hook uses the unix syntax 'editor +N filename', but
+        you can configure this by providing your own modified hook if your
+        favorite editor supports line-number specifications with a different
+        syntax.
 
-        gc.collect()
+        -p: this will call the editor with the same data as the previous time
+        it was used, regardless of how long ago (in your current session) it
+        was.
 
-    def magic_reset_selective(self, parameter_s=''):
-        """Resets the namespace by removing names defined by the user.
+        -r: use 'raw' input.  This option only applies to input taken from the
+        user's history.  By default, the 'processed' history is used, so that
+        magics are loaded in their transformed version to valid Python.  If
+        this option is given, the raw input as typed as the command line is
+        used instead.  When you exit the editor, it will be executed by
+        IPython's own processor.
 
-        Input/Output history are left around in case you need them.
+        -x: do not execute the edited code immediately upon exit. This is
+        mainly useful if you are editing programs which need to be called with
+        command line arguments, which you can then do using %run.
 
-        %reset_selective [-f] regex
 
-        No action is taken if regex is not included
+        Arguments:
 
-        Options
-          -f : force reset without asking for confirmation.
+        If arguments are given, the following possibilities exist:
 
-        See Also
-        --------
-        magic_reset : invoked as ``%reset``
+        - If the argument is a filename, IPython will load that into the
+          editor. It will execute its contents with execfile() when you exit,
+          loading any code in the file into your interactive namespace.
 
-        Examples
-        --------
+        - The arguments are ranges of input history,  e.g. "7 ~1/4-6".
+          The syntax is the same as in the %history magic.
 
-        We first fully reset the namespace so your output looks identical to
-        this example for pedagogical reasons; in practice you do not need a
-        full reset::
+        - If the argument is a string variable, its contents are loaded
+          into the editor. You can thus edit any string which contains
+          python code (including the result of previous edits).
 
-          In [1]: %reset -f
+        - If the argument is the name of an object (other than a string),
+          IPython will try to locate the file where it was defined and open the
+          editor at the point where it is defined. You can use `%edit function`
+          to load an editor exactly at the point where 'function' is defined,
+          edit it and have the file be executed automatically.
 
-        Now, with a clean namespace we can make a few variables and use
-        ``%reset_selective`` to only delete names that match our regexp::
+        - If the object is a macro (see %macro for details), this opens up your
+          specified editor with a temporary file containing the macro's data.
+          Upon exit, the macro is reloaded with the contents of the file.
 
-          In [2]: a=1; b=2; c=3; b1m=4; b2m=5; b3m=6; b4m=7; b2s=8
+        Note: opening at an exact line is only supported under Unix, and some
+        editors (like kedit and gedit up to Gnome 2.8) do not understand the
+        '+NUMBER' parameter necessary for this feature. Good editors like
+        (X)Emacs, vi, jed, pico and joe all do.
 
-          In [3]: who_ls
-          Out[3]: ['a', 'b', 'b1m', 'b2m', 'b2s', 'b3m', 'b4m', 'c']
+        After executing your code, %edit will return as output the code you
+        typed in the editor (except when it was an existing file). This way
+        you can reload the code in further invocations of %edit as a variable,
+        via _<NUMBER> or Out[<NUMBER>], where <NUMBER> is the prompt number of
+        the output.
 
-          In [4]: %reset_selective -f b[2-3]m
+        Note that %edit is also available through the alias %ed.
 
-          In [5]: who_ls
-          Out[5]: ['a', 'b', 'b1m', 'b2s', 'b4m', 'c']
+        This is an example of creating a simple function inside the editor and
+        then modifying it. First, start up the editor::
 
-          In [6]: %reset_selective -f d
+          In [1]: ed
+          Editing... done. Executing edited code...
+          Out[1]: 'def foo():\\n    print "foo() was defined in an editing
+          session"\\n'
 
-          In [7]: who_ls
-          Out[7]: ['a', 'b', 'b1m', 'b2s', 'b4m', 'c']
+        We can then call the function foo()::
 
-          In [8]: %reset_selective -f c
+          In [2]: foo()
+          foo() was defined in an editing session
 
-          In [9]: who_ls
-          Out[9]: ['a', 'b', 'b1m', 'b2s', 'b4m']
+        Now we edit foo.  IPython automatically loads the editor with the
+        (temporary) file where foo() was previously defined::
 
-          In [10]: %reset_selective -f b
+          In [3]: ed foo
+          Editing... done. Executing edited code...
 
-          In [11]: who_ls
-          Out[11]: ['a']
+        And if we call foo() again we get the modified version::
 
-        Notes
-        -----
-        Calling this magic from clients that do not implement standard input,
-        such as the ipython notebook interface, will reset the namespace
-        without confirmation.
-        """
+          In [4]: foo()
+          foo() has now been changed!
 
-        opts, regex = self.parse_options(parameter_s,'f')
+        Here is an example of how to edit a code snippet successive
+        times. First we call the editor::
 
-        if opts.has_key('f'):
-            ans = True
-        else:
-            try:
-                ans = self.shell.ask_yes_no(
-                "Once deleted, variables cannot be recovered. Proceed (y/[n])? ",
-                default='n')
-            except StdinNotImplementedError:
-                ans = True
-        if not ans:
-            print 'Nothing done.'
-            return
-        user_ns = self.shell.user_ns
-        if not regex:
-            print 'No regex pattern specified. Nothing done.'
-            return
-        else:
-            try:
-                m = re.compile(regex)
-            except TypeError:
-                raise TypeError('regex must be a string or compiled pattern')
-            for i in self.magic_who_ls():
-                if m.search(i):
-                    del(user_ns[i])
+          In [5]: ed
+          Editing... done. Executing edited code...
+          hello
+          Out[5]: "print 'hello'\\n"
 
-    def magic_xdel(self, parameter_s=''):
-        """Delete a variable, trying to clear it from anywhere that
-        IPython's machinery has references to it. By default, this uses
-        the identity of the named object in the user namespace to remove
-        references held under other names. The object is also removed
-        from the output history.
+        Now we call it again with the previous output (stored in _)::
 
-        Options
-          -n : Delete the specified name from all namespaces, without
-          checking their identity.
-        """
-        opts, varname = self.parse_options(parameter_s,'n')
-        try:
-            self.shell.del_var(varname, ('n' in opts))
-        except (NameError, ValueError) as e:
-            print type(e).__name__ +": "+ str(e)
+          In [6]: ed _
+          Editing... done. Executing edited code...
+          hello world
+          Out[6]: "print 'hello world'\\n"
 
-    def magic_logstart(self,parameter_s=''):
-        """Start logging anywhere in a session.
+        Now we call it with the output #8 (stored in _8, also as Out[8])::
 
-        %logstart [-o|-r|-t] [log_name [log_mode]]
+          In [7]: ed _8
+          Editing... done. Executing edited code...
+          hello again
+          Out[7]: "print 'hello again'\\n"
 
-        If no name is given, it defaults to a file named 'ipython_log.py' in your
-        current directory, in 'rotate' mode (see below).
 
-        '%logstart name' saves to file 'name' in 'backup' mode.  It saves your
-        history up to that point and then continues logging.
+        Changing the default editor hook:
 
-        %logstart takes a second optional parameter: logging mode. This can be one
-        of (note that the modes are given unquoted):\\
-          append: well, that says it.\\
-          backup: rename (if exists) to name~ and start name.\\
-          global: single logfile in your home dir, appended to.\\
-          over  : overwrite existing log.\\
-          rotate: create rotating logs name.1~, name.2~, etc.
+        If you wish to write your own editor hook, you can put it in a
+        configuration file which you load at startup time.  The default hook
+        is defined in the IPython.core.hooks module, and you can use that as a
+        starting example for further modifications.  That file also has
+        general instructions on how to set a new hook for use once you've
+        defined it."""
+        opts,args = self.parse_options(parameter_s,'prxn:')
 
-        Options:
+        try:
+            filename, lineno, is_temp = self._find_edit_target(args, opts, last_call)
+        except MacroToEdit as e:
+            self._edit_macro(args, e.args[0])
+            return
 
-          -o: log also IPython's output.  In this mode, all commands which
-          generate an Out[NN] prompt are recorded to the logfile, right after
-          their corresponding input line.  The output lines are always
-          prepended with a '#[Out]# ' marker, so that the log remains valid
-          Python code.
+        # do actual editing here
+        print 'Editing...',
+        sys.stdout.flush()
+        try:
+            # Quote filenames that may have spaces in them
+            if ' ' in filename:
+                filename = "'%s'" % filename
+            self.shell.hooks.editor(filename,lineno)
+        except TryNext:
+            warn('Could not open editor')
+            return
 
-          Since this marker is always the same, filtering only the output from
-          a log is very easy, using for example a simple awk call::
+        # XXX TODO: should this be generalized for all string vars?
+        # For now, this is special-cased to blocks created by cpaste
+        if args.strip() == 'pasted_block':
+            self.shell.user_ns['pasted_block'] = file_read(filename)
 
-            awk -F'#\\[Out\\]# ' '{if($2) {print $2}}' ipython_log.py
+        if 'x' in opts:  # -x prevents actual execution
+            print
+        else:
+            print 'done. Executing edited code...'
+            if 'r' in opts:    # Untranslated IPython code
+                self.shell.run_cell(file_read(filename),
+                                                    store_history=False)
+            else:
+                self.shell.safe_execfile(filename, self.shell.user_ns,
+                                         self.shell.user_ns)
 
-          -r: log 'raw' input.  Normally, IPython's logs contain the processed
-          input, so that user lines are logged in their final form, converted
-          into valid Python.  For example, %Exit is logged as
-          _ip.magic("Exit").  If the -r flag is given, all input is logged
-          exactly as typed, with no transformations applied.
+        if is_temp:
+            try:
+                return open(filename).read()
+            except IOError,msg:
+                if msg.filename == filename:
+                    warn('File not found. Did you forget to save?')
+                    return
+                else:
+                    self.shell.showtraceback()
 
-          -t: put timestamps before each input line logged (these are put in
-          comments)."""
 
-        opts,par = self.parse_options(parameter_s,'ort')
-        log_output = 'o' in opts
-        log_raw_input = 'r' in opts
-        timestamp = 't' in opts
+class ConfigMagics(MagicFunctions):
 
-        logger = self.shell.logger
+    def __init__(self, shell):
+        super(ProfileMagics, self).__init__(shell)
+        self.configurables = []
 
-        # if no args are given, the defaults set in the logger constructor by
-        # ipython remain valid
-        if par:
-            try:
-                logfname,logmode = par.split()
-            except:
-                logfname = par
-                logmode = 'backup'
-        else:
-            logfname = logger.logfname
-            logmode = logger.logmode
-        # put logfname into rc struct as if it had been called on the command
-        # line, so it ends up saved in the log header Save it in case we need
-        # to restore it...
-        old_logfile = self.shell.logfile
-        if logfname:
-            logfname = os.path.expanduser(logfname)
-        self.shell.logfile = logfname
+    def magic_config(self, s):
+        """configure IPython
 
-        loghead = '# IPython log file\n\n'
-        try:
-            started  = logger.logstart(logfname,loghead,logmode,
-                                       log_output,timestamp,log_raw_input)
-        except:
-            self.shell.logfile = old_logfile
-            warn("Couldn't start log: %s" % sys.exc_info()[1])
-        else:
-            # log input history up to this point, optionally interleaving
-            # output if requested
+            %config Class[.trait=value]
 
-            if timestamp:
-                # disable timestamping for the previous history, since we've
-                # lost those already (no time machine here).
-                logger.timestamp = False
+        This magic exposes most of the IPython config system. Any
+        Configurable class should be able to be configured with the simple
+        line::
 
-            if log_raw_input:
-                input_hist = self.shell.history_manager.input_hist_raw
-            else:
-                input_hist = self.shell.history_manager.input_hist_parsed
+            %config Class.trait=value
 
-            if log_output:
-                log_write = logger.log_write
-                output_hist = self.shell.history_manager.output_hist
-                for n in range(1,len(input_hist)-1):
-                    log_write(input_hist[n].rstrip() + '\n')
-                    if n in output_hist:
-                        log_write(repr(output_hist[n]),'output')
-            else:
-                logger.log_write('\n'.join(input_hist[1:]))
-                logger.log_write('\n')
-            if timestamp:
-                # re-enable timestamping
-                logger.timestamp = True
+        Where `value` will be resolved in the user's namespace, if it is an
+        expression or variable name.
 
-            print ('Activating auto-logging. '
-                   'Current session state plus future input saved.')
-            logger.logstate()
+        Examples
+        --------
 
-    def magic_logstop(self,parameter_s=''):
-        """Fully stop logging and close log file.
+        To see what classes are available for config, pass no arguments::
 
-        In order to start logging again, a new %logstart call needs to be made,
-        possibly (though not necessarily) with a new filename, mode and other
-        options."""
-        self.logger.logstop()
+            In [1]: %config
+            Available objects for config:
+                TerminalInteractiveShell
+                HistoryManager
+                PrefilterManager
+                AliasManager
+                IPCompleter
+                PromptManager
+                DisplayFormatter
 
-    def magic_logoff(self,parameter_s=''):
-        """Temporarily stop logging.
+        To view what is configurable on a given class, just pass the class
+        name::
 
-        You must have previously started logging."""
-        self.shell.logger.switch_log(0)
+            In [2]: %config IPCompleter
+            IPCompleter options
+            -----------------
+            IPCompleter.omit__names=<Enum>
+                Current: 2
+                Choices: (0, 1, 2)
+                Instruct the completer to omit private method names
+                Specifically, when completing on ``object.<tab>``.
+                When 2 [default]: all names that start with '_' will be excluded.
+                When 1: all 'magic' names (``__foo__``) will be excluded.
+                When 0: nothing will be excluded.
+            IPCompleter.merge_completions=<CBool>
+                Current: True
+                Whether to merge completion results into a single list
+                If False, only the completion results from the first non-empty completer
+                will be returned.
+            IPCompleter.limit_to__all__=<CBool>
+                Current: False
+                Instruct the completer to use __all__ for the completion
+                Specifically, when completing on ``object.<tab>``.
+                When True: only those names in obj.__all__ will be included.
+                When False [default]: the __all__ attribute is ignored
+            IPCompleter.greedy=<CBool>
+                Current: False
+                Activate greedy completion
+                This will enable completion on elements of lists, results of function calls,
+                etc., but can be unsafe because the code is actually evaluated on TAB.
 
-    def magic_logon(self,parameter_s=''):
-        """Restart logging.
+        but the real use is in setting values::
 
-        This function is for restarting logging which you've temporarily
-        stopped with %logoff. For starting logging for the first time, you
-        must use the %logstart function, which allows you to specify an
-        optional log filename."""
+            In [3]: %config IPCompleter.greedy = True
 
-        self.shell.logger.switch_log(1)
+        and these values are read from the user_ns if they are variables::
 
-    def magic_logstate(self,parameter_s=''):
-        """Print the status of the logging system."""
+            In [4]: feeling_greedy=False
 
-        self.shell.logger.logstate()
+            In [5]: %config IPCompleter.greedy = feeling_greedy
 
-    def magic_pdb(self, parameter_s=''):
-        """Control the automatic calling of the pdb interactive debugger.
+        """
+        from IPython.config.loader import Config
+        # some IPython objects are Configurable, but do not yet have
+        # any configurable traits.  Exclude them from the effects of
+        # this magic, as their presence is just noise:
+        configurables = [ c for c in self.shell.configurables
+                          if c.__class__.class_traits(config=True) ]
+        classnames = [ c.__class__.__name__ for c in configurables ]
 
-        Call as '%pdb on', '%pdb 1', '%pdb off' or '%pdb 0'. If called without
-        argument it works as a toggle.
+        line = s.strip()
+        if not line:
+            # print available configurable names
+            print "Available objects for config:"
+            for name in classnames:
+                print "    ", name
+            return
+        elif line in classnames:
+            # `%config TerminalInteractiveShell` will print trait info for
+            # TerminalInteractiveShell
+            c = configurables[classnames.index(line)]
+            cls = c.__class__
+            help = cls.class_get_help(c)
+            # strip leading '--' from cl-args:
+            help = re.sub(re.compile(r'^--', re.MULTILINE), '', help)
+            print help
+            return
+        elif '=' not in line:
+            raise UsageError("Invalid config statement: %r, should be Class.trait = value" % line)
 
-        When an exception is triggered, IPython can optionally call the
-        interactive pdb debugger after the traceback printout. %pdb toggles
-        this feature on and off.
 
-        The initial state of this feature is set in your configuration
-        file (the option is ``InteractiveShell.pdb``).
+        # otherwise, assume we are setting configurables.
+        # leave quotes on args when splitting, because we want
+        # unquoted args to eval in user_ns
+        cfg = Config()
+        exec "cfg."+line in locals(), self.shell.user_ns
 
-        If you want to just activate the debugger AFTER an exception has fired,
-        without having to type '%pdb on' and rerunning your code, you can use
-        the %debug magic."""
+        for configurable in configurables:
+            try:
+                configurable.update_config(cfg)
+            except Exception as e:
+                error(e)
 
-        par = parameter_s.strip().lower()
 
-        if par:
-            try:
-                new_pdb = {'off':0,'0':0,'on':1,'1':1}[par]
-            except KeyError:
-                print ('Incorrect argument. Use on/1, off/0, '
-                       'or nothing for a toggle.')
-                return
-        else:
-            # toggle
-            new_pdb = not self.shell.call_pdb
+class NamespaceMagics(MagicFunctions):
+    """Magics to manage various aspects of the user's namespace.
 
-        # set on the shell
-        self.shell.call_pdb = new_pdb
-        print 'Automatic pdb calling has been turned',on_off(new_pdb)
+    These include listing variables, introspecting into them, etc.
+    """
 
-    def magic_debug(self, parameter_s=''):
-        """Activate the interactive debugger in post-mortem mode.
+    def magic_pinfo(self, parameter_s='', namespaces=None):
+        """Provide detailed information about an object.
 
-        If an exception has just occurred, this lets you inspect its stack
-        frames interactively.  Note that this will always work only on the last
-        traceback that occurred, so you must call this quickly after an
-        exception that you wish to inspect has fired, because if another one
-        occurs, it clobbers the previous one.
+        '%pinfo object' is just a synonym for object? or ?object."""
 
-        If you want IPython to automatically do this on every exception, see
-        the %pdb magic for more details.
-        """
-        self.shell.debugger(force=True)
+        #print 'pinfo par: <%s>' % parameter_s  # dbg
 
-    @skip_doctest
-    def magic_prun(self, parameter_s ='',user_mode=1,
-                   opts=None,arg_lst=None,prog_ns=None):
 
-        """Run a statement through the python code profiler.
+        # detail_level: 0 -> obj? , 1 -> obj??
+        detail_level = 0
+        # We need to detect if we got called as 'pinfo pinfo foo', which can
+        # happen if the user types 'pinfo foo?' at the cmd line.
+        pinfo,qmark1,oname,qmark2 = \
+               re.match('(pinfo )?(\?*)(.*?)(\??$)',parameter_s).groups()
+        if pinfo or qmark1 or qmark2:
+            detail_level = 1
+        if "*" in oname:
+            self.magic_psearch(oname)
+        else:
+            self.shell._inspect('pinfo', oname, detail_level=detail_level,
+                                namespaces=namespaces)
 
-        Usage:
-          %prun [options] statement
+    def magic_pinfo2(self, parameter_s='', namespaces=None):
+        """Provide extra detailed information about an object.
 
-        The given statement (which doesn't require quote marks) is run via the
-        python profiler in a manner similar to the profile.run() function.
-        Namespaces are internally managed to work correctly; profile.run
-        cannot be used in IPython because it makes certain assumptions about
-        namespaces which do not hold under IPython.
+        '%pinfo2 object' is just a synonym for object?? or ??object."""
+        self.shell._inspect('pinfo', parameter_s, detail_level=1,
+                            namespaces=namespaces)
 
-        Options:
+    @skip_doctest
+    def magic_pdef(self, parameter_s='', namespaces=None):
+        """Print the definition header for any callable object.
 
-        -l <limit>: you can place restrictions on what or how much of the
-        profile gets printed. The limit value can be:
+        If the object is a class, print the constructor information.
 
-          * A string: only information for function names containing this string
-          is printed.
+        Examples
+        --------
+        ::
 
-          * An integer: only these many lines are printed.
+          In [3]: %pdef urllib.urlopen
+          urllib.urlopen(url, data=None, proxies=None)
+        """
+        self._inspect('pdef',parameter_s, namespaces)
 
-          * A float (between 0 and 1): this fraction of the report is printed
-          (for example, use a limit of 0.4 to see the topmost 40% only).
+    def magic_pdoc(self, parameter_s='', namespaces=None):
+        """Print the docstring for an object.
 
-        You can combine several limits with repeated use of the option. For
-        example, '-l __init__ -l 5' will print only the topmost 5 lines of
-        information about class constructors.
+        If the given object is a class, it will print both the class and the
+        constructor docstrings."""
+        self._inspect('pdoc',parameter_s, namespaces)
 
-        -r: return the pstats.Stats object generated by the profiling. This
-        object has all the information about the profile in it, and you can
-        later use it for further analysis or in other functions.
+    def magic_psource(self, parameter_s='', namespaces=None):
+        """Print (or run through pager) the source code for an object."""
+        self._inspect('psource',parameter_s, namespaces)
 
-       -s <key>: sort profile by given key. You can provide more than one key
-        by using the option several times: '-s key1 -s key2 -s key3...'. The
-        default sorting key is 'time'.
+    def magic_pfile(self, parameter_s=''):
+        """Print (or run through pager) the file where an object is defined.
 
-        The following is copied verbatim from the profile documentation
-        referenced below:
+        The file opens at the line where the object definition begins. IPython
+        will honor the environment variable PAGER if set, and otherwise will
+        do its best to print the file in a convenient form.
 
-        When more than one key is provided, additional keys are used as
-        secondary criteria when the there is equality in all keys selected
-        before them.
+        If the given argument is not an object currently defined, IPython will
+        try to interpret it as a filename (automatically adding a .py extension
+        if needed). You can thus use %pfile as a syntax highlighting code
+        viewer."""
 
-        Abbreviations can be used for any key names, as long as the
-        abbreviation is unambiguous.  The following are the keys currently
-        defined:
+        # first interpret argument as an object name
+        out = self._inspect('pfile',parameter_s)
+        # if not, try the input as a filename
+        if out == 'not found':
+            try:
+                filename = get_py_filename(parameter_s)
+            except IOError,msg:
+                print msg
+                return
+            page.page(self.shell.inspector.format(open(filename).read()))
 
-                Valid Arg       Meaning
-                  "calls"      call count
-                  "cumulative" cumulative time
-                  "file"       file name
-                  "module"     file name
-                  "pcalls"     primitive call count
-                  "line"       line number
-                  "name"       function name
-                  "nfl"        name/file/line
-                  "stdname"    standard name
-                  "time"       internal time
+    def magic_psearch(self, parameter_s=''):
+        """Search for object in namespaces by wildcard.
 
-        Note that all sorts on statistics are in descending order (placing
-        most time consuming items first), where as name, file, and line number
-        searches are in ascending order (i.e., alphabetical). The subtle
-        distinction between "nfl" and "stdname" is that the standard name is a
-        sort of the name as printed, which means that the embedded line
-        numbers get compared in an odd way.  For example, lines 3, 20, and 40
-        would (if the file names were the same) appear in the string order
-        "20" "3" and "40".  In contrast, "nfl" does a numeric compare of the
-        line numbers.  In fact, sort_stats("nfl") is the same as
-        sort_stats("name", "file", "line").
+        %psearch [options] PATTERN [OBJECT TYPE]
 
-        -T <filename>: save profile results as shown on screen to a text
-        file. The profile is still shown on screen.
+        Note: ? can be used as a synonym for %psearch, at the beginning or at
+        the end: both a*? and ?a* are equivalent to '%psearch a*'.  Still, the
+        rest of the command line must be unchanged (options come first), so
+        for example the following forms are equivalent
 
-        -D <filename>: save (via dump_stats) profile statistics to given
-        filename. This data is in a format understood by the pstats module, and
-        is generated by a call to the dump_stats() method of profile
-        objects. The profile is still shown on screen.
+        %psearch -i a* function
+        -i a* function?
+        ?-i a* function
 
-        -q: suppress output to the pager.  Best used with -T and/or -D above.
+        Arguments:
 
-        If you want to run complete programs under the profiler's control, use
-        '%run -p [prof_opts] filename.py [args to program]' where prof_opts
-        contains profiler specific options as described here.
+          PATTERN
 
-        You can read the complete documentation for the profile module with::
+          where PATTERN is a string containing * as a wildcard similar to its
+          use in a shell.  The pattern is matched in all namespaces on the
+          search path. By default objects starting with a single _ are not
+          matched, many IPython generated objects have a single
+          underscore. The default is case insensitive matching. Matching is
+          also done on the attributes of objects and not only on the objects
+          in a module.
 
-          In [1]: import profile; profile.help()
-        """
+          [OBJECT TYPE]
 
-        opts_def = Struct(D=[''],l=[],s=['time'],T=[''])
+          Is the name of a python type from the types module. The name is
+          given in lowercase without the ending type, ex. StringType is
+          written string. By adding a type here only objects matching the
+          given type are matched. Using all here makes the pattern match all
+          types (this is the default).
 
-        if user_mode:  # regular user call
-            opts,arg_str = self.parse_options(parameter_s,'D:l:rs:T:q',
-                                              list_all=1, posix=False)
-            namespace = self.shell.user_ns
-        else:  # called to run a program by %run -p
-            try:
-                filename = get_py_filename(arg_lst[0])
-            except IOError as e:
-                try:
-                    msg = str(e)
-                except UnicodeError:
-                    msg = e.message
-                error(msg)
-                return
+        Options:
 
-            arg_str = 'execfile(filename,prog_ns)'
-            namespace = {
-                'execfile': self.shell.safe_execfile,
-                'prog_ns': prog_ns,
-                'filename': filename
-                }
+          -a: makes the pattern match even objects whose names start with a
+          single underscore.  These names are normally omitted from the
+          search.
 
-        opts.merge(opts_def)
+          -i/-c: make the pattern case insensitive/sensitive.  If neither of
+          these options are given, the default is read from your configuration
+          file, with the option ``InteractiveShell.wildcards_case_sensitive``.
+          If this option is not specified in your configuration file, IPython's
+          internal default is to do a case sensitive search.
 
-        prof = profile.Profile()
-        try:
-            prof = prof.runctx(arg_str,namespace,namespace)
-            sys_exit = ''
-        except SystemExit:
-            sys_exit = """*** SystemExit exception caught in code being profiled."""
+          -e/-s NAMESPACE: exclude/search a given namespace.  The pattern you
+          specify can be searched in any of the following namespaces:
+          'builtin', 'user', 'user_global','internal', 'alias', where
+          'builtin' and 'user' are the search defaults.  Note that you should
+          not use quotes when specifying namespaces.
 
-        stats = pstats.Stats(prof).strip_dirs().sort_stats(*opts.s)
+          'Builtin' contains the python module builtin, 'user' contains all
+          user data, 'alias' only contain the shell aliases and no python
+          objects, 'internal' contains objects used by IPython.  The
+          'user_global' namespace is only used by embedded IPython instances,
+          and it contains module-level globals.  You can add namespaces to the
+          search with -s or exclude them with -e (these options can be given
+          more than once).
 
-        lims = opts.l
-        if lims:
-            lims = []  # rebuild lims with ints/floats/strings
-            for lim in opts.l:
-                try:
-                    lims.append(int(lim))
-                except ValueError:
-                    try:
-                        lims.append(float(lim))
-                    except ValueError:
-                        lims.append(lim)
+        Examples
+        --------
+        ::
 
-        # Trap output.
-        stdout_trap = StringIO()
+          %psearch a*            -> objects beginning with an a
+          %psearch -e builtin a* -> objects NOT in the builtin space starting in a
+          %psearch a* function   -> all functions beginning with an a
+          %psearch re.e*         -> objects beginning with an e in module re
+          %psearch r*.e*         -> objects that start with e in modules starting in r
+          %psearch r*.* string   -> all strings in modules beginning with r
 
-        if hasattr(stats,'stream'):
-            # In newer versions of python, the stats object has a 'stream'
-            # attribute to write into.
-            stats.stream = stdout_trap
-            stats.print_stats(*lims)
-        else:
-            # For older versions, we manually redirect stdout during printing
-            sys_stdout = sys.stdout
-            try:
-                sys.stdout = stdout_trap
-                stats.print_stats(*lims)
-            finally:
-                sys.stdout = sys_stdout
+        Case sensitive search::
 
-        output = stdout_trap.getvalue()
-        output = output.rstrip()
+          %psearch -c a*         list all object beginning with lower case a
 
-        if 'q' not in opts:
-            page.page(output)
-        print sys_exit,
+        Show objects beginning with a single _::
 
-        dump_file = opts.D[0]
-        text_file = opts.T[0]
-        if dump_file:
-            dump_file = unquote_filename(dump_file)
-            prof.dump_stats(dump_file)
-            print '\n*** Profile stats marshalled to file',\
-                  `dump_file`+'.',sys_exit
-        if text_file:
-            text_file = unquote_filename(text_file)
-            pfile = open(text_file,'w')
-            pfile.write(output)
-            pfile.close()
-            print '\n*** Profile printout saved to text file',\
-                  `text_file`+'.',sys_exit
+          %psearch -a _*         list objects beginning with a single underscore"""
+        try:
+            parameter_s.encode('ascii')
+        except UnicodeEncodeError:
+            print 'Python identifiers can only contain ascii characters.'
+            return
 
-        if opts.has_key('r'):
-            return stats
-        else:
-            return None
+        # default namespaces to be searched
+        def_search = ['user_local', 'user_global', 'builtin']
 
-    @skip_doctest
-    def magic_run(self, parameter_s ='', runner=None,
-                  file_finder=get_py_filename):
-        """Run the named file inside IPython as a program.
+        # Process options/args
+        opts,args = self.parse_options(parameter_s,'cias:e:',list_all=True)
+        opt = opts.get
+        shell = self.shell
+        psearch = shell.inspector.psearch
 
-        Usage:\\
-          %run [-n -i -t [-N<N>] -d [-b<N>] -p [profile options]] file [args]
+        # select case options
+        if opts.has_key('i'):
+            ignore_case = True
+        elif opts.has_key('c'):
+            ignore_case = False
+        else:
+            ignore_case = not shell.wildcards_case_sensitive
 
-        Parameters after the filename are passed as command-line arguments to
-        the program (put in sys.argv). Then, control returns to IPython's
-        prompt.
+        # Build list of namespaces to search from user options
+        def_search.extend(opt('s',[]))
+        ns_exclude = ns_exclude=opt('e',[])
+        ns_search = [nm for nm in def_search if nm not in ns_exclude]
 
-        This is similar to running at a system prompt:\\
-          $ python file args\\
-        but with the advantage of giving you IPython's tracebacks, and of
-        loading all variables into your interactive namespace for further use
-        (unless -p is used, see below).
+        # Call the actual search
+        try:
+            psearch(args,shell.ns_table,ns_search,
+                    show_all=opt('a'),ignore_case=ignore_case)
+        except:
+            shell.showtraceback()
 
-        The file is executed in a namespace initially consisting only of
-        __name__=='__main__' and sys.argv constructed as indicated. It thus
-        sees its environment as if it were being run as a stand-alone program
-        (except for sharing global objects such as previously imported
-        modules). But after execution, the IPython interactive namespace gets
-        updated with all variables defined in the program (except for __name__
-        and sys.argv). This allows for very convenient loading of code for
-        interactive work, while giving each program a 'clean sheet' to run in.
+    @skip_doctest
+    def magic_who_ls(self, parameter_s=''):
+        """Return a sorted list of all interactive variables.
 
-        Options:
+        If arguments are given, only variables of types matching these
+        arguments are returned.
 
-        -n: __name__ is NOT set to '__main__', but to the running file's name
-        without extension (as python does under import).  This allows running
-        scripts and reloading the definitions in them without calling code
-        protected by an ' if __name__ == "__main__" ' clause.
+        Examples
+        --------
 
-        -i: run the file in IPython's namespace instead of an empty one. This
-        is useful if you are experimenting with code written in a text editor
-        which depends on variables defined interactively.
+        Define two variables and list them with who_ls::
 
-        -e: ignore sys.exit() calls or SystemExit exceptions in the script
-        being run.  This is particularly useful if IPython is being used to
-        run unittests, which always exit with a sys.exit() call.  In such
-        cases you are interested in the output of the test results, not in
-        seeing a traceback of the unittest module.
+          In [1]: alpha = 123
 
-        -t: print timing information at the end of the run.  IPython will give
-        you an estimated CPU time consumption for your script, which under
-        Unix uses the resource module to avoid the wraparound problems of
-        time.clock().  Under Unix, an estimate of time spent on system tasks
-        is also given (for Windows platforms this is reported as 0.0).
+          In [2]: beta = 'test'
 
-        If -t is given, an additional -N<N> option can be given, where <N>
-        must be an integer indicating how many times you want the script to
-        run.  The final timing report will include total and per run results.
+          In [3]: %who_ls
+          Out[3]: ['alpha', 'beta']
 
-        For example (testing the script uniq_stable.py)::
+          In [4]: %who_ls int
+          Out[4]: ['alpha']
 
-            In [1]: run -t uniq_stable
+          In [5]: %who_ls str
+          Out[5]: ['beta']
+        """
 
-            IPython CPU timings (estimated):\\
-              User  :    0.19597 s.\\
-              System:        0.0 s.\\
+        user_ns = self.shell.user_ns
+        user_ns_hidden = self.shell.user_ns_hidden
+        out = [ i for i in user_ns
+                if not i.startswith('_') \
+                and not i in user_ns_hidden ]
 
-            In [2]: run -t -N5 uniq_stable
+        typelist = parameter_s.split()
+        if typelist:
+            typeset = set(typelist)
+            out = [i for i in out if type(user_ns[i]).__name__ in typeset]
 
-            IPython CPU timings (estimated):\\
-            Total runs performed: 5\\
-              Times :      Total       Per run\\
-              User  :   0.910862 s,  0.1821724 s.\\
-              System:        0.0 s,        0.0 s.
+        out.sort()
+        return out
 
-        -d: run your program under the control of pdb, the Python debugger.
-        This allows you to execute your program step by step, watch variables,
-        etc.  Internally, what IPython does is similar to calling:
+    @skip_doctest
+    def magic_who(self, parameter_s=''):
+        """Print all interactive variables, with some minimal formatting.
 
-          pdb.run('execfile("YOURFILENAME")')
+        If any arguments are given, only variables whose type matches one of
+        these are printed.  For example::
 
-        with a breakpoint set on line 1 of your file.  You can change the line
-        number for this automatic breakpoint to be <N> by using the -bN option
-        (where N must be an integer).  For example::
+          %who function str
 
-          %run -d -b40 myscript
+        will only list functions and strings, excluding all other types of
+        variables.  To find the proper type names, simply use type(var) at a
+        command line to see how python prints type names.  For example:
 
-        will set the first breakpoint at line 40 in myscript.py.  Note that
-        the first breakpoint must be set on a line which actually does
-        something (not a comment or docstring) for it to stop execution.
+        ::
 
-        When the pdb debugger starts, you will see a (Pdb) prompt.  You must
-        first enter 'c' (without quotes) to start execution up to the first
-        breakpoint.
+          In [1]: type('hello')\\
+          Out[1]: <type 'str'>
 
-        Entering 'help' gives information about the use of the debugger.  You
-        can easily see pdb's full documentation with "import pdb;pdb.help()"
-        at a prompt.
+        indicates that the type name for strings is 'str'.
 
-        -p: run program under the control of the Python profiler module (which
-        prints a detailed report of execution times, function calls, etc).
+        ``%who`` always excludes executed names loaded through your configuration
+        file and things which are internal to IPython.
 
-        You can pass other options after -p which affect the behavior of the
-        profiler itself. See the docs for %prun for details.
+        This is deliberate, as typically you may load many modules and the
+        purpose of %who is to show you only what you've manually defined.
 
-        In this mode, the program's variables do NOT propagate back to the
-        IPython interactive namespace (because they remain in the namespace
-        where the profiler executes them).
+        Examples
+        --------
 
-        Internally this triggers a call to %prun, see its documentation for
-        details on the options available specifically for profiling.
+        Define two variables and list them with who::
 
-        There is one special usage for which the text above doesn't apply:
-        if the filename ends with .ipy, the file is run as ipython script,
-        just as if the commands were written on IPython prompt.
+          In [1]: alpha = 123
 
-        -m: specify module name to load instead of script path. Similar to
-        the -m option for the python interpreter. Use this option last if you
-        want to combine with other %run options. Unlike the python interpreter
-        only source modules are allowed no .pyc or .pyo files.
-        For example::
+          In [2]: beta = 'test'
 
-            %run -m example
+          In [3]: %who
+          alpha   beta
 
-        will run the example module.
+          In [4]: %who int
+          alpha
 
+          In [5]: %who str
+          beta
         """
 
-        # get arguments and set sys.argv for program to be run.
-        opts, arg_lst = self.parse_options(parameter_s, 'nidtN:b:pD:l:rs:T:em:',
-                                           mode='list', list_all=1)
-        if "m" in opts:
-            modulename = opts["m"][0]
-            modpath = find_mod(modulename)
-            if modpath is None:
-                warn('%r is not a valid modulename on sys.path'%modulename)
-                return
-            arg_lst = [modpath] + arg_lst
-        try:
-            filename = file_finder(arg_lst[0])
-        except IndexError:
-            warn('you must provide at least a filename.')
-            print '\n%run:\n', oinspect.getdoc(self.magic_run)
-            return
-        except IOError as e:
-            try:
-                msg = str(e)
-            except UnicodeError:
-                msg = e.message
-            error(msg)
+        varlist = self.magic_who_ls(parameter_s)
+        if not varlist:
+            if parameter_s:
+                print 'No variables match your requested type.'
+            else:
+                print 'Interactive namespace is empty.'
             return
 
-        if filename.lower().endswith('.ipy'):
-            self.shell.safe_execfile_ipy(filename)
-            return
+        # if we have variables, move on...
+        count = 0
+        for i in varlist:
+            print i+'\t',
+            count += 1
+            if count > 8:
+                count = 0
+                print
+        print
 
-        # Control the response to exit() calls made by the script being run
-        exit_ignore = 'e' in opts
+    @skip_doctest
+    def magic_whos(self, parameter_s=''):
+        """Like %who, but gives some extra information about each variable.
 
-        # Make sure that the running script gets a proper sys.argv as if it
-        # were run from a system shell.
-        save_argv = sys.argv # save it for later restoring
+        The same type filtering of %who can be applied here.
 
-        # simulate shell expansion on arguments, at least tilde expansion
-        args = [ os.path.expanduser(a) for a in arg_lst[1:] ]
+        For all variables, the type is printed. Additionally it prints:
 
-        sys.argv = [filename] + args  # put in the proper filename
-        # protect sys.argv from potential unicode strings on Python 2:
-        if not py3compat.PY3:
-            sys.argv = [ py3compat.cast_bytes(a) for a in sys.argv ]
+          - For {},[],(): their length.
 
-        if 'i' in opts:
-            # Run in user's interactive namespace
-            prog_ns = self.shell.user_ns
-            __name__save = self.shell.user_ns['__name__']
-            prog_ns['__name__'] = '__main__'
-            main_mod = self.shell.new_main_mod(prog_ns)
-        else:
-            # Run in a fresh, empty namespace
-            if 'n' in opts:
-                name = os.path.splitext(os.path.basename(filename))[0]
-            else:
-                name = '__main__'
+          - For numpy arrays, a summary with shape, number of
+          elements, typecode and size in memory.
 
-            main_mod = self.shell.new_main_mod()
-            prog_ns = main_mod.__dict__
-            prog_ns['__name__'] = name
+          - Everything else: a string representation, snipping their middle if
+          too long.
 
-        # Since '%run foo' emulates 'python foo.py' at the cmd line, we must
-        # set the __file__ global in the script's namespace
-        prog_ns['__file__'] = filename
+        Examples
+        --------
 
-        # pickle fix.  See interactiveshell for an explanation.  But we need to make sure
-        # that, if we overwrite __main__, we replace it at the end
-        main_mod_name = prog_ns['__name__']
+        Define two variables and list them with whos::
 
-        if main_mod_name == '__main__':
-            restore_main = sys.modules['__main__']
-        else:
-            restore_main = False
+          In [1]: alpha = 123
 
-        # This needs to be undone at the end to prevent holding references to
-        # every single object ever created.
-        sys.modules[main_mod_name] = main_mod
+          In [2]: beta = 'test'
 
-        try:
-            stats = None
-            with self.shell.readline_no_record:
-                if 'p' in opts:
-                    stats = self.magic_prun('', 0, opts, arg_lst, prog_ns)
-                else:
-                    if 'd' in opts:
-                        deb = debugger.Pdb(self.shell.colors)
-                        # reset Breakpoint state, which is moronically kept
-                        # in a class
-                        bdb.Breakpoint.next = 1
-                        bdb.Breakpoint.bplist = {}
-                        bdb.Breakpoint.bpbynumber = [None]
-                        # Set an initial breakpoint to stop execution
-                        maxtries = 10
-                        bp = int(opts.get('b', [1])[0])
-                        checkline = deb.checkline(filename, bp)
-                        if not checkline:
-                            for bp in range(bp + 1, bp + maxtries + 1):
-                                if deb.checkline(filename, bp):
-                                    break
-                            else:
-                                msg = ("\nI failed to find a valid line to set "
-                                       "a breakpoint\n"
-                                       "after trying up to line: %s.\n"
-                                       "Please set a valid breakpoint manually "
-                                       "with the -b option." % bp)
-                                error(msg)
-                                return
-                        # if we find a good linenumber, set the breakpoint
-                        deb.do_break('%s:%s' % (filename, bp))
-                        # Start file run
-                        print "NOTE: Enter 'c' at the",
-                        print "%s prompt to start your script." % deb.prompt
-                        ns = {'execfile': py3compat.execfile, 'prog_ns': prog_ns}
-                        try:
-                            deb.run('execfile("%s", prog_ns)' % filename, ns)
+          In [3]: %whos
+          Variable   Type        Data/Info
+          --------------------------------
+          alpha      int         123
+          beta       str         test
+        """
 
-                        except:
-                            etype, value, tb = sys.exc_info()
-                            # Skip three frames in the traceback: the %run one,
-                            # one inside bdb.py, and the command-line typed by the
-                            # user (run by exec in pdb itself).
-                            self.shell.InteractiveTB(etype, value, tb, tb_offset=3)
-                    else:
-                        if runner is None:
-                            runner = self.default_runner
-                        if runner is None:
-                            runner = self.shell.safe_execfile
-                        if 't' in opts:
-                            # timed execution
-                            try:
-                                nruns = int(opts['N'][0])
-                                if nruns < 1:
-                                    error('Number of runs must be >=1')
-                                    return
-                            except (KeyError):
-                                nruns = 1
-                            twall0 = time.time()
-                            if nruns == 1:
-                                t0 = clock2()
-                                runner(filename, prog_ns, prog_ns,
-                                       exit_ignore=exit_ignore)
-                                t1 = clock2()
-                                t_usr = t1[0] - t0[0]
-                                t_sys = t1[1] - t0[1]
-                                print "\nIPython CPU timings (estimated):"
-                                print "  User   : %10.2f s." % t_usr
-                                print "  System : %10.2f s." % t_sys
-                            else:
-                                runs = range(nruns)
-                                t0 = clock2()
-                                for nr in runs:
-                                    runner(filename, prog_ns, prog_ns,
-                                           exit_ignore=exit_ignore)
-                                t1 = clock2()
-                                t_usr = t1[0] - t0[0]
-                                t_sys = t1[1] - t0[1]
-                                print "\nIPython CPU timings (estimated):"
-                                print "Total runs performed:", nruns
-                                print "  Times  : %10.2f    %10.2f" % ('Total', 'Per run')
-                                print "  User   : %10.2f s, %10.2f s." % (t_usr, t_usr / nruns)
-                                print "  System : %10.2f s, %10.2f s." % (t_sys, t_sys / nruns)
-                            twall1 = time.time()
-                            print "Wall time: %10.2f s." % (twall1 - twall0)
+        varnames = self.magic_who_ls(parameter_s)
+        if not varnames:
+            if parameter_s:
+                print 'No variables match your requested type.'
+            else:
+                print 'Interactive namespace is empty.'
+            return
 
-                        else:
-                            # regular execution
-                            runner(filename, prog_ns, prog_ns, exit_ignore=exit_ignore)
+        # if we have variables, move on...
 
-                if 'i' in opts:
-                    self.shell.user_ns['__name__'] = __name__save
-                else:
-                    # The shell MUST hold a reference to prog_ns so after %run
-                    # exits, the python deletion mechanism doesn't zero it out
-                    # (leaving dangling references).
-                    self.shell.cache_main_mod(prog_ns, filename)
-                    # update IPython interactive namespace
+        # for these types, show len() instead of data:
+        seq_types = ['dict', 'list', 'tuple']
 
-                    # Some forms of read errors on the file may mean the
-                    # __name__ key was never set; using pop we don't have to
-                    # worry about a possible KeyError.
-                    prog_ns.pop('__name__', None)
+        # for numpy arrays, display summary info
+        ndarray_type = None
+        if 'numpy' in sys.modules:
+            try:
+                from numpy import ndarray
+            except ImportError:
+                pass
+            else:
+                ndarray_type = ndarray.__name__
 
-                    self.shell.user_ns.update(prog_ns)
-        finally:
-            # It's a bit of a mystery why, but __builtins__ can change from
-            # being a module to becoming a dict missing some key data after
-            # %run.  As best I can see, this is NOT something IPython is doing
-            # at all, and similar problems have been reported before:
-            # http://coding.derkeiler.com/Archive/Python/comp.lang.python/2004-10/0188.html
-            # Since this seems to be done by the interpreter itself, the best
-            # we can do is to at least restore __builtins__ for the user on
-            # exit.
-            self.shell.user_ns['__builtins__'] = builtin_mod
+        # Find all variable names and types so we can figure out column sizes
+        def get_vars(i):
+            return self.shell.user_ns[i]
 
-            # Ensure key global structures are restored
-            sys.argv = save_argv
-            if restore_main:
-                sys.modules['__main__'] = restore_main
+        # some types are well known and can be shorter
+        abbrevs = {'IPython.core.macro.Macro' : 'Macro'}
+        def type_name(v):
+            tn = type(v).__name__
+            return abbrevs.get(tn,tn)
+
+        varlist = map(get_vars,varnames)
+
+        typelist = []
+        for vv in varlist:
+            tt = type_name(vv)
+
+            if tt=='instance':
+                typelist.append( abbrevs.get(str(vv.__class__),
+                                             str(vv.__class__)))
             else:
-                # Remove from sys.modules the reference to main_mod we'd
-                # added.  Otherwise it will trap references to objects
-                # contained therein.
-                del sys.modules[main_mod_name]
+                typelist.append(tt)
 
-        return stats
+        # column labels and # of spaces as separator
+        varlabel = 'Variable'
+        typelabel = 'Type'
+        datalabel = 'Data/Info'
+        colsep = 3
+        # variable format strings
+        vformat    = "{0:<{varwidth}}{1:<{typewidth}}"
+        aformat    = "%s: %s elems, type `%s`, %s bytes"
+        # find the size of the columns to format the output nicely
+        varwidth = max(max(map(len,varnames)), len(varlabel)) + colsep
+        typewidth = max(max(map(len,typelist)), len(typelabel)) + colsep
+        # table header
+        print varlabel.ljust(varwidth) + typelabel.ljust(typewidth) + \
+              ' '+datalabel+'\n' + '-'*(varwidth+typewidth+len(datalabel)+1)
+        # and the table itself
+        kb = 1024
+        Mb = 1048576  # kb**2
+        for vname,var,vtype in zip(varnames,varlist,typelist):
+            print vformat.format(vname, vtype, varwidth=varwidth, typewidth=typewidth),
+            if vtype in seq_types:
+                print "n="+str(len(var))
+            elif vtype == ndarray_type:
+                vshape = str(var.shape).replace(',','').replace(' ','x')[1:-1]
+                if vtype==ndarray_type:
+                    # numpy
+                    vsize  = var.size
+                    vbytes = vsize*var.itemsize
+                    vdtype = var.dtype
 
-    @skip_doctest
-    def magic_timeit(self, parameter_s =''):
-        """Time execution of a Python statement or expression
+                if vbytes < 100000:
+                    print aformat % (vshape,vsize,vdtype,vbytes)
+                else:
+                    print aformat % (vshape,vsize,vdtype,vbytes),
+                    if vbytes < Mb:
+                        print '(%s kb)' % (vbytes/kb,)
+                    else:
+                        print '(%s Mb)' % (vbytes/Mb,)
+            else:
+                try:
+                    vstr = str(var)
+                except UnicodeEncodeError:
+                    vstr = unicode(var).encode(DEFAULT_ENCODING,
+                                               'backslashreplace')
+                except:
+                    vstr = "<object with id %d (str() failed)>" % id(var)
+                vstr = vstr.replace('\n','\\n')
+                if len(vstr) < 50:
+                    print vstr
+                else:
+                    print vstr[:25] + "<...>" + vstr[-25:]
 
-        Usage:\\
-          %timeit [-n<N> -r<R> [-t|-c]] statement
+    def magic_reset(self, parameter_s=''):
+        """Resets the namespace by removing all names defined by the user, if
+        called without arguments, or by removing some types of objects, such
+        as everything currently in IPython's In[] and Out[] containers (see
+        the parameters for details).
 
-        Time execution of a Python statement or expression using the timeit
-        module.
+        Parameters
+        ----------
+        -f : force reset without asking for confirmation.
 
-        Options:
-        -n<N>: execute the given statement <N> times in a loop. If this value
-        is not given, a fitting value is chosen.
+        -s : 'Soft' reset: Only clears your namespace, leaving history intact.
+            References to objects may be kept. By default (without this option),
+            we do a 'hard' reset, giving you a new session and removing all
+            references to objects from the current session.
+
+        in : reset input history
+
+        out : reset output history
+
+        dhist : reset directory history
+
+        array : reset only variables that are NumPy arrays
+
+        See Also
+        --------
+        magic_reset_selective : invoked as ``%reset_selective``
+
+        Examples
+        --------
+        ::
+
+          In [6]: a = 1
+
+          In [7]: a
+          Out[7]: 1
+
+          In [8]: 'a' in _ip.user_ns
+          Out[8]: True
+
+          In [9]: %reset -f
+
+          In [1]: 'a' in _ip.user_ns
+          Out[1]: False
+
+          In [2]: %reset -f in
+          Flushing input history
+
+          In [3]: %reset -f dhist in
+          Flushing directory history
+          Flushing input history
+
+        Notes
+        -----
+        Calling this magic from clients that do not implement standard input,
+        such as the ipython notebook interface, will reset the namespace
+        without confirmation.
+        """
+        opts, args = self.parse_options(parameter_s,'sf', mode='list')
+        if 'f' in opts:
+            ans = True
+        else:
+            try:
+                ans = self.shell.ask_yes_no(
+                "Once deleted, variables cannot be recovered. Proceed (y/[n])? ", default='n')
+            except StdinNotImplementedError:
+                ans = True
+        if not ans:
+            print 'Nothing done.'
+            return
+
+        if 's' in opts:                     # Soft reset
+            user_ns = self.shell.user_ns
+            for i in self.magic_who_ls():
+                del(user_ns[i])
+        elif len(args) == 0:                # Hard reset
+            self.shell.reset(new_session = False)
+
+        # reset in/out/dhist/array: previously extensinions/clearcmd.py
+        ip = self.shell
+        user_ns = self.shell.user_ns  # local lookup, heavily used
+
+        for target in args:
+            target = target.lower() # make matches case insensitive
+            if target == 'out':
+                print "Flushing output cache (%d entries)" % len(user_ns['_oh'])
+                self.shell.displayhook.flush()
+
+            elif target == 'in':
+                print "Flushing input history"
+                pc = self.shell.displayhook.prompt_count + 1
+                for n in range(1, pc):
+                    key = '_i'+repr(n)
+                    user_ns.pop(key,None)
+                user_ns.update(dict(_i=u'',_ii=u'',_iii=u''))
+                hm = ip.history_manager
+                # don't delete these, as %save and %macro depending on the length
+                # of these lists to be preserved
+                hm.input_hist_parsed[:] = [''] * pc
+                hm.input_hist_raw[:] = [''] * pc
+                # hm has internal machinery for _i,_ii,_iii, clear it out
+                hm._i = hm._ii = hm._iii = hm._i00 =  u''
+
+            elif target == 'array':
+                # Support cleaning up numpy arrays
+                try:
+                    from numpy import ndarray
+                    # This must be done with items and not iteritems because we're
+                    # going to modify the dict in-place.
+                    for x,val in user_ns.items():
+                        if isinstance(val,ndarray):
+                            del user_ns[x]
+                except ImportError:
+                    print "reset array only works if Numpy is available."
+
+            elif target == 'dhist':
+                print "Flushing directory history"
+                del user_ns['_dh'][:]
+
+            else:
+                print "Don't know how to reset ",
+                print target + ", please run `%reset?` for details"
+
+        gc.collect()
+
+    def magic_reset_selective(self, parameter_s=''):
+        """Resets the namespace by removing names defined by the user.
+
+        Input/Output history are left around in case you need them.
+
+        %reset_selective [-f] regex
+
+        No action is taken if regex is not included
+
+        Options
+          -f : force reset without asking for confirmation.
+
+        See Also
+        --------
+        magic_reset : invoked as ``%reset``
+
+        Examples
+        --------
+
+        We first fully reset the namespace so your output looks identical to
+        this example for pedagogical reasons; in practice you do not need a
+        full reset::
+
+          In [1]: %reset -f
+
+        Now, with a clean namespace we can make a few variables and use
+        ``%reset_selective`` to only delete names that match our regexp::
+
+          In [2]: a=1; b=2; c=3; b1m=4; b2m=5; b3m=6; b4m=7; b2s=8
+
+          In [3]: who_ls
+          Out[3]: ['a', 'b', 'b1m', 'b2m', 'b2s', 'b3m', 'b4m', 'c']
+
+          In [4]: %reset_selective -f b[2-3]m
+
+          In [5]: who_ls
+          Out[5]: ['a', 'b', 'b1m', 'b2s', 'b4m', 'c']
+
+          In [6]: %reset_selective -f d
+
+          In [7]: who_ls
+          Out[7]: ['a', 'b', 'b1m', 'b2s', 'b4m', 'c']
+
+          In [8]: %reset_selective -f c
+
+          In [9]: who_ls
+          Out[9]: ['a', 'b', 'b1m', 'b2s', 'b4m']
+
+          In [10]: %reset_selective -f b
+
+          In [11]: who_ls
+          Out[11]: ['a']
+
+        Notes
+        -----
+        Calling this magic from clients that do not implement standard input,
+        such as the ipython notebook interface, will reset the namespace
+        without confirmation.
+        """
+
+        opts, regex = self.parse_options(parameter_s,'f')
+
+        if opts.has_key('f'):
+            ans = True
+        else:
+            try:
+                ans = self.shell.ask_yes_no(
+                "Once deleted, variables cannot be recovered. Proceed (y/[n])? ",
+                default='n')
+            except StdinNotImplementedError:
+                ans = True
+        if not ans:
+            print 'Nothing done.'
+            return
+        user_ns = self.shell.user_ns
+        if not regex:
+            print 'No regex pattern specified. Nothing done.'
+            return
+        else:
+            try:
+                m = re.compile(regex)
+            except TypeError:
+                raise TypeError('regex must be a string or compiled pattern')
+            for i in self.magic_who_ls():
+                if m.search(i):
+                    del(user_ns[i])
+
+    def magic_xdel(self, parameter_s=''):
+        """Delete a variable, trying to clear it from anywhere that
+        IPython's machinery has references to it. By default, this uses
+        the identity of the named object in the user namespace to remove
+        references held under other names. The object is also removed
+        from the output history.
+
+        Options
+          -n : Delete the specified name from all namespaces, without
+          checking their identity.
+        """
+        opts, varname = self.parse_options(parameter_s,'n')
+        try:
+            self.shell.del_var(varname, ('n' in opts))
+        except (NameError, ValueError) as e:
+            print type(e).__name__ +": "+ str(e)
+
+
+class ExecutionMagics(MagicFunctions):
+    """Magics related to code execution, debugging, profiling, etc.
+
+    """
+
+    def __init__(self, shell):
+        super(ProfileMagics, self).__init__(shell)
+        if profile is None:
+            self.magic_prun = self.profile_missing_notice
+        # Default execution function used to actually run user code.
+        self.default_runner = None
+
+    def profile_missing_notice(self, *args, **kwargs):
+        error("""\
+The profile module could not be found. It has been removed from the standard
+python packages because of its non-free license. To use profiling, install the
+python-profiler package from non-free.""")
+
+    @skip_doctest
+    def magic_prun(self, parameter_s ='',user_mode=1,
+                   opts=None,arg_lst=None,prog_ns=None):
+
+        """Run a statement through the python code profiler.
+
+        Usage:
+          %prun [options] statement
+
+        The given statement (which doesn't require quote marks) is run via the
+        python profiler in a manner similar to the profile.run() function.
+        Namespaces are internally managed to work correctly; profile.run
+        cannot be used in IPython because it makes certain assumptions about
+        namespaces which do not hold under IPython.
+
+        Options:
+
+        -l <limit>: you can place restrictions on what or how much of the
+        profile gets printed. The limit value can be:
+
+          * A string: only information for function names containing this string
+          is printed.
+
+          * An integer: only these many lines are printed.
+
+          * A float (between 0 and 1): this fraction of the report is printed
+          (for example, use a limit of 0.4 to see the topmost 40% only).
+
+        You can combine several limits with repeated use of the option. For
+        example, '-l __init__ -l 5' will print only the topmost 5 lines of
+        information about class constructors.
+
+        -r: return the pstats.Stats object generated by the profiling. This
+        object has all the information about the profile in it, and you can
+        later use it for further analysis or in other functions.
+
+       -s <key>: sort profile by given key. You can provide more than one key
+        by using the option several times: '-s key1 -s key2 -s key3...'. The
+        default sorting key is 'time'.
+
+        The following is copied verbatim from the profile documentation
+        referenced below:
+
+        When more than one key is provided, additional keys are used as
+        secondary criteria when the there is equality in all keys selected
+        before them.
+
+        Abbreviations can be used for any key names, as long as the
+        abbreviation is unambiguous.  The following are the keys currently
+        defined:
+
+                Valid Arg       Meaning
+                  "calls"      call count
+                  "cumulative" cumulative time
+                  "file"       file name
+                  "module"     file name
+                  "pcalls"     primitive call count
+                  "line"       line number
+                  "name"       function name
+                  "nfl"        name/file/line
+                  "stdname"    standard name
+                  "time"       internal time
+
+        Note that all sorts on statistics are in descending order (placing
+        most time consuming items first), where as name, file, and line number
+        searches are in ascending order (i.e., alphabetical). The subtle
+        distinction between "nfl" and "stdname" is that the standard name is a
+        sort of the name as printed, which means that the embedded line
+        numbers get compared in an odd way.  For example, lines 3, 20, and 40
+        would (if the file names were the same) appear in the string order
+        "20" "3" and "40".  In contrast, "nfl" does a numeric compare of the
+        line numbers.  In fact, sort_stats("nfl") is the same as
+        sort_stats("name", "file", "line").
+
+        -T <filename>: save profile results as shown on screen to a text
+        file. The profile is still shown on screen.
+
+        -D <filename>: save (via dump_stats) profile statistics to given
+        filename. This data is in a format understood by the pstats module, and
+        is generated by a call to the dump_stats() method of profile
+        objects. The profile is still shown on screen.
+
+        -q: suppress output to the pager.  Best used with -T and/or -D above.
+
+        If you want to run complete programs under the profiler's control, use
+        '%run -p [prof_opts] filename.py [args to program]' where prof_opts
+        contains profiler specific options as described here.
+
+        You can read the complete documentation for the profile module with::
+
+          In [1]: import profile; profile.help()
+        """
+
+        opts_def = Struct(D=[''],l=[],s=['time'],T=[''])
+
+        if user_mode:  # regular user call
+            opts,arg_str = self.parse_options(parameter_s,'D:l:rs:T:q',
+                                              list_all=1, posix=False)
+            namespace = self.shell.user_ns
+        else:  # called to run a program by %run -p
+            try:
+                filename = get_py_filename(arg_lst[0])
+            except IOError as e:
+                try:
+                    msg = str(e)
+                except UnicodeError:
+                    msg = e.message
+                error(msg)
+                return
+
+            arg_str = 'execfile(filename,prog_ns)'
+            namespace = {
+                'execfile': self.shell.safe_execfile,
+                'prog_ns': prog_ns,
+                'filename': filename
+                }
+
+        opts.merge(opts_def)
+
+        prof = profile.Profile()
+        try:
+            prof = prof.runctx(arg_str,namespace,namespace)
+            sys_exit = ''
+        except SystemExit:
+            sys_exit = """*** SystemExit exception caught in code being profiled."""
+
+        stats = pstats.Stats(prof).strip_dirs().sort_stats(*opts.s)
+
+        lims = opts.l
+        if lims:
+            lims = []  # rebuild lims with ints/floats/strings
+            for lim in opts.l:
+                try:
+                    lims.append(int(lim))
+                except ValueError:
+                    try:
+                        lims.append(float(lim))
+                    except ValueError:
+                        lims.append(lim)
+
+        # Trap output.
+        stdout_trap = StringIO()
+
+        if hasattr(stats,'stream'):
+            # In newer versions of python, the stats object has a 'stream'
+            # attribute to write into.
+            stats.stream = stdout_trap
+            stats.print_stats(*lims)
+        else:
+            # For older versions, we manually redirect stdout during printing
+            sys_stdout = sys.stdout
+            try:
+                sys.stdout = stdout_trap
+                stats.print_stats(*lims)
+            finally:
+                sys.stdout = sys_stdout
 
-        -r<R>: repeat the loop iteration <R> times and take the best result.
-        Default: 3
+        output = stdout_trap.getvalue()
+        output = output.rstrip()
 
-        -t: use time.time to measure the time, which is the default on Unix.
-        This function measures wall time.
+        if 'q' not in opts:
+            page.page(output)
+        print sys_exit,
 
-        -c: use time.clock to measure the time, which is the default on
-        Windows and measures wall time. On Unix, resource.getrusage is used
-        instead and returns the CPU user time.
+        dump_file = opts.D[0]
+        text_file = opts.T[0]
+        if dump_file:
+            dump_file = unquote_filename(dump_file)
+            prof.dump_stats(dump_file)
+            print '\n*** Profile stats marshalled to file',\
+                  `dump_file`+'.',sys_exit
+        if text_file:
+            text_file = unquote_filename(text_file)
+            pfile = open(text_file,'w')
+            pfile.write(output)
+            pfile.close()
+            print '\n*** Profile printout saved to text file',\
+                  `text_file`+'.',sys_exit
 
-        -p<P>: use a precision of <P> digits to display the timing result.
-        Default: 3
+        if opts.has_key('r'):
+            return stats
+        else:
+            return None
 
+    def magic_pdb(self, parameter_s=''):
+        """Control the automatic calling of the pdb interactive debugger.
 
-        Examples
-        --------
-        ::
+        Call as '%pdb on', '%pdb 1', '%pdb off' or '%pdb 0'. If called without
+        argument it works as a toggle.
 
-          In [1]: %timeit pass
-          10000000 loops, best of 3: 53.3 ns per loop
+        When an exception is triggered, IPython can optionally call the
+        interactive pdb debugger after the traceback printout. %pdb toggles
+        this feature on and off.
 
-          In [2]: u = None
+        The initial state of this feature is set in your configuration
+        file (the option is ``InteractiveShell.pdb``).
 
-          In [3]: %timeit u is None
-          10000000 loops, best of 3: 184 ns per loop
+        If you want to just activate the debugger AFTER an exception has fired,
+        without having to type '%pdb on' and rerunning your code, you can use
+        the %debug magic."""
 
-          In [4]: %timeit -r 4 u == None
-          1000000 loops, best of 4: 242 ns per loop
+        par = parameter_s.strip().lower()
 
-          In [5]: import time
+        if par:
+            try:
+                new_pdb = {'off':0,'0':0,'on':1,'1':1}[par]
+            except KeyError:
+                print ('Incorrect argument. Use on/1, off/0, '
+                       'or nothing for a toggle.')
+                return
+        else:
+            # toggle
+            new_pdb = not self.shell.call_pdb
 
-          In [6]: %timeit -n1 time.sleep(2)
-          1 loops, best of 3: 2 s per loop
+        # set on the shell
+        self.shell.call_pdb = new_pdb
+        print 'Automatic pdb calling has been turned',on_off(new_pdb)
 
+    def magic_debug(self, parameter_s=''):
+        """Activate the interactive debugger in post-mortem mode.
 
-        The times reported by %timeit will be slightly higher than those
-        reported by the timeit.py script when variables are accessed. This is
-        due to the fact that %timeit executes the statement in the namespace
-        of the shell, compared with timeit.py, which uses a single setup
-        statement to import function or create variables. Generally, the bias
-        does not matter as long as results from timeit.py are not mixed with
-        those from %timeit."""
+        If an exception has just occurred, this lets you inspect its stack
+        frames interactively.  Note that this will always work only on the last
+        traceback that occurred, so you must call this quickly after an
+        exception that you wish to inspect has fired, because if another one
+        occurs, it clobbers the previous one.
 
-        import timeit
-        import math
+        If you want IPython to automatically do this on every exception, see
+        the %pdb magic for more details.
+        """
+        self.shell.debugger(force=True)
 
-        # XXX: Unfortunately the unicode 'micro' symbol can cause problems in
-        # certain terminals.  Until we figure out a robust way of
-        # auto-detecting if the terminal can deal with it, use plain 'us' for
-        # microseconds.  I am really NOT happy about disabling the proper
-        # 'micro' prefix, but crashing is worse... If anyone knows what the
-        # right solution for this is, I'm all ears...
-        #
-        # Note: using
-        #
-        # s = u'\xb5'
-        # s.encode(sys.getdefaultencoding())
-        #
-        # is not sufficient, as I've seen terminals where that fails but
-        # print s
-        #
-        # succeeds
-        #
-        # See bug: https://bugs.launchpad.net/ipython/+bug/348466
+    def magic_tb(self, s):
+        """Print the last traceback with the currently active exception mode.
 
-        #units = [u"s", u"ms",u'\xb5',"ns"]
-        units = [u"s", u"ms",u'us',"ns"]
+        See %xmode for changing exception reporting modes."""
+        self.shell.showtraceback()
 
-        scaling = [1, 1e3, 1e6, 1e9]
+    @skip_doctest
+    def magic_run(self, parameter_s ='', runner=None,
+                  file_finder=get_py_filename):
+        """Run the named file inside IPython as a program.
 
-        opts, stmt = self.parse_options(parameter_s,'n:r:tcp:',
-                                        posix=False, strict=False)
-        if stmt == "":
-            return
-        timefunc = timeit.default_timer
-        number = int(getattr(opts, "n", 0))
-        repeat = int(getattr(opts, "r", timeit.default_repeat))
-        precision = int(getattr(opts, "p", 3))
-        if hasattr(opts, "t"):
-            timefunc = time.time
-        if hasattr(opts, "c"):
-            timefunc = clock
+        Usage:\\
+          %run [-n -i -t [-N<N>] -d [-b<N>] -p [profile options]] file [args]
 
-        timer = timeit.Timer(timer=timefunc)
-        # this code has tight coupling to the inner workings of timeit.Timer,
-        # but is there a better way to achieve that the code stmt has access
-        # to the shell namespace?
+        Parameters after the filename are passed as command-line arguments to
+        the program (put in sys.argv). Then, control returns to IPython's
+        prompt.
 
-        src = timeit.template % {'stmt': timeit.reindent(stmt, 8),
-                                 'setup': "pass"}
-        # Track compilation time so it can be reported if too long
-        # Minimum time above which compilation time will be reported
-        tc_min = 0.1
+        This is similar to running at a system prompt:\\
+          $ python file args\\
+        but with the advantage of giving you IPython's tracebacks, and of
+        loading all variables into your interactive namespace for further use
+        (unless -p is used, see below).
 
-        t0 = clock()
-        code = compile(src, "<magic-timeit>", "exec")
-        tc = clock()-t0
+        The file is executed in a namespace initially consisting only of
+        __name__=='__main__' and sys.argv constructed as indicated. It thus
+        sees its environment as if it were being run as a stand-alone program
+        (except for sharing global objects such as previously imported
+        modules). But after execution, the IPython interactive namespace gets
+        updated with all variables defined in the program (except for __name__
+        and sys.argv). This allows for very convenient loading of code for
+        interactive work, while giving each program a 'clean sheet' to run in.
 
-        ns = {}
-        exec code in self.shell.user_ns, ns
-        timer.inner = ns["inner"]
+        Options:
 
-        if number == 0:
-            # determine number so that 0.2 <= total time < 2.0
-            number = 1
-            for i in range(1, 10):
-                if timer.timeit(number) >= 0.2:
-                    break
-                number *= 10
+        -n: __name__ is NOT set to '__main__', but to the running file's name
+        without extension (as python does under import).  This allows running
+        scripts and reloading the definitions in them without calling code
+        protected by an ' if __name__ == "__main__" ' clause.
 
-        best = min(timer.repeat(repeat, number)) / number
+        -i: run the file in IPython's namespace instead of an empty one. This
+        is useful if you are experimenting with code written in a text editor
+        which depends on variables defined interactively.
 
-        if best > 0.0 and best < 1000.0:
-            order = min(-int(math.floor(math.log10(best)) // 3), 3)
-        elif best >= 1000.0:
-            order = 0
-        else:
-            order = 3
-        print u"%d loops, best of %d: %.*g %s per loop" % (number, repeat,
-                                                          precision,
-                                                          best * scaling[order],
-                                                          units[order])
-        if tc > tc_min:
-            print "Compiler time: %.2f s" % tc
+        -e: ignore sys.exit() calls or SystemExit exceptions in the script
+        being run.  This is particularly useful if IPython is being used to
+        run unittests, which always exit with a sys.exit() call.  In such
+        cases you are interested in the output of the test results, not in
+        seeing a traceback of the unittest module.
 
-    @skip_doctest
-    @needs_local_scope
-    def magic_time(self,parameter_s, user_locals):
-        """Time execution of a Python statement or expression.
+        -t: print timing information at the end of the run.  IPython will give
+        you an estimated CPU time consumption for your script, which under
+        Unix uses the resource module to avoid the wraparound problems of
+        time.clock().  Under Unix, an estimate of time spent on system tasks
+        is also given (for Windows platforms this is reported as 0.0).
 
-        The CPU and wall clock times are printed, and the value of the
-        expression (if any) is returned.  Note that under Win32, system time
-        is always reported as 0, since it can not be measured.
+        If -t is given, an additional -N<N> option can be given, where <N>
+        must be an integer indicating how many times you want the script to
+        run.  The final timing report will include total and per run results.
 
-        This function provides very basic timing functionality.  In Python
-        2.3, the timeit module offers more control and sophistication, so this
-        could be rewritten to use it (patches welcome).
+        For example (testing the script uniq_stable.py)::
 
-        Examples
-        --------
-        ::
+            In [1]: run -t uniq_stable
 
-          In [1]: time 2**128
-          CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
-          Wall time: 0.00
-          Out[1]: 340282366920938463463374607431768211456L
+            IPython CPU timings (estimated):\\
+              User  :    0.19597 s.\\
+              System:        0.0 s.\\
 
-          In [2]: n = 1000000
+            In [2]: run -t -N5 uniq_stable
 
-          In [3]: time sum(range(n))
-          CPU times: user 1.20 s, sys: 0.05 s, total: 1.25 s
-          Wall time: 1.37
-          Out[3]: 499999500000L
+            IPython CPU timings (estimated):\\
+            Total runs performed: 5\\
+              Times :      Total       Per run\\
+              User  :   0.910862 s,  0.1821724 s.\\
+              System:        0.0 s,        0.0 s.
 
-          In [4]: time print 'hello world'
-          hello world
-          CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
-          Wall time: 0.00
+        -d: run your program under the control of pdb, the Python debugger.
+        This allows you to execute your program step by step, watch variables,
+        etc.  Internally, what IPython does is similar to calling:
 
-          Note that the time needed by Python to compile the given expression
-          will be reported if it is more than 0.1s.  In this example, the
-          actual exponentiation is done by Python at compilation time, so while
-          the expression can take a noticeable amount of time to compute, that
-          time is purely due to the compilation:
+          pdb.run('execfile("YOURFILENAME")')
 
-          In [5]: time 3**9999;
-          CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
-          Wall time: 0.00 s
+        with a breakpoint set on line 1 of your file.  You can change the line
+        number for this automatic breakpoint to be <N> by using the -bN option
+        (where N must be an integer).  For example::
 
-          In [6]: time 3**999999;
-          CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
-          Wall time: 0.00 s
-          Compiler : 0.78 s
-          """
+          %run -d -b40 myscript
 
-        # fail immediately if the given expression can't be compiled
+        will set the first breakpoint at line 40 in myscript.py.  Note that
+        the first breakpoint must be set on a line which actually does
+        something (not a comment or docstring) for it to stop execution.
 
-        expr = self.shell.prefilter(parameter_s,False)
+        When the pdb debugger starts, you will see a (Pdb) prompt.  You must
+        first enter 'c' (without quotes) to start execution up to the first
+        breakpoint.
 
-        # Minimum time above which compilation time will be reported
-        tc_min = 0.1
+        Entering 'help' gives information about the use of the debugger.  You
+        can easily see pdb's full documentation with "import pdb;pdb.help()"
+        at a prompt.
 
-        try:
-            mode = 'eval'
-            t0 = clock()
-            code = compile(expr,'<timed eval>',mode)
-            tc = clock()-t0
-        except SyntaxError:
-            mode = 'exec'
-            t0 = clock()
-            code = compile(expr,'<timed exec>',mode)
-            tc = clock()-t0
-        # skew measurement as little as possible
-        glob = self.shell.user_ns
-        wtime = time.time
-        # time execution
-        wall_st = wtime()
-        if mode=='eval':
-            st = clock2()
-            out = eval(code, glob, user_locals)
-            end = clock2()
-        else:
-            st = clock2()
-            exec code in glob, user_locals
-            end = clock2()
-            out = None
-        wall_end = wtime()
-        # Compute actual times and report
-        wall_time = wall_end-wall_st
-        cpu_user = end[0]-st[0]
-        cpu_sys = end[1]-st[1]
-        cpu_tot = cpu_user+cpu_sys
-        print "CPU times: user %.2f s, sys: %.2f s, total: %.2f s" % \
-              (cpu_user,cpu_sys,cpu_tot)
-        print "Wall time: %.2f s" % wall_time
-        if tc > tc_min:
-            print "Compiler : %.2f s" % tc
-        return out
+        -p: run program under the control of the Python profiler module (which
+        prints a detailed report of execution times, function calls, etc).
 
-    @skip_doctest
-    def magic_macro(self,parameter_s = ''):
-        """Define a macro for future re-execution. It accepts ranges of history,
-        filenames or string objects.
+        You can pass other options after -p which affect the behavior of the
+        profiler itself. See the docs for %prun for details.
 
-        Usage:\\
-          %macro [options] name n1-n2 n3-n4 ... n5 .. n6 ...
+        In this mode, the program's variables do NOT propagate back to the
+        IPython interactive namespace (because they remain in the namespace
+        where the profiler executes them).
 
-        Options:
+        Internally this triggers a call to %prun, see its documentation for
+        details on the options available specifically for profiling.
 
-          -r: use 'raw' input.  By default, the 'processed' history is used,
-          so that magics are loaded in their transformed version to valid
-          Python.  If this option is given, the raw input as typed as the
-          command line is used instead.
+        There is one special usage for which the text above doesn't apply:
+        if the filename ends with .ipy, the file is run as ipython script,
+        just as if the commands were written on IPython prompt.
 
-        This will define a global variable called `name` which is a string
-        made of joining the slices and lines you specify (n1,n2,... numbers
-        above) from your input history into a single string. This variable
-        acts like an automatic function which re-executes those lines as if
-        you had typed them. You just type 'name' at the prompt and the code
-        executes.
+        -m: specify module name to load instead of script path. Similar to
+        the -m option for the python interpreter. Use this option last if you
+        want to combine with other %run options. Unlike the python interpreter
+        only source modules are allowed no .pyc or .pyo files.
+        For example::
 
-        The syntax for indicating input ranges is described in %history.
+            %run -m example
 
-        Note: as a 'hidden' feature, you can also use traditional python slice
-        notation, where N:M means numbers N through M-1.
+        will run the example module.
 
-        For example, if your history contains (%hist prints it)::
+        """
 
-          44: x=1
-          45: y=3
-          46: z=x+y
-          47: print x
-          48: a=5
-          49: print 'x',x,'y',y
+        # get arguments and set sys.argv for program to be run.
+        opts, arg_lst = self.parse_options(parameter_s, 'nidtN:b:pD:l:rs:T:em:',
+                                           mode='list', list_all=1)
+        if "m" in opts:
+            modulename = opts["m"][0]
+            modpath = find_mod(modulename)
+            if modpath is None:
+                warn('%r is not a valid modulename on sys.path'%modulename)
+                return
+            arg_lst = [modpath] + arg_lst
+        try:
+            filename = file_finder(arg_lst[0])
+        except IndexError:
+            warn('you must provide at least a filename.')
+            print '\n%run:\n', oinspect.getdoc(self.magic_run)
+            return
+        except IOError as e:
+            try:
+                msg = str(e)
+            except UnicodeError:
+                msg = e.message
+            error(msg)
+            return
 
-        you can create a macro with lines 44 through 47 (included) and line 49
-        called my_macro with::
+        if filename.lower().endswith('.ipy'):
+            self.shell.safe_execfile_ipy(filename)
+            return
 
-          In [55]: %macro my_macro 44-47 49
+        # Control the response to exit() calls made by the script being run
+        exit_ignore = 'e' in opts
 
-        Now, typing `my_macro` (without quotes) will re-execute all this code
-        in one pass.
+        # Make sure that the running script gets a proper sys.argv as if it
+        # were run from a system shell.
+        save_argv = sys.argv # save it for later restoring
 
-        You don't need to give the line-numbers in order, and any given line
-        number can appear multiple times. You can assemble macros with any
-        lines from your input history in any order.
+        # simulate shell expansion on arguments, at least tilde expansion
+        args = [ os.path.expanduser(a) for a in arg_lst[1:] ]
 
-        The macro is a simple object which holds its value in an attribute,
-        but IPython's display system checks for macros and executes them as
-        code instead of printing them when you type their name.
+        sys.argv = [filename] + args  # put in the proper filename
+        # protect sys.argv from potential unicode strings on Python 2:
+        if not py3compat.PY3:
+            sys.argv = [ py3compat.cast_bytes(a) for a in sys.argv ]
 
-        You can view a macro's contents by explicitly printing it with::
+        if 'i' in opts:
+            # Run in user's interactive namespace
+            prog_ns = self.shell.user_ns
+            __name__save = self.shell.user_ns['__name__']
+            prog_ns['__name__'] = '__main__'
+            main_mod = self.shell.new_main_mod(prog_ns)
+        else:
+            # Run in a fresh, empty namespace
+            if 'n' in opts:
+                name = os.path.splitext(os.path.basename(filename))[0]
+            else:
+                name = '__main__'
 
-          print macro_name
+            main_mod = self.shell.new_main_mod()
+            prog_ns = main_mod.__dict__
+            prog_ns['__name__'] = name
 
-        """
-        opts,args = self.parse_options(parameter_s,'r',mode='list')
-        if not args:   # List existing macros
-            return sorted(k for k,v in self.shell.user_ns.iteritems() if\
-                                                        isinstance(v, Macro))
-        if len(args) == 1:
-            raise UsageError(
-                "%macro insufficient args; usage '%macro name n1-n2 n3-4...")
-        name, codefrom = args[0], " ".join(args[1:])
+        # Since '%run foo' emulates 'python foo.py' at the cmd line, we must
+        # set the __file__ global in the script's namespace
+        prog_ns['__file__'] = filename
 
-        #print 'rng',ranges  # dbg
-        try:
-            lines = self.shell.find_user_code(codefrom, 'r' in opts)
-        except (ValueError, TypeError) as e:
-            print e.args[0]
-            return
-        macro = Macro(lines)
-        self.shell.define_macro(name, macro)
-        print 'Macro `%s` created. To execute, type its name (without quotes).' % name
-        print '=== Macro contents: ==='
-        print macro,
+        # pickle fix.  See interactiveshell for an explanation.  But we need to make sure
+        # that, if we overwrite __main__, we replace it at the end
+        main_mod_name = prog_ns['__name__']
 
-    def magic_save(self,parameter_s = ''):
-        """Save a set of lines or a macro to a given filename.
+        if main_mod_name == '__main__':
+            restore_main = sys.modules['__main__']
+        else:
+            restore_main = False
 
-        Usage:\\
-          %save [options] filename n1-n2 n3-n4 ... n5 .. n6 ...
+        # This needs to be undone at the end to prevent holding references to
+        # every single object ever created.
+        sys.modules[main_mod_name] = main_mod
+
+        try:
+            stats = None
+            with self.shell.readline_no_record:
+                if 'p' in opts:
+                    stats = self.magic_prun('', 0, opts, arg_lst, prog_ns)
+                else:
+                    if 'd' in opts:
+                        deb = debugger.Pdb(self.shell.colors)
+                        # reset Breakpoint state, which is moronically kept
+                        # in a class
+                        bdb.Breakpoint.next = 1
+                        bdb.Breakpoint.bplist = {}
+                        bdb.Breakpoint.bpbynumber = [None]
+                        # Set an initial breakpoint to stop execution
+                        maxtries = 10
+                        bp = int(opts.get('b', [1])[0])
+                        checkline = deb.checkline(filename, bp)
+                        if not checkline:
+                            for bp in range(bp + 1, bp + maxtries + 1):
+                                if deb.checkline(filename, bp):
+                                    break
+                            else:
+                                msg = ("\nI failed to find a valid line to set "
+                                       "a breakpoint\n"
+                                       "after trying up to line: %s.\n"
+                                       "Please set a valid breakpoint manually "
+                                       "with the -b option." % bp)
+                                error(msg)
+                                return
+                        # if we find a good linenumber, set the breakpoint
+                        deb.do_break('%s:%s' % (filename, bp))
+                        # Start file run
+                        print "NOTE: Enter 'c' at the",
+                        print "%s prompt to start your script." % deb.prompt
+                        ns = {'execfile': py3compat.execfile, 'prog_ns': prog_ns}
+                        try:
+                            deb.run('execfile("%s", prog_ns)' % filename, ns)
 
-        Options:
+                        except:
+                            etype, value, tb = sys.exc_info()
+                            # Skip three frames in the traceback: the %run one,
+                            # one inside bdb.py, and the command-line typed by the
+                            # user (run by exec in pdb itself).
+                            self.shell.InteractiveTB(etype, value, tb, tb_offset=3)
+                    else:
+                        if runner is None:
+                            runner = self.default_runner
+                        if runner is None:
+                            runner = self.shell.safe_execfile
+                        if 't' in opts:
+                            # timed execution
+                            try:
+                                nruns = int(opts['N'][0])
+                                if nruns < 1:
+                                    error('Number of runs must be >=1')
+                                    return
+                            except (KeyError):
+                                nruns = 1
+                            twall0 = time.time()
+                            if nruns == 1:
+                                t0 = clock2()
+                                runner(filename, prog_ns, prog_ns,
+                                       exit_ignore=exit_ignore)
+                                t1 = clock2()
+                                t_usr = t1[0] - t0[0]
+                                t_sys = t1[1] - t0[1]
+                                print "\nIPython CPU timings (estimated):"
+                                print "  User   : %10.2f s." % t_usr
+                                print "  System : %10.2f s." % t_sys
+                            else:
+                                runs = range(nruns)
+                                t0 = clock2()
+                                for nr in runs:
+                                    runner(filename, prog_ns, prog_ns,
+                                           exit_ignore=exit_ignore)
+                                t1 = clock2()
+                                t_usr = t1[0] - t0[0]
+                                t_sys = t1[1] - t0[1]
+                                print "\nIPython CPU timings (estimated):"
+                                print "Total runs performed:", nruns
+                                print "  Times  : %10.2f    %10.2f" % ('Total', 'Per run')
+                                print "  User   : %10.2f s, %10.2f s." % (t_usr, t_usr / nruns)
+                                print "  System : %10.2f s, %10.2f s." % (t_sys, t_sys / nruns)
+                            twall1 = time.time()
+                            print "Wall time: %10.2f s." % (twall1 - twall0)
 
-          -r: use 'raw' input.  By default, the 'processed' history is used,
-          so that magics are loaded in their transformed version to valid
-          Python.  If this option is given, the raw input as typed as the
-          command line is used instead.
+                        else:
+                            # regular execution
+                            runner(filename, prog_ns, prog_ns, exit_ignore=exit_ignore)
 
-        This function uses the same syntax as %history for input ranges,
-        then saves the lines to the filename you specify.
+                if 'i' in opts:
+                    self.shell.user_ns['__name__'] = __name__save
+                else:
+                    # The shell MUST hold a reference to prog_ns so after %run
+                    # exits, the python deletion mechanism doesn't zero it out
+                    # (leaving dangling references).
+                    self.shell.cache_main_mod(prog_ns, filename)
+                    # update IPython interactive namespace
 
-        It adds a '.py' extension to the file if you don't do so yourself, and
-        it asks for confirmation before overwriting existing files."""
+                    # Some forms of read errors on the file may mean the
+                    # __name__ key was never set; using pop we don't have to
+                    # worry about a possible KeyError.
+                    prog_ns.pop('__name__', None)
 
-        opts,args = self.parse_options(parameter_s,'r',mode='list')
-        fname, codefrom = unquote_filename(args[0]), " ".join(args[1:])
-        if not fname.endswith('.py'):
-            fname += '.py'
-        if os.path.isfile(fname):
-            overwrite = self.shell.ask_yes_no('File `%s` exists. Overwrite (y/[N])? ' % fname, default='n')
-            if not overwrite :
-                print 'Operation cancelled.'
-                return
-        try:
-            cmds = self.shell.find_user_code(codefrom, 'r' in opts)
-        except (TypeError, ValueError) as e:
-            print e.args[0]
-            return
-        with io.open(fname,'w', encoding="utf-8") as f:
-            f.write(u"# coding: utf-8\n")
-            f.write(py3compat.cast_unicode(cmds))
-        print 'The following commands were written to file `%s`:' % fname
-        print cmds
+                    self.shell.user_ns.update(prog_ns)
+        finally:
+            # It's a bit of a mystery why, but __builtins__ can change from
+            # being a module to becoming a dict missing some key data after
+            # %run.  As best I can see, this is NOT something IPython is doing
+            # at all, and similar problems have been reported before:
+            # http://coding.derkeiler.com/Archive/Python/comp.lang.python/2004-10/0188.html
+            # Since this seems to be done by the interpreter itself, the best
+            # we can do is to at least restore __builtins__ for the user on
+            # exit.
+            self.shell.user_ns['__builtins__'] = builtin_mod
 
-    def magic_pastebin(self, parameter_s = ''):
-        """Upload code to Github's Gist paste bin, returning the URL.
-        
-        Usage:\\
-          %pastebin [-d "Custom description"] 1-7
-          
-        The argument can be an input history range, a filename, or the name of a
-        string or macro.
-        
-        Options:
-        
-          -d: Pass a custom description for the gist. The default will say
-              "Pasted from IPython".
-        """
-        opts, args = self.parse_options(parameter_s, 'd:')
-        
-        try:
-            code = self.shell.find_user_code(args)
-        except (ValueError, TypeError) as e:
-            print e.args[0]
-            return
-        
-        post_data = json.dumps({
-          "description": opts.get('d', "Pasted from IPython"),
-          "public": True,
-          "files": {
-            "file1.py": {
-              "content": code
-            }
-          }
-        }).encode('utf-8')
-        
-        response = urlopen("https://api.github.com/gists", post_data)
-        response_data = json.loads(response.read().decode('utf-8'))
-        return response_data['html_url']
+            # Ensure key global structures are restored
+            sys.argv = save_argv
+            if restore_main:
+                sys.modules['__main__'] = restore_main
+            else:
+                # Remove from sys.modules the reference to main_mod we'd
+                # added.  Otherwise it will trap references to objects
+                # contained therein.
+                del sys.modules[main_mod_name]
 
-    def magic_loadpy(self, arg_s):
-        """Alias of `%load`
-        
-        `%loadpy` has gained some flexibility and droped the requirement of a `.py`
-        extension. So it has been renamed simply into %load. You can look at
-        `%load`'s docstring for more info.
-        """
-        self.magic_load(arg_s)
+        return stats
 
-    def magic_load(self, arg_s):
-        """Load code into the current frontend.
+    @skip_doctest
+    def magic_timeit(self, parameter_s =''):
+        """Time execution of a Python statement or expression
 
         Usage:\\
-          %load [options] source
+          %timeit [-n<N> -r<R> [-t|-c]] statement
 
-          where source can be a filename, URL, input history range or macro
+        Time execution of a Python statement or expression using the timeit
+        module.
 
         Options:
-        --------
-          -y : Don't ask confirmation for loading source above 200 000 characters.
+        -n<N>: execute the given statement <N> times in a loop. If this value
+        is not given, a fitting value is chosen.
 
-        This magic command can either take a local filename, a URL, an history
-        range (see %history) or a macro as argument, it will prompt for
-        confirmation before loading source with more than 200 000 characters, unless
-        -y flag is passed or if the frontend does not support raw_input::
+        -r<R>: repeat the loop iteration <R> times and take the best result.
+        Default: 3
 
-        %load myscript.py
-        %load 7-27
-        %load myMacro
-        %load http://www.example.com/myscript.py
-        """
-        opts,args = self.parse_options(arg_s,'y')
+        -t: use time.time to measure the time, which is the default on Unix.
+        This function measures wall time.
 
-        contents = self.shell.find_user_code(args)
-        l = len(contents)
+        -c: use time.clock to measure the time, which is the default on
+        Windows and measures wall time. On Unix, resource.getrusage is used
+        instead and returns the CPU user time.
 
-        # 200 000 is ~ 2500 full 80 caracter lines
-        # so in average, more than 5000 lines
-        if l > 200000 and 'y' not in opts:
-            try:
-                ans = self.shell.ask_yes_no(("The text you're trying to load seems pretty big"\
-                " (%d characters). Continue (y/[N]) ?" % l), default='n' )
-            except StdinNotImplementedError:
-                #asume yes if raw input not implemented
-                ans = True
+        -p<P>: use a precision of <P> digits to display the timing result.
+        Default: 3
 
-            if ans is False :
-                print 'Operation cancelled.'
-                return
 
-        self.set_next_input(contents)
+        Examples
+        --------
+        ::
 
-    def _find_edit_target(self, args, opts, last_call):
-        """Utility method used by magic_edit to find what to edit."""
+          In [1]: %timeit pass
+          10000000 loops, best of 3: 53.3 ns per loop
 
-        def make_filename(arg):
-            "Make a filename from the given args"
-            arg = unquote_filename(arg)
-            try:
-                filename = get_py_filename(arg)
-            except IOError:
-                # If it ends with .py but doesn't already exist, assume we want
-                # a new file.
-                if arg.endswith('.py'):
-                    filename = arg
-                else:
-                    filename = None
-            return filename
+          In [2]: u = None
 
-        # Set a few locals from the options for convenience:
-        opts_prev = 'p' in opts
-        opts_raw = 'r' in opts
+          In [3]: %timeit u is None
+          10000000 loops, best of 3: 184 ns per loop
 
-        # custom exceptions
-        class DataIsObject(Exception): pass
+          In [4]: %timeit -r 4 u == None
+          1000000 loops, best of 4: 242 ns per loop
 
-        # Default line number value
-        lineno = opts.get('n',None)
+          In [5]: import time
 
-        if opts_prev:
-            args = '_%s' % last_call[0]
-            if not self.shell.user_ns.has_key(args):
-                args = last_call[1]
+          In [6]: %timeit -n1 time.sleep(2)
+          1 loops, best of 3: 2 s per loop
 
-        # use last_call to remember the state of the previous call, but don't
-        # let it be clobbered by successive '-p' calls.
-        try:
-            last_call[0] = self.shell.displayhook.prompt_count
-            if not opts_prev:
-                last_call[1] = args
-        except:
-            pass
 
-        # by default this is done with temp files, except when the given
-        # arg is a filename
-        use_temp = True
+        The times reported by %timeit will be slightly higher than those
+        reported by the timeit.py script when variables are accessed. This is
+        due to the fact that %timeit executes the statement in the namespace
+        of the shell, compared with timeit.py, which uses a single setup
+        statement to import function or create variables. Generally, the bias
+        does not matter as long as results from timeit.py are not mixed with
+        those from %timeit."""
 
-        data = ''
+        import timeit
+        import math
 
-        # First, see if the arguments should be a filename.
-        filename = make_filename(args)
-        if filename:
-            use_temp = False
-        elif args:
-            # Mode where user specifies ranges of lines, like in %macro.
-            data = self.shell.extract_input_lines(args, opts_raw)
-            if not data:
-                try:
-                    # Load the parameter given as a variable. If not a string,
-                    # process it as an object instead (below)
+        # XXX: Unfortunately the unicode 'micro' symbol can cause problems in
+        # certain terminals.  Until we figure out a robust way of
+        # auto-detecting if the terminal can deal with it, use plain 'us' for
+        # microseconds.  I am really NOT happy about disabling the proper
+        # 'micro' prefix, but crashing is worse... If anyone knows what the
+        # right solution for this is, I'm all ears...
+        #
+        # Note: using
+        #
+        # s = u'\xb5'
+        # s.encode(sys.getdefaultencoding())
+        #
+        # is not sufficient, as I've seen terminals where that fails but
+        # print s
+        #
+        # succeeds
+        #
+        # See bug: https://bugs.launchpad.net/ipython/+bug/348466
 
-                    #print '*** args',args,'type',type(args)  # dbg
-                    data = eval(args, self.shell.user_ns)
-                    if not isinstance(data, basestring):
-                        raise DataIsObject
+        #units = [u"s", u"ms",u'\xb5',"ns"]
+        units = [u"s", u"ms",u'us',"ns"]
 
-                except (NameError,SyntaxError):
-                    # given argument is not a variable, try as a filename
-                    filename = make_filename(args)
-                    if filename is None:
-                        warn("Argument given (%s) can't be found as a variable "
-                             "or as a filename." % args)
-                        return
-                    use_temp = False
+        scaling = [1, 1e3, 1e6, 1e9]
 
-                except DataIsObject:
-                    # macros have a special edit function
-                    if isinstance(data, Macro):
-                        raise MacroToEdit(data)
+        opts, stmt = self.parse_options(parameter_s,'n:r:tcp:',
+                                        posix=False, strict=False)
+        if stmt == "":
+            return
+        timefunc = timeit.default_timer
+        number = int(getattr(opts, "n", 0))
+        repeat = int(getattr(opts, "r", timeit.default_repeat))
+        precision = int(getattr(opts, "p", 3))
+        if hasattr(opts, "t"):
+            timefunc = time.time
+        if hasattr(opts, "c"):
+            timefunc = clock
 
-                    # For objects, try to edit the file where they are defined
-                    try:
-                        filename = inspect.getabsfile(data)
-                        if 'fakemodule' in filename.lower() and inspect.isclass(data):
-                            # class created by %edit? Try to find source
-                            # by looking for method definitions instead, the
-                            # __module__ in those classes is FakeModule.
-                            attrs = [getattr(data, aname) for aname in dir(data)]
-                            for attr in attrs:
-                                if not inspect.ismethod(attr):
-                                    continue
-                                filename = inspect.getabsfile(attr)
-                                if filename and 'fakemodule' not in filename.lower():
-                                    # change the attribute to be the edit target instead
-                                    data = attr
-                                    break
+        timer = timeit.Timer(timer=timefunc)
+        # this code has tight coupling to the inner workings of timeit.Timer,
+        # but is there a better way to achieve that the code stmt has access
+        # to the shell namespace?
 
-                        datafile = 1
-                    except TypeError:
-                        filename = make_filename(args)
-                        datafile = 1
-                        warn('Could not find file where `%s` is defined.\n'
-                             'Opening a file named `%s`' % (args,filename))
-                    # Now, make sure we can actually read the source (if it was in
-                    # a temp file it's gone by now).
-                    if datafile:
-                        try:
-                            if lineno is None:
-                                lineno = inspect.getsourcelines(data)[1]
-                        except IOError:
-                            filename = make_filename(args)
-                            if filename is None:
-                                warn('The file `%s` where `%s` was defined cannot '
-                                     'be read.' % (filename,data))
-                                return
-                    use_temp = False
+        src = timeit.template % {'stmt': timeit.reindent(stmt, 8),
+                                 'setup': "pass"}
+        # Track compilation time so it can be reported if too long
+        # Minimum time above which compilation time will be reported
+        tc_min = 0.1
 
-        if use_temp:
-            filename = self.shell.mktempfile(data)
-            print 'IPython will make a temporary file named:',filename
+        t0 = clock()
+        code = compile(src, "<magic-timeit>", "exec")
+        tc = clock()-t0
 
-        return filename, lineno, use_temp
+        ns = {}
+        exec code in self.shell.user_ns, ns
+        timer.inner = ns["inner"]
 
-    def _edit_macro(self,mname,macro):
-        """open an editor with the macro data in a file"""
-        filename = self.shell.mktempfile(macro.value)
-        self.shell.hooks.editor(filename)
+        if number == 0:
+            # determine number so that 0.2 <= total time < 2.0
+            number = 1
+            for i in range(1, 10):
+                if timer.timeit(number) >= 0.2:
+                    break
+                number *= 10
 
-        # and make a new macro object, to replace the old one
-        mfile = open(filename)
-        mvalue = mfile.read()
-        mfile.close()
-        self.shell.user_ns[mname] = Macro(mvalue)
+        best = min(timer.repeat(repeat, number)) / number
 
-    def magic_ed(self,parameter_s=''):
-        """Alias to %edit."""
-        return self.magic_edit(parameter_s)
+        if best > 0.0 and best < 1000.0:
+            order = min(-int(math.floor(math.log10(best)) // 3), 3)
+        elif best >= 1000.0:
+            order = 0
+        else:
+            order = 3
+        print u"%d loops, best of %d: %.*g %s per loop" % (number, repeat,
+                                                          precision,
+                                                          best * scaling[order],
+                                                          units[order])
+        if tc > tc_min:
+            print "Compiler time: %.2f s" % tc
 
     @skip_doctest
-    def magic_edit(self,parameter_s='',last_call=['','']):
-        """Bring up an editor and execute the resulting code.
-
-        Usage:
-          %edit [options] [args]
-
-        %edit runs IPython's editor hook. The default version of this hook is
-        set to call the editor specified by your $EDITOR environment variable.
-        If this isn't found, it will default to vi under Linux/Unix and to
-        notepad under Windows. See the end of this docstring for how to change
-        the editor hook.
+    @needs_local_scope
+    def magic_time(self,parameter_s, user_locals):
+        """Time execution of a Python statement or expression.
 
-        You can also set the value of this editor via the
-        ``TerminalInteractiveShell.editor`` option in your configuration file.
-        This is useful if you wish to use a different editor from your typical
-        default with IPython (and for Windows users who typically don't set
-        environment variables).
+        The CPU and wall clock times are printed, and the value of the
+        expression (if any) is returned.  Note that under Win32, system time
+        is always reported as 0, since it can not be measured.
 
-        This command allows you to conveniently edit multi-line code right in
-        your IPython session.
+        This function provides very basic timing functionality.  In Python
+        2.3, the timeit module offers more control and sophistication, so this
+        could be rewritten to use it (patches welcome).
 
-        If called without arguments, %edit opens up an empty editor with a
-        temporary file and will execute the contents of this file when you
-        close it (don't forget to save it!).
+        Examples
+        --------
+        ::
 
+          In [1]: time 2**128
+          CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
+          Wall time: 0.00
+          Out[1]: 340282366920938463463374607431768211456L
 
-        Options:
+          In [2]: n = 1000000
 
-        -n <number>: open the editor at a specified line number.  By default,
-        the IPython editor hook uses the unix syntax 'editor +N filename', but
-        you can configure this by providing your own modified hook if your
-        favorite editor supports line-number specifications with a different
-        syntax.
+          In [3]: time sum(range(n))
+          CPU times: user 1.20 s, sys: 0.05 s, total: 1.25 s
+          Wall time: 1.37
+          Out[3]: 499999500000L
 
-        -p: this will call the editor with the same data as the previous time
-        it was used, regardless of how long ago (in your current session) it
-        was.
+          In [4]: time print 'hello world'
+          hello world
+          CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
+          Wall time: 0.00
 
-        -r: use 'raw' input.  This option only applies to input taken from the
-        user's history.  By default, the 'processed' history is used, so that
-        magics are loaded in their transformed version to valid Python.  If
-        this option is given, the raw input as typed as the command line is
-        used instead.  When you exit the editor, it will be executed by
-        IPython's own processor.
+          Note that the time needed by Python to compile the given expression
+          will be reported if it is more than 0.1s.  In this example, the
+          actual exponentiation is done by Python at compilation time, so while
+          the expression can take a noticeable amount of time to compute, that
+          time is purely due to the compilation:
 
-        -x: do not execute the edited code immediately upon exit. This is
-        mainly useful if you are editing programs which need to be called with
-        command line arguments, which you can then do using %run.
+          In [5]: time 3**9999;
+          CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
+          Wall time: 0.00 s
 
+          In [6]: time 3**999999;
+          CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
+          Wall time: 0.00 s
+          Compiler : 0.78 s
+          """
 
-        Arguments:
+        # fail immediately if the given expression can't be compiled
 
-        If arguments are given, the following possibilities exist:
+        expr = self.shell.prefilter(parameter_s,False)
 
-        - If the argument is a filename, IPython will load that into the
-          editor. It will execute its contents with execfile() when you exit,
-          loading any code in the file into your interactive namespace.
+        # Minimum time above which compilation time will be reported
+        tc_min = 0.1
 
-        - The arguments are ranges of input history,  e.g. "7 ~1/4-6".
-          The syntax is the same as in the %history magic.
+        try:
+            mode = 'eval'
+            t0 = clock()
+            code = compile(expr,'<timed eval>',mode)
+            tc = clock()-t0
+        except SyntaxError:
+            mode = 'exec'
+            t0 = clock()
+            code = compile(expr,'<timed exec>',mode)
+            tc = clock()-t0
+        # skew measurement as little as possible
+        glob = self.shell.user_ns
+        wtime = time.time
+        # time execution
+        wall_st = wtime()
+        if mode=='eval':
+            st = clock2()
+            out = eval(code, glob, user_locals)
+            end = clock2()
+        else:
+            st = clock2()
+            exec code in glob, user_locals
+            end = clock2()
+            out = None
+        wall_end = wtime()
+        # Compute actual times and report
+        wall_time = wall_end-wall_st
+        cpu_user = end[0]-st[0]
+        cpu_sys = end[1]-st[1]
+        cpu_tot = cpu_user+cpu_sys
+        print "CPU times: user %.2f s, sys: %.2f s, total: %.2f s" % \
+              (cpu_user,cpu_sys,cpu_tot)
+        print "Wall time: %.2f s" % wall_time
+        if tc > tc_min:
+            print "Compiler : %.2f s" % tc
+        return out
 
-        - If the argument is a string variable, its contents are loaded
-          into the editor. You can thus edit any string which contains
-          python code (including the result of previous edits).
+    @skip_doctest
+    def magic_macro(self,parameter_s = ''):
+        """Define a macro for future re-execution. It accepts ranges of history,
+        filenames or string objects.
 
-        - If the argument is the name of an object (other than a string),
-          IPython will try to locate the file where it was defined and open the
-          editor at the point where it is defined. You can use `%edit function`
-          to load an editor exactly at the point where 'function' is defined,
-          edit it and have the file be executed automatically.
+        Usage:\\
+          %macro [options] name n1-n2 n3-n4 ... n5 .. n6 ...
 
-        - If the object is a macro (see %macro for details), this opens up your
-          specified editor with a temporary file containing the macro's data.
-          Upon exit, the macro is reloaded with the contents of the file.
+        Options:
 
-        Note: opening at an exact line is only supported under Unix, and some
-        editors (like kedit and gedit up to Gnome 2.8) do not understand the
-        '+NUMBER' parameter necessary for this feature. Good editors like
-        (X)Emacs, vi, jed, pico and joe all do.
+          -r: use 'raw' input.  By default, the 'processed' history is used,
+          so that magics are loaded in their transformed version to valid
+          Python.  If this option is given, the raw input as typed as the
+          command line is used instead.
 
-        After executing your code, %edit will return as output the code you
-        typed in the editor (except when it was an existing file). This way
-        you can reload the code in further invocations of %edit as a variable,
-        via _<NUMBER> or Out[<NUMBER>], where <NUMBER> is the prompt number of
-        the output.
+        This will define a global variable called `name` which is a string
+        made of joining the slices and lines you specify (n1,n2,... numbers
+        above) from your input history into a single string. This variable
+        acts like an automatic function which re-executes those lines as if
+        you had typed them. You just type 'name' at the prompt and the code
+        executes.
 
-        Note that %edit is also available through the alias %ed.
+        The syntax for indicating input ranges is described in %history.
 
-        This is an example of creating a simple function inside the editor and
-        then modifying it. First, start up the editor::
+        Note: as a 'hidden' feature, you can also use traditional python slice
+        notation, where N:M means numbers N through M-1.
 
-          In [1]: ed
-          Editing... done. Executing edited code...
-          Out[1]: 'def foo():\\n    print "foo() was defined in an editing
-          session"\\n'
+        For example, if your history contains (%hist prints it)::
 
-        We can then call the function foo()::
+          44: x=1
+          45: y=3
+          46: z=x+y
+          47: print x
+          48: a=5
+          49: print 'x',x,'y',y
 
-          In [2]: foo()
-          foo() was defined in an editing session
+        you can create a macro with lines 44 through 47 (included) and line 49
+        called my_macro with::
 
-        Now we edit foo.  IPython automatically loads the editor with the
-        (temporary) file where foo() was previously defined::
+          In [55]: %macro my_macro 44-47 49
 
-          In [3]: ed foo
-          Editing... done. Executing edited code...
+        Now, typing `my_macro` (without quotes) will re-execute all this code
+        in one pass.
 
-        And if we call foo() again we get the modified version::
+        You don't need to give the line-numbers in order, and any given line
+        number can appear multiple times. You can assemble macros with any
+        lines from your input history in any order.
 
-          In [4]: foo()
-          foo() has now been changed!
+        The macro is a simple object which holds its value in an attribute,
+        but IPython's display system checks for macros and executes them as
+        code instead of printing them when you type their name.
 
-        Here is an example of how to edit a code snippet successive
-        times. First we call the editor::
+        You can view a macro's contents by explicitly printing it with::
 
-          In [5]: ed
-          Editing... done. Executing edited code...
-          hello
-          Out[5]: "print 'hello'\\n"
+          print macro_name
 
-        Now we call it again with the previous output (stored in _)::
+        """
+        opts,args = self.parse_options(parameter_s,'r',mode='list')
+        if not args:   # List existing macros
+            return sorted(k for k,v in self.shell.user_ns.iteritems() if\
+                                                        isinstance(v, Macro))
+        if len(args) == 1:
+            raise UsageError(
+                "%macro insufficient args; usage '%macro name n1-n2 n3-4...")
+        name, codefrom = args[0], " ".join(args[1:])
 
-          In [6]: ed _
-          Editing... done. Executing edited code...
-          hello world
-          Out[6]: "print 'hello world'\\n"
+        #print 'rng',ranges  # dbg
+        try:
+            lines = self.shell.find_user_code(codefrom, 'r' in opts)
+        except (ValueError, TypeError) as e:
+            print e.args[0]
+            return
+        macro = Macro(lines)
+        self.shell.define_macro(name, macro)
+        print 'Macro `%s` created. To execute, type its name (without quotes).' % name
+        print '=== Macro contents: ==='
+        print macro,
 
-        Now we call it with the output #8 (stored in _8, also as Out[8])::
 
-          In [7]: ed _8
-          Editing... done. Executing edited code...
-          hello again
-          Out[7]: "print 'hello again'\\n"
+class AutoMagics(MagicFunctions):
+    """Magics that control various autoX behaviors."""
 
+    def __init__(self, shell):
+        super(ProfileMagics, self).__init__(shell)
+        # namespace for holding state we may need
+        self._magic_state = Bunch()
 
-        Changing the default editor hook:
+    def magic_automagic(self, parameter_s = ''):
+        """Make magic functions callable without having to type the initial %.
 
-        If you wish to write your own editor hook, you can put it in a
-        configuration file which you load at startup time.  The default hook
-        is defined in the IPython.core.hooks module, and you can use that as a
-        starting example for further modifications.  That file also has
-        general instructions on how to set a new hook for use once you've
-        defined it."""
-        opts,args = self.parse_options(parameter_s,'prxn:')
+        Without argumentsl toggles on/off (when off, you must call it as
+        %automagic, of course).  With arguments it sets the value, and you can
+        use any of (case insensitive):
 
-        try:
-            filename, lineno, is_temp = self._find_edit_target(args, opts, last_call)
-        except MacroToEdit as e:
-            self._edit_macro(args, e.args[0])
-            return
+         - on,1,True: to activate
 
-        # do actual editing here
-        print 'Editing...',
-        sys.stdout.flush()
-        try:
-            # Quote filenames that may have spaces in them
-            if ' ' in filename:
-                filename = "'%s'" % filename
-            self.shell.hooks.editor(filename,lineno)
-        except TryNext:
-            warn('Could not open editor')
-            return
+         - off,0,False: to deactivate.
 
-        # XXX TODO: should this be generalized for all string vars?
-        # For now, this is special-cased to blocks created by cpaste
-        if args.strip() == 'pasted_block':
-            self.shell.user_ns['pasted_block'] = file_read(filename)
+        Note that magic functions have lowest priority, so if there's a
+        variable whose name collides with that of a magic fn, automagic won't
+        work for that function (you get the variable instead). However, if you
+        delete the variable (del var), the previously shadowed magic function
+        becomes visible to automagic again."""
 
-        if 'x' in opts:  # -x prevents actual execution
-            print
+        arg = parameter_s.lower()
+        if parameter_s in ('on','1','true'):
+            self.shell.automagic = True
+        elif parameter_s in ('off','0','false'):
+            self.shell.automagic = False
         else:
-            print 'done. Executing edited code...'
-            if 'r' in opts:    # Untranslated IPython code
-                self.shell.run_cell(file_read(filename),
-                                                    store_history=False)
-            else:
-                self.shell.safe_execfile(filename, self.shell.user_ns,
-                                         self.shell.user_ns)
-
-        if is_temp:
-            try:
-                return open(filename).read()
-            except IOError,msg:
-                if msg.filename == filename:
-                    warn('File not found. Did you forget to save?')
-                    return
-                else:
-                    self.shell.showtraceback()
-
-    def magic_xmode(self,parameter_s = ''):
-        """Switch modes for the exception handlers.
-
-        Valid modes: Plain, Context and Verbose.
+            self.shell.automagic = not self.shell.automagic
+        print '\n' + Magic.auto_status[self.shell.automagic]
 
-        If called without arguments, acts as a toggle."""
+    @skip_doctest
+    def magic_autocall(self, parameter_s = ''):
+        """Make functions callable without having to type parentheses.
 
-        def xmode_switch_err(name):
-            warn('Error changing %s exception modes.\n%s' %
-                 (name,sys.exc_info()[1]))
+        Usage:
 
-        shell = self.shell
-        new_mode = parameter_s.strip().capitalize()
-        try:
-            shell.InteractiveTB.set_mode(mode=new_mode)
-            print 'Exception reporting mode:',shell.InteractiveTB.mode
-        except:
-            xmode_switch_err('user')
+           %autocall [mode]
 
-    def magic_colors(self,parameter_s = ''):
-        """Switch color scheme for prompts, info system and exception handlers.
+        The mode can be one of: 0->Off, 1->Smart, 2->Full.  If not given, the
+        value is toggled on and off (remembering the previous state).
 
-        Currently implemented schemes: NoColor, Linux, LightBG.
+        In more detail, these values mean:
 
-        Color scheme names are not case-sensitive.
+        0 -> fully disabled
 
-        Examples
-        --------
-        To get a plain black and white terminal::
+        1 -> active, but do not apply if there are no arguments on the line.
 
-          %colors nocolor
-        """
+        In this mode, you get::
 
-        def color_switch_err(name):
-            warn('Error changing %s color schemes.\n%s' %
-                 (name,sys.exc_info()[1]))
+          In [1]: callable
+          Out[1]: <built-in function callable>
 
+          In [2]: callable 'hello'
+          ------> callable('hello')
+          Out[2]: False
 
-        new_scheme = parameter_s.strip()
-        if not new_scheme:
-            raise UsageError(
-                "%colors: you must specify a color scheme. See '%colors?'")
-            return
-        # local shortcut
-        shell = self.shell
+        2 -> Active always.  Even if no arguments are present, the callable
+        object is called::
 
-        import IPython.utils.rlineimpl as readline
+          In [2]: float
+          ------> float()
+          Out[2]: 0.0
 
-        if not shell.colors_force and \
-                not readline.have_readline and sys.platform == "win32":
-            msg = """\
-Proper color support under MS Windows requires the pyreadline library.
-You can find it at:
-http://ipython.org/pyreadline.html
-Gary's readline needs the ctypes module, from:
-http://starship.python.net/crew/theller/ctypes
-(Note that ctypes is already part of Python versions 2.5 and newer).
+        Note that even with autocall off, you can still use '/' at the start of
+        a line to treat the first argument on the command line as a function
+        and add parentheses to it::
 
-Defaulting color scheme to 'NoColor'"""
-            new_scheme = 'NoColor'
-            warn(msg)
+          In [8]: /str 43
+          ------> str(43)
+          Out[8]: '43'
 
-        # readline option is 0
-        if not shell.colors_force and not shell.has_readline:
-            new_scheme = 'NoColor'
+        # all-random (note for auto-testing)
+        """
 
-        # Set prompt colors
-        try:
-            shell.prompt_manager.color_scheme = new_scheme
-        except:
-            color_switch_err('prompt')
+        if parameter_s:
+            arg = int(parameter_s)
         else:
-            shell.colors = \
-                   shell.prompt_manager.color_scheme_table.active_scheme_name
-        # Set exception colors
-        try:
-            shell.InteractiveTB.set_colors(scheme = new_scheme)
-            shell.SyntaxTB.set_colors(scheme = new_scheme)
-        except:
-            color_switch_err('exception')
+            arg = 'toggle'
 
-        # Set info (for 'object?') colors
-        if shell.color_info:
-            try:
-                shell.inspector.set_active_scheme(new_scheme)
-            except:
-                color_switch_err('object inspector')
-        else:
-            shell.inspector.set_active_scheme('NoColor')
+        if not arg in (0,1,2,'toggle'):
+            error('Valid modes: (0->Off, 1->Smart, 2->Full')
+            return
+
+        if arg in (0,1,2):
+            self.shell.autocall = arg
+        else: # toggle
+            if self.shell.autocall:
+                self._magic_state.autocall_save = self.shell.autocall
+                self.shell.autocall = 0
+            else:
+                try:
+                    self.shell.autocall = self._magic_state.autocall_save
+                except AttributeError:
+                    self.shell.autocall = self._magic_state.autocall_save = 1
+
+        print "Automatic calling is:",['OFF','Smart','Full'][self.shell.autocall]
 
-    def magic_pprint(self, parameter_s=''):
-        """Toggle pretty printing on/off."""
-        ptformatter = self.shell.display_formatter.formatters['text/plain']
-        ptformatter.pprint = bool(1 - ptformatter.pprint)
-        print 'Pretty printing has been turned', \
-              ['OFF','ON'][ptformatter.pprint]
 
-    #......................................................................
-    # Functions to implement unix shell-type things
+class OSMagics(MagicFunctions):
+    """Magics to interact with the underlying OS (shell-type functionality).
+    """
 
     @skip_doctest
     def magic_alias(self, parameter_s = ''):
@@ -3345,133 +3614,146 Defaulting color scheme to 'NoColor'"""
 
         page.page(self.shell.pycolorize(cont))
 
-    def magic_quickref(self,arg):
-        """ Show a quick reference sheet """
-        import IPython.core.usage
-        qr = IPython.core.usage.quick_reference + self.magic_magic('-brief')
 
-        page.page(qr)
+class LoggingMagics(MagicFunctions):
+    """Magics related to all logging machinery."""
+    def magic_logstart(self,parameter_s=''):
+        """Start logging anywhere in a session.
 
-    def magic_doctest_mode(self,parameter_s=''):
-        """Toggle doctest mode on and off.
+        %logstart [-o|-r|-t] [log_name [log_mode]]
 
-        This mode is intended to make IPython behave as much as possible like a
-        plain Python shell, from the perspective of how its prompts, exceptions
-        and output look.  This makes it easy to copy and paste parts of a
-        session into doctests.  It does so by:
+        If no name is given, it defaults to a file named 'ipython_log.py' in your
+        current directory, in 'rotate' mode (see below).
 
-        - Changing the prompts to the classic ``>>>`` ones.
-        - Changing the exception reporting mode to 'Plain'.
-        - Disabling pretty-printing of output.
+        '%logstart name' saves to file 'name' in 'backup' mode.  It saves your
+        history up to that point and then continues logging.
 
-        Note that IPython also supports the pasting of code snippets that have
-        leading '>>>' and '...' prompts in them.  This means that you can paste
-        doctests from files or docstrings (even if they have leading
-        whitespace), and the code will execute correctly.  You can then use
-        '%history -t' to see the translated history; this will give you the
-        input after removal of all the leading prompts and whitespace, which
-        can be pasted back into an editor.
+        %logstart takes a second optional parameter: logging mode. This can be one
+        of (note that the modes are given unquoted):\\
+          append: well, that says it.\\
+          backup: rename (if exists) to name~ and start name.\\
+          global: single logfile in your home dir, appended to.\\
+          over  : overwrite existing log.\\
+          rotate: create rotating logs name.1~, name.2~, etc.
 
-        With these features, you can switch into this mode easily whenever you
-        need to do testing and changes to doctests, without having to leave
-        your existing IPython session.
-        """
+        Options:
 
-        from IPython.utils.ipstruct import Struct
+          -o: log also IPython's output.  In this mode, all commands which
+          generate an Out[NN] prompt are recorded to the logfile, right after
+          their corresponding input line.  The output lines are always
+          prepended with a '#[Out]# ' marker, so that the log remains valid
+          Python code.
 
-        # Shorthands
-        shell = self.shell
-        pm = shell.prompt_manager
-        meta = shell.meta
-        disp_formatter = self.shell.display_formatter
-        ptformatter = disp_formatter.formatters['text/plain']
-        # dstore is a data store kept in the instance metadata bag to track any
-        # changes we make, so we can undo them later.
-        dstore = meta.setdefault('doctest_mode',Struct())
-        save_dstore = dstore.setdefault
+          Since this marker is always the same, filtering only the output from
+          a log is very easy, using for example a simple awk call::
 
-        # save a few values we'll need to recover later
-        mode = save_dstore('mode',False)
-        save_dstore('rc_pprint',ptformatter.pprint)
-        save_dstore('xmode',shell.InteractiveTB.mode)
-        save_dstore('rc_separate_out',shell.separate_out)
-        save_dstore('rc_separate_out2',shell.separate_out2)
-        save_dstore('rc_prompts_pad_left',pm.justify)
-        save_dstore('rc_separate_in',shell.separate_in)
-        save_dstore('rc_plain_text_only',disp_formatter.plain_text_only)
-        save_dstore('prompt_templates',(pm.in_template, pm.in2_template, pm.out_template))
+            awk -F'#\\[Out\\]# ' '{if($2) {print $2}}' ipython_log.py
 
-        if mode == False:
-            # turn on
-            pm.in_template = '>>> '
-            pm.in2_template = '... '
-            pm.out_template = ''
+          -r: log 'raw' input.  Normally, IPython's logs contain the processed
+          input, so that user lines are logged in their final form, converted
+          into valid Python.  For example, %Exit is logged as
+          _ip.magic("Exit").  If the -r flag is given, all input is logged
+          exactly as typed, with no transformations applied.
 
-            # Prompt separators like plain python
-            shell.separate_in = ''
-            shell.separate_out = ''
-            shell.separate_out2 = ''
+          -t: put timestamps before each input line logged (these are put in
+          comments)."""
 
-            pm.justify = False
+        opts,par = self.parse_options(parameter_s,'ort')
+        log_output = 'o' in opts
+        log_raw_input = 'r' in opts
+        timestamp = 't' in opts
 
-            ptformatter.pprint = False
-            disp_formatter.plain_text_only = True
+        logger = self.shell.logger
 
-            shell.magic('xmode Plain')
+        # if no args are given, the defaults set in the logger constructor by
+        # ipython remain valid
+        if par:
+            try:
+                logfname,logmode = par.split()
+            except:
+                logfname = par
+                logmode = 'backup'
         else:
-            # turn off
-            pm.in_template, pm.in2_template, pm.out_template = dstore.prompt_templates
+            logfname = logger.logfname
+            logmode = logger.logmode
+        # put logfname into rc struct as if it had been called on the command
+        # line, so it ends up saved in the log header Save it in case we need
+        # to restore it...
+        old_logfile = self.shell.logfile
+        if logfname:
+            logfname = os.path.expanduser(logfname)
+        self.shell.logfile = logfname
 
-            shell.separate_in = dstore.rc_separate_in
+        loghead = '# IPython log file\n\n'
+        try:
+            started  = logger.logstart(logfname,loghead,logmode,
+                                       log_output,timestamp,log_raw_input)
+        except:
+            self.shell.logfile = old_logfile
+            warn("Couldn't start log: %s" % sys.exc_info()[1])
+        else:
+            # log input history up to this point, optionally interleaving
+            # output if requested
 
-            shell.separate_out = dstore.rc_separate_out
-            shell.separate_out2 = dstore.rc_separate_out2
+            if timestamp:
+                # disable timestamping for the previous history, since we've
+                # lost those already (no time machine here).
+                logger.timestamp = False
 
-            pm.justify = dstore.rc_prompts_pad_left
+            if log_raw_input:
+                input_hist = self.shell.history_manager.input_hist_raw
+            else:
+                input_hist = self.shell.history_manager.input_hist_parsed
 
-            ptformatter.pprint = dstore.rc_pprint
-            disp_formatter.plain_text_only = dstore.rc_plain_text_only
+            if log_output:
+                log_write = logger.log_write
+                output_hist = self.shell.history_manager.output_hist
+                for n in range(1,len(input_hist)-1):
+                    log_write(input_hist[n].rstrip() + '\n')
+                    if n in output_hist:
+                        log_write(repr(output_hist[n]),'output')
+            else:
+                logger.log_write('\n'.join(input_hist[1:]))
+                logger.log_write('\n')
+            if timestamp:
+                # re-enable timestamping
+                logger.timestamp = True
 
-            shell.magic('xmode ' + dstore.xmode)
+            print ('Activating auto-logging. '
+                   'Current session state plus future input saved.')
+            logger.logstate()
 
-        # Store new mode and inform
-        dstore.mode = bool(1-int(mode))
-        mode_label = ['OFF','ON'][dstore.mode]
-        print 'Doctest mode is:', mode_label
+    def magic_logstop(self,parameter_s=''):
+        """Fully stop logging and close log file.
 
-    def magic_gui(self, parameter_s=''):
-        """Enable or disable IPython GUI event loop integration.
+        In order to start logging again, a new %logstart call needs to be made,
+        possibly (though not necessarily) with a new filename, mode and other
+        options."""
+        self.logger.logstop()
 
-        %gui [GUINAME]
+    def magic_logoff(self,parameter_s=''):
+        """Temporarily stop logging.
 
-        This magic replaces IPython's threaded shells that were activated
-        using the (pylab/wthread/etc.) command line flags.  GUI toolkits
-        can now be enabled at runtime and keyboard
-        interrupts should work without any problems.  The following toolkits
-        are supported:  wxPython, PyQt4, PyGTK, Tk and Cocoa (OSX)::
+        You must have previously started logging."""
+        self.shell.logger.switch_log(0)
+
+    def magic_logon(self,parameter_s=''):
+        """Restart logging.
+
+        This function is for restarting logging which you've temporarily
+        stopped with %logoff. For starting logging for the first time, you
+        must use the %logstart function, which allows you to specify an
+        optional log filename."""
+
+        self.shell.logger.switch_log(1)
 
-            %gui wx      # enable wxPython event loop integration
-            %gui qt4|qt  # enable PyQt4 event loop integration
-            %gui gtk     # enable PyGTK event loop integration
-            %gui gtk3    # enable Gtk3 event loop integration
-            %gui tk      # enable Tk event loop integration
-            %gui OSX     # enable Cocoa event loop integration
-                         # (requires %matplotlib 1.1) 
-            %gui         # disable all event loop integration
+    def magic_logstate(self,parameter_s=''):
+        """Print the status of the logging system."""
 
-        WARNING:  after any of these has been called you can simply create
-        an application object, but DO NOT start the event loop yourself, as
-        we have already handled that.
-        """
-        opts, arg = self.parse_options(parameter_s, '')
-        if arg=='': arg = None
-        try:
-            return self.enable_gui(arg)
-        except Exception as e:
-            # print simple error message, rather than traceback if we can't
-            # hook up the GUI
-            error(str(e))
-    
+        self.shell.logger.logstate()
+
+class ExtensionsMagics(MagicFunctions):
+    """Magics to manage the IPython extensions system."""
     def magic_install_ext(self, parameter_s):
         """Download and install an extension from a URL, e.g.::
         
@@ -3510,6 +3792,9 Defaulting color scheme to 'NoColor'"""
         """Reload an IPython extension by its module name."""
         self.shell.extension_manager.reload_extension(module_str)
 
+
+class DeprecatedMagics(MagicFunctions):
+    """Magics slated for later removal."""
     def magic_install_profiles(self, s):
         """%install_profiles has been deprecated."""
         print '\n'.join([
@@ -3529,6 +3814,10 Defaulting color scheme to 'NoColor'"""
             "Add `--reset` to overwrite already existing config files with defaults."
         ])
 
+
+class PylabMagics(MagicFunctions):
+    """Magics related to matplotlib's pylab support"""
+
     @skip_doctest
     def magic_pylab(self, s):
         """Load numpy and matplotlib to work interactively.
@@ -3589,234 +3878,3 Defaulting color scheme to 'NoColor'"""
             import_all_status = True
 
         self.shell.enable_pylab(s, import_all=import_all_status)
-
-    def magic_tb(self, s):
-        """Print the last traceback with the currently active exception mode.
-
-        See %xmode for changing exception reporting modes."""
-        self.shell.showtraceback()
-
-    @skip_doctest
-    def magic_precision(self, s=''):
-        """Set floating point precision for pretty printing.
-
-        Can set either integer precision or a format string.
-
-        If numpy has been imported and precision is an int,
-        numpy display precision will also be set, via ``numpy.set_printoptions``.
-
-        If no argument is given, defaults will be restored.
-
-        Examples
-        --------
-        ::
-
-            In [1]: from math import pi
-
-            In [2]: %precision 3
-            Out[2]: u'%.3f'
-
-            In [3]: pi
-            Out[3]: 3.142
-
-            In [4]: %precision %i
-            Out[4]: u'%i'
-
-            In [5]: pi
-            Out[5]: 3
-
-            In [6]: %precision %e
-            Out[6]: u'%e'
-
-            In [7]: pi**10
-            Out[7]: 9.364805e+04
-
-            In [8]: %precision
-            Out[8]: u'%r'
-
-            In [9]: pi**10
-            Out[9]: 93648.047476082982
-
-        """
-
-        ptformatter = self.shell.display_formatter.formatters['text/plain']
-        ptformatter.float_precision = s
-        return ptformatter.float_format
-
-
-    @magic_arguments.magic_arguments()
-    @magic_arguments.argument(
-        '-e', '--export', action='store_true', default=False,
-        help='Export IPython history as a notebook. The filename argument '
-             'is used to specify the notebook name and format. For example '
-             'a filename of notebook.ipynb will result in a notebook name '
-             'of "notebook" and a format of "xml". Likewise using a ".json" '
-             'or ".py" file extension will write the notebook in the json '
-             'or py formats.'
-    )
-    @magic_arguments.argument(
-        '-f', '--format',
-        help='Convert an existing IPython notebook to a new format. This option '
-             'specifies the new format and can have the values: xml, json, py. '
-             'The target filename is chosen automatically based on the new '
-             'format. The filename argument gives the name of the source file.'
-    )
-    @magic_arguments.argument(
-        'filename', type=unicode,
-        help='Notebook name or filename'
-    )
-    def magic_notebook(self, s):
-        """Export and convert IPython notebooks.
-
-        This function can export the current IPython history to a notebook file
-        or can convert an existing notebook file into a different format. For
-        example, to export the history to "foo.ipynb" do "%notebook -e foo.ipynb".
-        To export the history to "foo.py" do "%notebook -e foo.py". To convert
-        "foo.ipynb" to "foo.json" do "%notebook -f json foo.ipynb". Possible
-        formats include (json/ipynb, py).
-        """
-        args = magic_arguments.parse_argstring(self.magic_notebook, s)
-
-        from IPython.nbformat import current
-        args.filename = unquote_filename(args.filename)
-        if args.export:
-            fname, name, format = current.parse_filename(args.filename)
-            cells = []
-            hist = list(self.shell.history_manager.get_range())
-            for session, prompt_number, input in hist[:-1]:
-                cells.append(current.new_code_cell(prompt_number=prompt_number,
-                                                   input=input))
-            worksheet = current.new_worksheet(cells=cells)
-            nb = current.new_notebook(name=name,worksheets=[worksheet])
-            with io.open(fname, 'w', encoding='utf-8') as f:
-                current.write(nb, f, format);
-        elif args.format is not None:
-            old_fname, old_name, old_format = current.parse_filename(args.filename)
-            new_format = args.format
-            if new_format == u'xml':
-                raise ValueError('Notebooks cannot be written as xml.')
-            elif new_format == u'ipynb' or new_format == u'json':
-                new_fname = old_name + u'.ipynb'
-                new_format = u'json'
-            elif new_format == u'py':
-                new_fname = old_name + u'.py'
-            else:
-                raise ValueError('Invalid notebook format: %s' % new_format)
-            with io.open(old_fname, 'r', encoding='utf-8') as f:
-                nb = current.read(f, old_format)
-            with io.open(new_fname, 'w', encoding='utf-8') as f:
-                current.write(nb, f, new_format)
-
-    def magic_config(self, s):
-        """configure IPython
-        
-            %config Class[.trait=value]
-        
-        This magic exposes most of the IPython config system. Any
-        Configurable class should be able to be configured with the simple
-        line::
-        
-            %config Class.trait=value
-        
-        Where `value` will be resolved in the user's namespace, if it is an
-        expression or variable name.
-        
-        Examples
-        --------
-        
-        To see what classes are available for config, pass no arguments::
-
-            In [1]: %config
-            Available objects for config:
-                TerminalInteractiveShell
-                HistoryManager
-                PrefilterManager
-                AliasManager
-                IPCompleter
-                PromptManager
-                DisplayFormatter
-
-        To view what is configurable on a given class, just pass the class
-        name::
-
-            In [2]: %config IPCompleter
-            IPCompleter options
-            -----------------
-            IPCompleter.omit__names=<Enum>
-                Current: 2
-                Choices: (0, 1, 2)
-                Instruct the completer to omit private method names
-                Specifically, when completing on ``object.<tab>``.
-                When 2 [default]: all names that start with '_' will be excluded.
-                When 1: all 'magic' names (``__foo__``) will be excluded.
-                When 0: nothing will be excluded.
-            IPCompleter.merge_completions=<CBool>
-                Current: True
-                Whether to merge completion results into a single list
-                If False, only the completion results from the first non-empty completer
-                will be returned.
-            IPCompleter.limit_to__all__=<CBool>
-                Current: False
-                Instruct the completer to use __all__ for the completion
-                Specifically, when completing on ``object.<tab>``.
-                When True: only those names in obj.__all__ will be included.
-                When False [default]: the __all__ attribute is ignored
-            IPCompleter.greedy=<CBool>
-                Current: False
-                Activate greedy completion
-                This will enable completion on elements of lists, results of function calls,
-                etc., but can be unsafe because the code is actually evaluated on TAB.
-
-        but the real use is in setting values::
-
-            In [3]: %config IPCompleter.greedy = True
-
-        and these values are read from the user_ns if they are variables::
-
-            In [4]: feeling_greedy=False
-
-            In [5]: %config IPCompleter.greedy = feeling_greedy
-
-        """
-        from IPython.config.loader import Config
-        # some IPython objects are Configurable, but do not yet have
-        # any configurable traits.  Exclude them from the effects of
-        # this magic, as their presence is just noise:
-        configurables = [ c for c in self.shell.configurables
-                          if c.__class__.class_traits(config=True) ]
-        classnames = [ c.__class__.__name__ for c in configurables ]
-        
-        line = s.strip()
-        if not line:
-            # print available configurable names
-            print "Available objects for config:"
-            for name in classnames:
-                print "    ", name
-            return
-        elif line in classnames:
-            # `%config TerminalInteractiveShell` will print trait info for
-            # TerminalInteractiveShell
-            c = configurables[classnames.index(line)]
-            cls = c.__class__
-            help = cls.class_get_help(c)
-            # strip leading '--' from cl-args:
-            help = re.sub(re.compile(r'^--', re.MULTILINE), '', help)
-            print help
-            return
-        elif '=' not in line:
-            raise UsageError("Invalid config statement: %r, should be Class.trait = value" % line)
-            
-        
-        # otherwise, assume we are setting configurables.
-        # leave quotes on args when splitting, because we want
-        # unquoted args to eval in user_ns
-        cfg = Config()
-        exec "cfg."+line in locals(), self.shell.user_ns
-        
-        for configurable in configurables:
-            try:
-                configurable.update_config(cfg)
-            except Exception as e:
-                error(e)
-
-# end Magic