Merging the new-extensions branch.

* Previously, the latex Sphinx docs were in a single chapter.  This has been
  fixed by adding a sixth argument of True to the ``latex_documents``
  attribute of :file:`conf.py`.

* The ``psum`` example in the MPI documentation has been updated to mpi4py
  version 1.1.0.  Thanks to J. Thomas for this fix.

* The top-level, zero-install :file:`ipython.py` script has been updated to 
  the new application launching API.

* The extension loading functions have been renamed to
  :func:`load_ipython_extension` and :func:`unload_ipython_extension`.

* The :mod:`IPython.extensions.pretty` extension has been moved out of
  quarantine and fully updated to the new extension API.

* New magics for loading/unloading/reloading extensions have been added:
  ``%load_ext``, ``%unload_ext`` and ``%reload_ext``.

IPython/extensions/pretty.py

@@ -0,0 +1,222 b''
+"""Use pretty.py for configurable pretty-printing.
+
+To enable this extension in your configuration
+file, add the following to :file:`ipython_config.py`::
+
+    c.Global.extensions = ['IPython.extensions.pretty']
+    def dict_pprinter(obj, p, cycle):
+        return p.text("<dict>")
+    c.PrettyResultDisplay.verbose = True
+    c.PrettyResultDisplay.defaults_for_type = [
+        (dict, dict_pprinter)
+    ]
+    c.PrettyResultDisplay.defaults_for_type_by_name = [
+        ('numpy', 'dtype', 'IPython.extensions.pretty.dtype_pprinter')
+    ]
+
+This extension can also be loaded by using the ``%load_ext`` magic::
+
+    %load_ext IPython.extensions.pretty
+
+If this extension is enabled, you can always add additional pretty printers
+by doing::
+
+    ip = get_ipython()
+    prd = ip.get_component('pretty_result_display')
+    import numpy
+    from IPython.extensions.pretty import dtype_pprinter
+    prd.for_type(numpy.dtype, dtype_pprinter)
+
+    # If you don't want to have numpy imported until it needs to be:
+    prd.for_type_by_name('numpy', 'dtype', dtype_pprinter)
+"""
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+from IPython.core.error import TryNext
+from IPython.external import pretty
+from IPython.core.component import Component
+from IPython.utils.traitlets import Bool, List
+from IPython.utils.genutils import Term
+from IPython.utils.autoattr import auto_attr
+from IPython.utils.importstring import import_item
+
+#-----------------------------------------------------------------------------
+# Code
+#-----------------------------------------------------------------------------
+
+
+_loaded = False
+
+
+class PrettyResultDisplay(Component):
+    """A component for pretty printing on steroids."""
+
+    verbose = Bool(False, config=True)
+
+    # A list of (type, func_name), like
+    # [(dict, 'my_dict_printer')]
+    # The final argument can also be a callable
+    defaults_for_type = List(default_value=[], config=True)
+
+    # A list of (module_name, type_name, func_name), like
+    # [('numpy', 'dtype', 'IPython.extensions.pretty.dtype_pprinter')]
+    # The final argument can also be a callable
+    defaults_for_type_by_name = List(default_value=[], config=True)
+
+    def __init__(self, parent, name=None, config=None):
+        super(PrettyResultDisplay, self).__init__(parent, name=name, config=config)
+        self._setup_defaults()
+
+    def _setup_defaults(self):
+        """Initialize the default pretty printers."""
+        for typ, func_name in self.defaults_for_type:
+            func = self._resolve_func_name(func_name)
+            self.for_type(typ, func)
+        for type_module, type_name, func_name in self.defaults_for_type_by_name:
+            func = self._resolve_func_name(func_name)
+            self.for_type_by_name(type_module, type_name, func)
+
+    def _resolve_func_name(self, func_name):
+        if callable(func_name):
+            return func_name
+        elif isinstance(func_name, basestring):
+            return import_item(func_name)
+        else:
+            raise TypeError('func_name must be a str or callable, got: %r' % func_name)
+
+    # Access other components like this rather than by a regular attribute.
+    # This won't lookup the InteractiveShell object until it is used and
+    # then it is cached.  This is both efficient and couples this class 
+    # more loosely to InteractiveShell.
+    @auto_attr
+    def shell(self):
+        return Component.get_instances(
+            root=self.root,
+            klass='IPython.core.iplib.InteractiveShell')[0]
+
+    def __call__(self, otherself, arg):
+        """Uber-pretty-printing display hook.
+
+        Called for displaying the result to the user.
+        """
+        
+        if self.shell.pprint:
+            out = pretty.pretty(arg, verbose=self.verbose)
+            if '\n' in out:
+                # So that multi-line strings line up with the left column of
+                # the screen, instead of having the output prompt mess up
+                # their first line.                
+                Term.cout.write('\n')
+            print >>Term.cout, out
+        else:
+            raise TryNext
+
+    def for_type(self, typ, func):
+        """Add a pretty printer for a type."""
+        return pretty.for_type(typ, func)
+
+    def for_type_by_name(self, type_module, type_name, func):
+        """Add a pretty printer for a type by its name and module name."""
+        return pretty.for_type_by_name(type_module, type_name, func)
+
+
+#-----------------------------------------------------------------------------
+# Initialization code for the extension
+#-----------------------------------------------------------------------------
+
+
+def load_ipython_extension(ip):
+    """Load the extension in IPython as a hook."""
+    global _loaded
+    if not _loaded:
+        prd = PrettyResultDisplay(ip, name='pretty_result_display')
+        ip.set_hook('result_display', prd, priority=99)
+        _loaded = True
+
+def unload_ipython_extension(ip):
+    """Unload the extension."""
+    # The hook system does not have a way to remove a hook so this is a pass
+    pass
+
+
+#-----------------------------------------------------------------------------
+# Example pretty printers
+#-----------------------------------------------------------------------------
+
+
+def dtype_pprinter(obj, p, cycle):
+    """ A pretty-printer for numpy dtype objects.
+    """
+    if cycle:
+        return p.text('dtype(...)')
+    if hasattr(obj, 'fields'):
+        if obj.fields is None:
+            p.text(repr(obj))
+        else:
+            p.begin_group(7, 'dtype([')
+            for i, field in enumerate(obj.descr):
+                if i > 0:
+                    p.text(',')
+                    p.breakable()
+                p.pretty(field)
+            p.end_group(7, '])')
+
+
+#-----------------------------------------------------------------------------
+# Tests
+#-----------------------------------------------------------------------------
+
+
+def test_pretty():
+    """
+    In [1]: from IPython.extensions import ipy_pretty
+
+    In [2]: ipy_pretty.activate()
+
+    In [3]: class A(object):
+       ...:     def __repr__(self):
+       ...:         return 'A()'
+       ...:     
+       ...:     
+
+    In [4]: a = A()
+
+    In [5]: a
+    Out[5]: A()
+
+    In [6]: def a_pretty_printer(obj, p, cycle):
+       ...:     p.text('<A>')
+       ...:     
+       ...:     
+
+    In [7]: ipy_pretty.for_type(A, a_pretty_printer)
+
+    In [8]: a
+    Out[8]: <A>
+
+    In [9]: class B(object):
+       ...:     def __repr__(self):
+       ...:         return 'B()'
+       ...:     
+       ...:     
+
+    In [10]: B.__module__, B.__name__
+    Out[10]: ('__main__', 'B')
+
+    In [11]: def b_pretty_printer(obj, p, cycle):
+       ....:     p.text('<B>')
+       ....:     
+       ....:     
+
+    In [12]: ipy_pretty.for_type_by_name('__main__', 'B', b_pretty_printer)
+
+    In [13]: b = B()
+
+    In [14]: b
+    Out[14]: <B>
+    """
+    assert False, "This should only be doctested, not run."
+

IPython/extensions/tests/test_pretty.py

@@ -0,0 +1,56 b''
+#!/usr/bin/env python
+# encoding: utf-8
+"""
+Simple tests for :mod:`IPython.extensions.pretty`.
+"""
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2009  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+import sys
+from unittest import TestCase
+
+from IPython.core.component import Component, masquerade_as
+from IPython.core.iplib import InteractiveShell
+from IPython.extensions import pretty as pretty_ext
+from IPython.external import pretty
+
+from IPython.utils.traitlets import Bool
+
+#-----------------------------------------------------------------------------
+# Tests
+#-----------------------------------------------------------------------------
+
+
+class InteractiveShellStub(Component):
+    pprint = Bool(True)
+
+class A(object):
+    pass
+
+def a_pprinter(o, p, c):
+    return p.text("<A>")
+
+class TestPrettyResultDisplay(TestCase):
+
+    def setUp(self):
+        self.ip = InteractiveShellStub(None)
+        # This allows our stub to be retrieved instead of the real InteractiveShell
+        masquerade_as(self.ip, InteractiveShell)
+        self.prd = pretty_ext.PrettyResultDisplay(self.ip, name='pretty_result_display')    
+
+    def test_for_type(self):
+        self.prd.for_type(A, a_pprinter)
+        a = A()
+        result = pretty.pretty(a)
+        self.assertEquals(result, "<A>")
+
+

IPython/core/builtin_trap.py

@@ -61,7 +61,8 b' class BuiltinTrap(Component):'
         if self._nested_level == 1:
             self.unset()
         self._nested_level -= 1
-        return True
+        # Returning False will cause exceptions to propagate
+        return False
 
     def add_builtin(self, key, value):
         """Add a builtin and save the original."""

IPython/core/display_trap.py

@@ -62,7 +62,8 b' class DisplayTrap(Component):'
         if self._nested_level == 1:
             self.unset()
         self._nested_level -= 1
-        return True
+        # Returning False will cause exceptions to propagate
+        return False
 
     def set(self):
         """Set the hook."""

IPython/core/iplib.py

@@ -1603,9 +1603,7 b' class InteractiveShell(Component, Magic):'
             magic_args = self.var_expand(magic_args,1)
             with nested(self.builtin_trap,):
                 result = fn(magic_args)
-            # Unfortunately, the return statement is what will trigger
-            # the displayhook, but it is no longer set!
-            return result
+                return result
 
     def define_magic(self, magicname, func):
         """Expose own function as magic function for ipython 
@@ -2274,16 +2272,28 b' class InteractiveShell(Component, Magic):'
             return lineout
 
     #-------------------------------------------------------------------------
+    # Working with components
+    #-------------------------------------------------------------------------
+
+    def get_component(self, name=None, klass=None):
+        """Fetch a component by name and klass in my tree."""
+        c = Component.get_instances(root=self, name=name, klass=klass)
+        if len(c) == 1:
+            return c[0]
+        else:
+            return c
+
+    #-------------------------------------------------------------------------
     # IPython extensions
     #-------------------------------------------------------------------------
 
     def load_extension(self, module_str):
-        """Load an IPython extension.
+        """Load an IPython extension by its module name.
 
         An IPython extension is an importable Python module that has
         a function with the signature::
 
-            def load_in_ipython(ipython):
+            def load_ipython_extension(ipython):
                 # Do things with ipython
 
         This function is called after your extension is imported and the 
@@ -2292,6 +2302,10 b' class InteractiveShell(Component, Magic):'
         that point, including defining new magic and aliases, adding new
         components, etc.
 
+        The :func:`load_ipython_extension` will be called again is you 
+        load or reload the extension again.  It is up to the extension
+        author to add code to manage that.
+
         You can put your extension modules anywhere you want, as long as
         they can be imported by Python's standard import mechanism.  However,
         to make it easy to write extensions, you can also put your extensions
@@ -2300,29 +2314,47 b' class InteractiveShell(Component, Magic):'
         """
         from IPython.utils.syspathcontext import prepended_to_syspath
 
-        if module_str in sys.modules:
-            return
+        if module_str not in sys.modules:
+            with prepended_to_syspath(self.ipython_extension_dir):
+                __import__(module_str)
+        mod = sys.modules[module_str]
+        self._call_load_ipython_extension(mod)
 
-        with prepended_to_syspath(self.ipython_extension_dir):
-            __import__(module_str)
+    def unload_extension(self, module_str):
+        """Unload an IPython extension by its module name.
+
+        This function looks up the extension's name in ``sys.modules`` and
+        simply calls ``mod.unload_ipython_extension(self)``.
+        """
+        if module_str in sys.modules:
             mod = sys.modules[module_str]
-            self._call_load_in_ipython(mod)
+            self._call_unload_ipython_extension(mod)
 
     def reload_extension(self, module_str):
-        """Reload an IPython extension by doing reload."""
+        """Reload an IPython extension by calling reload.
+
+        If the module has not been loaded before,
+        :meth:`InteractiveShell.load_extension` is called. Otherwise
+        :func:`reload` is called and then the :func:`load_ipython_extension`
+        function of the module, if it exists is called.
+        """
         from IPython.utils.syspathcontext import prepended_to_syspath
 
         with prepended_to_syspath(self.ipython_extension_dir):
             if module_str in sys.modules:
                 mod = sys.modules[module_str]
                 reload(mod)
-                self._call_load_in_ipython(mod)
+                self._call_load_ipython_extension(mod)
             else:
-                self.load_extension(self, module_str)
+                self.load_extension(module_str)
+
+    def _call_load_ipython_extension(self, mod):
+        if hasattr(mod, 'load_ipython_extension'):
+            mod.load_ipython_extension(self)
 
-    def _call_load_in_ipython(self, mod):
-        if hasattr(mod, 'load_in_ipython'):
-            mod.load_in_ipython(self)
+    def _call_unload_ipython_extension(self, mod):
+        if hasattr(mod, 'unload_ipython_extension'):
+            mod.unload_ipython_extension(self)
 
     #-------------------------------------------------------------------------
     # Things related to the prefilter

IPython/core/magic.py

@@ -3538,5 +3538,16 b' Defaulting color scheme to \'NoColor\'"""'
         elif 'tk' in parameter_s:
             return inputhook.enable_tk(app)
 
+    def magic_load_ext(self, module_str):
+        """Load an IPython extension by its module name."""
+        self.load_extension(module_str)
+
+    def magic_unload_ext(self, module_str):
+        """Unload an IPython extension by its module name."""
+        self.unload_extension(module_str)
+
+    def magic_reload_ext(self, module_str):
+        """Reload an IPython extension by its module name."""
+        self.reload_extension(module_str)
 
 # end Magic

docs/source/conf.py

@@ -164,7 +164,7 b" latex_font_size = '11pt'"
 
 latex_documents = [ ('index', 'ipython.tex', 'IPython Documentation',
                      ur"""The IPython Development Team""",
-                     'manual'),
+                     'manual', True),
                     ]
 
 # The name of an image file (relative to this directory) to place at the top of

docs/source/config/ipython.txt

@@ -54,15 +54,17 b' the following attributes can be set in the ``Global`` section.'
 :attr:`c.Global.extensions`
     A list of strings, each of which is an importable IPython extension. An
     IPython extension is a regular Python module or package that has a
-    :func:`load_in_ipython(ip)` method. This method gets called when the
-    extension is loaded with the currently running
+    :func:`load_ipython_extension(ip)` method. This method gets called when
+    the extension is loaded with the currently running
     :class:`~IPython.core.iplib.InteractiveShell` as its only argument. You
     can put your extensions anywhere they can be imported but we add the
     :file:`extensions` subdirectory of the ipython directory to ``sys.path``
-    during extension loading, so you can put them there as well.  Extensions
-    are not executed in the user's interactive namespace and they must
-    be pure Python code.  Extensions are the recommended way of customizing
-    :command:`ipython`.
+    during extension loading, so you can put them there as well. Extensions
+    are not executed in the user's interactive namespace and they must be pure
+    Python code. Extensions are the recommended way of customizing
+    :command:`ipython`. Extensions can provide an
+    :func:`unload_ipython_extension` that will be called when the extension is
+    unloaded.
 
 :attr:`c.Global.exec_lines`
     A list of strings, each of which is Python code that is run in the user's

docs/source/development/reorg.txt

@@ -23,11 +23,16 b' Subpackage descriptions'
   this code will either i) be revived by someone willing to maintain it with
   tests and docs and re-included into IPython or 2) be removed from IPython
   proper, but put into a separate third-party Python package. No new code will
-  be allowed here.
+  be allowed here. If your favorite extension has been moved here please
+  contact the IPython developer mailing list to help us determine the best
+  course of action.
 
 * :mod:`IPython.extensions`.  This package contains fully supported IPython
   extensions.  These extensions adhere to the official IPython extension API
   and can be enabled by adding them to a field in the configuration file.
+  If your extension is no longer in this location, please look in
+  :mod:`IPython.quarantine` and :mod:`IPython.deathrow` and contact the
+  IPython developer mailing list.
 
 * :mod:`IPython.external`. This package contains third party packages and
   modules that IPython ships internally to reduce the number of dependencies.
@@ -49,7 +54,9 b' Subpackage descriptions'
 * :mod:`IPython.quarantine`. This is for code that doesn't meet IPython's
   standards, but that we plan on keeping. To be moved out of this sub-package
   a module needs to have approval of the core IPython developers, tests and
-  documentation.
+  documentation. If your favorite extension has been moved here please contact
+  the IPython developer mailing list to help us determine the best course of
+  action.
 
 * :mod:`IPython.scripts`.  This package contains a variety of top-level
   command line scripts.  Eventually, these should be moved to the 

docs/source/parallel/parallel_mpi.txt

@@ -85,7 +85,7 b' Actually using MPI'
 Once the engines are running with MPI enabled, you are ready to go. You can
 now call any code that uses MPI in the IPython engines. And, all of this can
 be done interactively. Here we show a simple example that uses mpi4py
-[mpi4py]_.
+[mpi4py]_ version 1.1.0 or later.
 
 First, lets define a simply function that uses MPI to calculate the sum of a
 distributed array. Save the following text in a file called :file:`psum.py`:
@@ -94,10 +94,14 b' distributed array. Save the following text in a file called :file:`psum.py`:'
 
     from mpi4py import MPI
     import numpy as np
-    
+
     def psum(a):
         s = np.sum(a)
-        return MPI.COMM_WORLD.Allreduce(s,MPI.SUM)
+        rcvBuf = np.array(0.0,'d')
+        MPI.COMM_WORLD.Allreduce([s, MPI.DOUBLE],
+            [rcvBuf, MPI.DOUBLE],
+            op=MPI.SUM)
+        return rcvBuf
 
 Now, start an IPython cluster in the same directory as :file:`psum.py`::
 

docs/source/whatsnew/development.txt

@@ -26,6 +26,12 b' For more details, please consult the actual source.'
 New features
 ------------
 
+* The :mod:`IPython.extensions.pretty` extension has been moved out of
+  quarantine and fully updated to the new extension API.
+
+* New magics for loading/unloading/reloading extensions have been added:
+  ``%load_ext``, ``%unload_ext`` and ``%reload_ext``.
+
 * The configuration system and configuration files are brand new. See the
   configuration system :ref:`documentation <config_index>` for more details.
 
@@ -133,12 +139,25 b' New features'
 Bug fixes
 ---------
 
+* Previously, the latex Sphinx docs were in a single chapter.  This has been
+  fixed by adding a sixth argument of True to the ``latex_documents``
+  attribute of :file:`conf.py`.
+
+* The ``psum`` example in the MPI documentation has been updated to mpi4py
+  version 1.1.0.  Thanks to J. Thomas for this fix.
+
+* The top-level, zero-install :file:`ipython.py` script has been updated to 
+  the new application launching API.
+
 * Keyboard interrupts now work with GUI support enabled across all platforms
   and all GUI toolkits reliably.
 
 Backwards incompatible changes
 ------------------------------
 
+* The extension loading functions have been renamed to
+  :func:`load_ipython_extension` and :func:`unload_ipython_extension`.
+
 * :class:`~IPython.core.iplib.InteractiveShell` no longer takes an
   ``embedded`` argument. Instead just use the
   :class:`~IPython.core.iplib.InteractiveShellEmbed` class.

ipython.py

@@ -7,5 +7,6 b" in './scripts' directory. This file is here (ipython source root directory)"
 to facilitate non-root 'zero-installation' (just copy the source tree
 somewhere and run ipython.py) and development. """
 
-import IPython.core.shell
-IPython.core.shell.start().mainloop()
+from IPython.core.ipapp import launch_new_instance
+
+launch_new_instance()