General work on inputhook and the docs.

* Re-wrote the Sphinx docs on GUI integration.
* Improved docstrings of inputhook module.
* Added current_gui function to inputhook.
* Fixed lots of small bugs in the docs.

IPython/core/magic.py

@@ -3542,6 +3542,8 b' Defaulting color scheme to \'NoColor\'"""'
     def magic_gui(self, parameter_s=''):
         """Enable or disable IPython GUI event loop integration.
 
+        %gui [-a] [GUINAME]
+
         This magic replaces IPython's threaded shells that were activated
         using the (pylab/wthread/etc.) command line flags.  GUI toolkits
         can now be enabled, disabled and swtiched at runtime and keyboard

IPython/lib/inputhook.py

@@ -24,7 +24,11 b' import sys'
 
 
 class InputHookManager(object):
-    """Manage PyOS_InputHook for different GUI toolkits."""
+    """Manage PyOS_InputHook for different GUI toolkits.
+
+    This class installs various hooks under ``PyOSInputHook`` to handle
+    GUI event loop integration.
+    """
     
     def __init__(self):
         self.PYFUNC = ctypes.PYFUNCTYPE(ctypes.c_int)
@@ -34,6 +38,7 b' class InputHookManager(object):'
         self._callback_pyfunctype = None
         self._callback = None
         self._installed = False
+        self._current_gui = None
 
     def get_pyos_inputhook(self):
         """Return the current PyOS_InputHook as a ctypes.c_void_p.
@@ -68,7 +73,14 b' class InputHookManager(object):'
 
     def enable_wx(self, app=False):
         """Enable event loop integration with wxPython.
-        
+
+        Parameters
+        ----------
+        app : bool
+            Create a running application object or not.
+
+        Notes
+        -----
         This methods sets the PyOS_InputHook for wxPython, which allows
         the wxPython to integrate with terminal based applications like
         IPython.
@@ -86,6 +98,7 b' class InputHookManager(object):'
         """
         from IPython.lib.inputhookwx import inputhook_wx
         self.set_inputhook(inputhook_wx)
+        self._current_gui = 'wx'
         if app:
             import wx
             app = wx.App(redirect=False, clearSigInt=False)
@@ -101,6 +114,13 b' class InputHookManager(object):'
     def enable_qt4(self, app=False):
         """Enable event loop integration with PyQt4.
         
+        Parameters
+        ----------
+        app : bool
+            Create a running application object or not.
+
+        Notes
+        -----
         This methods sets the PyOS_InputHook for wxPython, which allows
         the PyQt4 to integrate with terminal based applications like
         IPython.
@@ -118,6 +138,7 b' class InputHookManager(object):'
             QtCore.pyqtRestoreInputHook()
         except AttributeError:
             pass
+        self._current_gui = 'qt4'
         if app:
             from PyQt4 import QtGui
             app = QtGui.QApplication(sys.argv)
@@ -132,7 +153,14 b' class InputHookManager(object):'
 
     def enable_gtk(self, app=False):
         """Enable event loop integration with PyGTK.
-        
+
+        Parameters
+        ----------
+        app : bool
+            Create a running application object or not.
+
+        Notes
+        -----
         This methods sets the PyOS_InputHook for PyGTK, which allows
         the PyGTK to integrate with terminal based applications like
         IPython.
@@ -144,6 +172,7 b' class InputHookManager(object):'
         import gtk
         try:
             gtk.set_interactive(True)
+            self._current_gui = 'gtk'
         except AttributeError:
             # For older versions of gtk, use our own ctypes version
             from IPython.lib.inputhookgtk import inputhook_gtk
@@ -157,8 +186,19 b' class InputHookManager(object):'
         self.clear_inputhook()
 
     def enable_tk(self, app=False):
-        # Creating a Tkinter.Tk object sets PyOS_InputHook()
-        pass
+        """Enable event loop integration with Tk.
+
+        Parameters
+        ----------
+        app : bool
+            Create a running application object or not.
+
+        Notes
+        -----
+        Currently this is a no-op as creating a :class:`Tkinter.Tk` object 
+        sets ``PyOS_InputHook``.
+        """
+        self._current_gui = 'tk'
 
     def disable_tk(self):
         """Disable event loop integration with Tkinter.
@@ -167,6 +207,10 b' class InputHookManager(object):'
         """
         self.clear_inputhook()
 
+    def current_gui(self):
+        """Return a string indicating the currently active GUI or None."""
+        return self._current_gui
+
 inputhook_manager = InputHookManager()
 
 enable_wx = inputhook_manager.enable_wx
@@ -178,4 +222,5 b' disable_gtk = inputhook_manager.disable_gtk'
 enable_tk = inputhook_manager.enable_tk
 disable_tk = inputhook_manager.disable_tk
 clear_inputhook = inputhook_manager.clear_inputhook
-set_inputhook = inputhook_manager.set_inputhook
  No newline at end of file
+set_inputhook = inputhook_manager.set_inputhook
+current_gui = inputhook_manager.current_gui
  No newline at end of file

docs/autogen_api.py

@@ -20,8 +20,10 b" if __name__ == '__main__':"
                                         r'\.extensions',
                                         r'\.kernel.config',
                                         r'\.attic',
+                                        r'\.quarantine',
+                                        r'\.deathrow'
                                         ]
-    docwriter.module_skip_patterns += [ r'\.FakeModule',
+    docwriter.module_skip_patterns += [ r'\.core.fakemodule',
                                         r'\.cocoa',
                                         r'\.ipdoctest',
                                         r'\.Gnuplot',

docs/source/development/overview.txt

@@ -132,7 +132,8 b' using the :command:`iptest` command (which is basically a customized version of'
     $ iptest
 
 The :command:`iptest` command will also pick up and run any tests you have
-written.  See :ref:`_devel_testing` for further details on the testing system.
+written. See :ref:`testing documentation <devel_testing>` for further details
+on the testing system.
 
 
 Post your branch and request a code review
@@ -170,35 +171,36 b' Some notes for core developers when merging third-party contributions'
 
 Core developers, who ultimately merge any approved branch (from themselves,
 another developer, or any third-party contribution) will typically use
-:command:`bzr merge` to merge the branch into the trunk and push it to the main
-Launcphad site.  This is a short list of things to keep in mind when doing this
-process, so that the project history is easy to understand in the long run, and
-that generating release notes is as painless and accurate as possible.
-
-- When you merge any non-trivial functionality (from one small bug fix to a big
-  feature branch), please remember to always edit the changes_ file
-  accordingly.  This file has one main section for each release, and if you
-  edit it as you go, noting what new features, bug fixes or API changes you
-  have made, the release notes will be almost finished when they are needed
-  later.  This is much easier if done when you merge the work, rather than
-  weeks or months later by re-reading a massive Bazaar log.
-
-- When big merges are done, the practice of putting a summary commit message in
-  the merge is *extremely* useful.  It makes this kind of job much nicer,
+:command:`bzr merge` to merge the branch into the trunk and push it to the
+main Launcphad site. This is a short list of things to keep in mind when doing
+this process, so that the project history is easy to understand in the long
+run, and that generating release notes is as painless and accurate as
+possible.
+
+- When you merge any non-trivial functionality (from one small bug fix to a
+  big feature branch), please remember to always edit the :file:`changes.txt`
+  file accordingly. This file has one main section for each release, and if
+  you edit it as you go, noting what new features, bug fixes or API changes
+  you have made, the release notes will be almost finished when they are
+  needed later. This is much easier if done when you merge the work, rather
+  than weeks or months later by re-reading a massive Bazaar log.
+
+- When big merges are done, the practice of putting a summary commit message
+  in the merge is *extremely* useful. It makes this kind of job much nicer,
   because that summary log message can be almost copy/pasted without changes,
   if it was well written, rather than dissecting the next-level messages from
   the individual commits.
 
 - It's important that we remember to always credit who gave us something if
-  it's not the committer.  In general, we have been fairly good on this front,
-  this is just a reminder to keep things up.  As a note, if you are ever
+  it's not the committer. In general, we have been fairly good on this front,
+  this is just a reminder to keep things up. As a note, if you are ever
   committing something that is completely (or almost so) a third-party
   contribution, do the commit as::
 
     $ bzr commit --author="Someone Else"
 
   This way it will show that name separately in the log, which makes it even
-  easier to spot.  Obviously we often rework third party contributions
+  easier to spot. Obviously we often rework third party contributions
   extensively, but this is still good to keep in mind for cases when we don't
   touch the code too much.
 

docs/source/development/reorg.txt

@@ -76,19 +76,9 b' Procedure'
 1. Move the file to its new location with its new name.
 2. Rename all import statements to reflect the change.
 3. Run PyFlakes on each changes module.
-3. Add tests/test_imports.py to test it.
+4. Add tests/test_imports.py to test it.
 
 Status
 ======
 
-The new subpackages have been created and the top-level modules have been
-moved and renamed. Import tests have been created for all of the moved and
-renamed modules. The build infrastructure (setup.py and friends) have been
-updated and tested on Mac and Windows. Finally, a compatibility layer has been
-added for iplib, ipapi and Shell. The follow things still need to be done::
-
-* I need to modify iptests to properly skip modules that are no longer top
-  level modules.
-
-* When running python setup.py sdist, the Sphinx API docs fail to build
-  because of something going on with IPython.core.fakemodule
+This branch was merged into trunk in early August of 2009.

docs/source/development/roadmap.txt

@@ -4,42 +4,34 b''
 Development roadmap
 ===================
 
-IPython is an ambitious project that is still under heavy development.  However, we want IPython to become useful to as many people as possible, as quickly as possible.  To help us accomplish this, we are laying out a roadmap of where we are headed and what needs to happen to get there.  Hopefully, this will help the IPython developers figure out the best things to work on for each upcoming release.
+IPython is an ambitious project that is still under heavy development.
+However, we want IPython to become useful to as many people as possible, as
+quickly as possible. To help us accomplish this, we are laying out a roadmap
+of where we are headed and what needs to happen to get there. Hopefully, this
+will help the IPython developers figure out the best things to work on for
+each upcoming release.
 
 Work targeted to particular releases
 ====================================
 
-Release 0.10
-------------
-
-* Initial refactor of :command:`ipcluster`.
-
-* Better TextMate integration.
-
-* Merge in the daemon branch.
-
 Release 0.11
 ------------
 
-* Refactor the configuration system and command line options for
-  :command:`ipengine` and :command:`ipcontroller`. This will include the
-  creation of cluster directories that encapsulate all the configuration
-  files, log files and security related files for a particular cluster.
+* [DONE] Full module and package reorganization.
+
+* [DONE] Removal of the threaded shells and new implementation of GUI support
+  based on ``PyOSInputHook``.
 
-* Refactor :command:`ipcluster` to support the new configuration system.
+* Refactor the configuration system.
 
-* Refactor the daemon stuff to support the new configuration system.
+* Prepare to refactor IPython's core by creating a new component and 
+  application system.
 
-* Merge back in the core of the notebook.
 
 Release 0.12
 ------------
 
-* Fully integrate process startup with the daemons for full process 
-  management.
 
-* Make the capabilites of :command:`ipcluster` available from simple Python 
-  classes.
 
 Major areas of work
 ===================

docs/source/interactive/reference.txt

@@ -25,71 +25,16 b' $HOME resolves to C:\\\\Documents and Settings\\\\YourUserName in most'
 instances. In the rest of this text, we will refer to this directory as
 IPYTHONDIR.
 
-.. _Threading options:
 
 
 Special Threading Options
 -------------------------
 
-The following special options are ONLY valid at the beginning of the
-command line, and not later. This is because they control the initial-
-ization of ipython itself, before the normal option-handling mechanism
-is active.
-
-    -gthread, -qthread, -q4thread, -wthread, -pylab: 
-	Only one of these can be given, and it can only be given as
-	the first option passed to IPython (it will have no effect in
-	any other position).  They provide threading support for the
-	GTK, Qt (versions 3 and 4) and WXPython toolkits, and for the
-	matplotlib library.
-
-	With any of the first four options, IPython starts running a
-	separate thread for the graphical toolkit's operation, so that
-	you can open and control graphical elements from within an
-	IPython command line, without blocking. All four provide
-	essentially the same functionality, respectively for GTK, Qt3,
-	Qt4 and WXWidgets (via their Python interfaces).
-
-	Note that with -wthread, you can additionally use the
-	-wxversion option to request a specific version of wx to be
-	used.  This requires that you have the wxversion Python module
-	installed, which is part of recent wxPython distributions.
-
-	If -pylab is given, IPython loads special support for the mat
-	plotlib library (http://matplotlib.sourceforge.net), allowing
-	interactive usage of any of its backends as defined in the
-	user's ~/.matplotlib/matplotlibrc file. It automatically
-	activates GTK, Qt or WX threading for IPyhton if the choice of
-	matplotlib backend requires it. It also modifies the %run
-	command to correctly execute (without blocking) any
-	matplotlib-based script which calls show() at the end.
-
-    -tk
-	The -g/q/q4/wthread options, and -pylab (if matplotlib is
-	configured to use GTK, Qt3, Qt4 or WX), will normally block Tk
-	graphical interfaces. This means that when either GTK, Qt or WX
-	threading is active, any attempt to open a Tk GUI will result in a
-	dead window, and possibly cause the Python interpreter to crash.
-	An extra option, -tk, is available to address this issue. It can
-	only be given as a second option after any of the above (-gthread,
-	-wthread or -pylab).
-
-	If -tk is given, IPython will try to coordinate Tk threading
-	with GTK, Qt or WX. This is however potentially unreliable, and
-	you will have to test on your platform and Python configuration to
-	determine whether it works for you. Debian users have reported
-	success, apparently due to the fact that Debian builds all of Tcl,
-	Tk, Tkinter and Python with pthreads support. Under other Linux
-	environments (such as Fedora Core 2/3), this option has caused
-	random crashes and lockups of the Python interpreter. Under other
-	operating systems (Mac OSX and Windows), you'll need to try it to
-	find out, since currently no user reports are available.
-    
-	There is unfortunately no way for IPython to determine at run time
-	whether -tk will work reliably or not, so you will need to do some
-	experiments before relying on it for regular work.
-
-
+Previously IPython had command line options for controlling GUI event loop
+integration (-gthread, -qthread, -q4thread, -wthread, -pylab). As of IPython
+version 0.11, these have been deprecated. Please see the new ``%gui``
+magic command or :ref:`this section <gui_support>` for details on the new
+interface.
 
 Regular Options
 ---------------
@@ -109,16 +54,8 b' All options with a [no] prepended can be specified in negated form'
     -help  print a help message and exit.
 
     -pylab
-	this can only be given as the first option passed to IPython
-	(it will have no effect in any other position). It adds
-	special support for the matplotlib library
-	(http://matplotlib.sourceforge.ne), allowing interactive usage
-	of any of its backends as defined in the user's .matplotlibrc
-	file.  It automatically activates GTK or WX threading for
-	IPyhton if the choice of matplotlib backend requires it. It
-	also modifies the %run command to correctly execute (without
-	blocking) any matplotlib-based script which calls show() at
-	the end. See `Matplotlib support`_ for more details.
+    Deprecated.  See :ref:`Matplotlib support <matplotlib_support>`
+    for more details.
 
     -autocall <val>
 	Make IPython automatically call any callable object even if you
@@ -367,9 +304,7 b' All options with a [no] prepended can be specified in negated form'
     -Version    print version information and exit.
 
     -wxversion <string> 
-	Select a specific version of wxPython (used in conjunction
-	with -wthread). Requires the wxversion module, part of recent
-	wxPython distributions
+    Deprecated.
 
     -xmode <modename>
 
@@ -1435,77 +1370,74 b' from math import * # math MUST be imported BEFORE PhysicalQInteractive'
 from IPython.extensions.PhysicalQInteractive import * 
 import IPython.extensions.PhysicalQInput
 
+.. _gui_support:
 
-Threading support
-=================
+GUI event loop support support
+==============================
+
+.. versionadded:: 0.11
+    The ``%gui`` magic and :mod:`IPython.lib.inputhook`.
+
+IPython has excellent support for working interactively with Graphical User
+Interface (GUI) toolkits, such as wxPython, PyQt4, PyGTK and Tk. This is
+implemented using Python's builtin ``PyOSInputHook`` hook. This implementation
+is extremely robust compared to our previous threaded based version. The
+advantages of
+
+* GUIs can be enabled and disabled dynamically at runtime.
+* The active GUI can be switched dynamically at runtime.
+* In some cases, multiple GUIs can run simultaneously with no problems.
+* There is a developer API in :mod:`IPython.lib.inputhook` for customizing 
+  all of these things.
+
+For users, enabling GUI event loop integration is simple.  You simple use the
+``%gui`` magic as follows::
+
+    %gui [-a] [GUINAME]
+
+With no arguments, ``%gui`` removes all GUI support.  Valid ``GUINAME``
+arguments are ``wx``, ``qt4``, ``gtk`` and ``tk``.  The ``-a`` option will
+create and return a running application object for the selected GUI toolkit.
+
+This to use wxPython interactively and create a running :class:`wx.App`
+object, do::
+
+    %gui -a wx
+
+For information on IPython's Matplotlib integration (and the ``pylab`` mode)
+see :ref:`this section <matplotlib_support>`.
 
-WARNING: The threading support is still somewhat experimental, and it
-has only seen reasonable testing under Linux. Threaded code is
-particularly tricky to debug, and it tends to show extremely
-platform-dependent behavior. Since I only have access to Linux machines,
-I will have to rely on user's experiences and assistance for this area
-of IPython to improve under other platforms.
-
-IPython, via the -gthread , -qthread, -q4thread and -wthread options
-(described in Sec. `Threading options`_), can run in
-multithreaded mode to support pyGTK, Qt3, Qt4 and WXPython applications
-respectively. These GUI toolkits need to control the python main loop of
-execution, so under a normal Python interpreter, starting a pyGTK, Qt3,
-Qt4 or WXPython application will immediately freeze the shell.
-
-IPython, with one of these options (you can only use one at a time),
-separates the graphical loop and IPython's code execution run into
-different threads. This allows you to test interactively (with %run, for
-example) your GUI code without blocking.
-
-A nice mini-tutorial on using IPython along with the Qt Designer
-application is available at the SciPy wiki:
-http://www.scipy.org/Cookbook/Matplotlib/Qt_with_IPython_and_Designer.
-
-
-Tk issues
----------
-
-As indicated in Sec. `Threading options`_, a special -tk option is
-provided to try and allow Tk graphical applications to coexist
-interactively with WX, Qt or GTK ones. Whether this works at all,
-however, is very platform and configuration dependent. Please
-experiment with simple test cases before committing to using this
-combination of Tk and GTK/Qt/WX threading in a production environment.
-
-
-I/O pitfalls
-------------
-
-Be mindful that the Python interpreter switches between threads every
-$N$ bytecodes, where the default value as of Python 2.3 is $N=100.$ This
-value can be read by using the sys.getcheckinterval() function, and it
-can be reset via sys.setcheckinterval(N). This switching of threads can
-cause subtly confusing effects if one of your threads is doing file I/O.
-In text mode, most systems only flush file buffers when they encounter a
-'\n'. An instruction as simple as:: 
-
-  print >> filehandle, ''hello world''
-
-actually consists of several bytecodes, so it is possible that the
-newline does not reach your file before the next thread switch.
-Similarly, if you are writing to a file in binary mode, the file won't
-be flushed until the buffer fills, and your other thread may see
-apparently truncated files.
-
-For this reason, if you are using IPython's thread support and have (for
-example) a GUI application which will read data generated by files
-written to from the IPython thread, the safest approach is to open all
-of your files in unbuffered mode (the third argument to the file/open
-function is the buffering value)::
-
-  filehandle = open(filename,mode,0)
-
-This is obviously a brute force way of avoiding race conditions with the
-file buffering. If you want to do it cleanly, and you have a resource
-which is being shared by the interactive IPython loop and your GUI
-thread, you should really handle it with thread locking and
-syncrhonization properties. The Python documentation discusses these.
+For developers that want to use IPython's GUI event loop integration in
+the form of a library, the capabilities are exposed in library form
+in the :mod:`IPython.lib.inputhook`.  Interested developers should see the
+module docstrings for more information.
+
+.. _matplotlib_support:
+
+Plotting with matplotlib
+========================
+
+
+`Matplotlib`_ provides high quality 2D and
+3D plotting for Python. Matplotlib can produce plots on screen using a variety
+of GUI toolkits, including Tk, PyGTK, PyQt4 and wxPython. It also provides a
+number of commands useful for scientific computing, all with a syntax
+compatible with that of the popular Matlab program.
+
+Many IPython users have come to rely on IPython's ``-pylab`` mode which
+automates the integration of Matplotlib with IPython. We are still in the
+process of working with the Matplotlib developers to finalize the new pylab
+API, but for now you can use Matplotlib interactively using the following
+commands::
+
+    %gui -a wx
+    import matplotlib
+    matplotlib.use('wxagg')
+    from matplotlib import pylab
+    pylab.interactive(True)
+
+All of this will soon be automated as Matplotlib beings to include
+new logic that uses our new GUI support.
 
 .. _interactive_demos:
 
@@ -1603,30 +1535,5 b' divisions are allowed. If you want to be able to open an IPython'
 instance at an arbitrary point in a program, you can use IPython's
 embedding facilities, described in detail in Sec. 9
 
+.. [Matplotlib] Matplotlib. http://matplotlib.sourceforge.net
 
-.. _Matplotlib support:
-
-Plotting with matplotlib
-========================
-
-The matplotlib library (http://matplotlib.sourceforge.net
-http://matplotlib.sourceforge.net) provides high quality 2D plotting for
-Python. Matplotlib can produce plots on screen using a variety of GUI
-toolkits, including Tk, GTK and WXPython. It also provides a number of
-commands useful for scientific computing, all with a syntax compatible
-with that of the popular Matlab program.
-
-IPython accepts the special option -pylab (see :ref:`here
-<command_line_options>`). This configures it to support matplotlib, honoring
-the settings in the .matplotlibrc file. IPython will detect the user's choice
-of matplotlib GUI backend, and automatically select the proper threading model
-to prevent blocking. It also sets matplotlib in interactive mode and modifies
-%run slightly, so that any matplotlib-based script can be executed using %run
-and the final show() command does not block the interactive shell.
-
-The -pylab option must be given first in order for IPython to configure its
-threading mode. However, you can still issue other options afterwards. This
-allows you to have a matplotlib-based environment customized with additional
-modules using the standard IPython profile mechanism (see :ref:`here
-<profiles>`): ``ipython -pylab -p myprofile`` will load the profile defined in
-ipythonrc-myprofile after configuring matplotlib.

docs/source/parallel/index.txt

@@ -13,4 +13,3 b' Using IPython for parallel computing'
    parallel_task.txt
    parallel_mpi.txt
    parallel_security.txt
-   visionhpc.txt

docs/source/parallel/parallel_intro.txt

@@ -90,13 +90,15 b' asynchronous interface to a set of engines.'
     Because the controller listens on a network port for engines to 
     connect to it, it must be started *before* any engines are started.  
 
-The controller also provides a single point of contact for users who wish
-to utilize the engines connected to the controller. There are different
-ways of working with a controller. In IPython these ways correspond to different interfaces that the controller is adapted to.  Currently we have two default interfaces to the controller: 
+The controller also provides a single point of contact for users who wish to
+utilize the engines connected to the controller. There are different ways of
+working with a controller. In IPython these ways correspond to different
+interfaces that the controller is adapted to. Currently we have two default
+interfaces to the controller:
 
 * The MultiEngine interface, which provides the simplest possible way of
   working with engines interactively.
-* The Task interface, which provides presents the engines as a load balanced
+* The Task interface, which presents the engines as a load balanced
   task farming system.
 
 Advanced users can easily add new custom interfaces to enable other
@@ -121,11 +123,30 b' interface. Here are the two default clients:'
 Security
 --------
 
-By default (as long as `pyOpenSSL` is installed) all network connections between the controller and engines and the controller and clients are secure.  What does this mean?  First of all, all of the connections will be encrypted using SSL.  Second, the connections are authenticated.  We handle authentication in a capability based security model [Capability]_.  In this model,  a "capability (known in some systems as a key) is a communicable, unforgeable token of authority".  Put simply, a capability is like a key to your house.  If you have the key to your house, you can get in.  If not, you can't.  
-
-In our architecture, the controller is the only process that listens on network ports, and is thus responsible to creating these keys.  In IPython, these keys are known as Foolscap URLs, or FURLs, because of the underlying network protocol we are using.  As a user, you don't need to know anything about the details of these FURLs, other than that when the controller starts, it saves a set of FURLs to files named :file:`something.furl`.  The default location of these files is the :file:`~./ipython/security` directory.
-
-To connect and authenticate to the controller an engine or client simply needs to present an appropriate FURL (that was originally created by the controller) to the controller.  Thus, the FURL files need to be copied to a location where the clients and engines can find them.  Typically, this is the :file:`~./ipython/security` directory on the host where the client/engine is running (which could be a different host than the controller).  Once the FURL files are copied over, everything should work fine.
+By default (as long as `pyOpenSSL` is installed) all network connections
+between the controller and engines and the controller and clients are secure.
+What does this mean? First of all, all of the connections will be encrypted
+using SSL. Second, the connections are authenticated. We handle authentication
+in a capability based security model [Capability]_. In this model, a
+"capability (known in some systems as a key) is a communicable, unforgeable
+token of authority". Put simply, a capability is like a key to your house. If
+you have the key to your house, you can get in. If not, you can't.
+
+In our architecture, the controller is the only process that listens on
+network ports, and is thus responsible to creating these keys. In IPython,
+these keys are known as Foolscap URLs, or FURLs, because of the underlying
+network protocol we are using. As a user, you don't need to know anything
+about the details of these FURLs, other than that when the controller starts,
+it saves a set of FURLs to files named :file:`something.furl`. The default
+location of these files is the :file:`~./ipython/security` directory.
+
+To connect and authenticate to the controller an engine or client simply needs
+to present an appropriate FURL (that was originally created by the controller)
+to the controller. Thus, the FURL files need to be copied to a location where
+the clients and engines can find them. Typically, this is the
+:file:`~./ipython/security` directory on the host where the client/engine is
+running (which could be a different host than the controller). Once the FURL
+files are copied over, everything should work fine.
 
 Currently, there are three FURL files that the controller creates:
 
@@ -149,12 +170,16 b' can be found :ref:`here <parallelsecurity>`.'
 Getting Started
 ===============
 
-To use IPython for parallel computing, you need to start one instance of
-the controller and one or more instances of the engine. Initially, it is best to simply start a controller and engines on a single host using the :command:`ipcluster` command.  To start a controller and 4 engines on you localhost, just do::
+To use IPython for parallel computing, you need to start one instance of the
+controller and one or more instances of the engine. Initially, it is best to
+simply start a controller and engines on a single host using the
+:command:`ipcluster` command. To start a controller and 4 engines on your
+localhost, just do::
 
     $ ipcluster local -n 4
 
-More details about starting the IPython controller and engines can be found :ref:`here <parallel_process>`
+More details about starting the IPython controller and engines can be found
+:ref:`here <parallel_process>`
 
 Once you have started the IPython controller and one or more engines, you
 are ready to use the engines to do something useful. To make sure
@@ -184,7 +209,12 b' everything is working correctly, try the following commands:'
 	[3] In [1]: print "Hello World"
 	[3] Out[1]: Hello World
 
-Remember, a client also needs to present a FURL file to the controller.  How does this happen?  When a multiengine client is created with no arguments, the client tries to find the corresponding FURL file in the local :file:`~./ipython/security` directory.  If it finds it, you are set.  If you have put the FURL file in a different location or it has a different name, create the client like this::
+Remember, a client also needs to present a FURL file to the controller. How
+does this happen? When a multiengine client is created with no arguments, the
+client tries to find the corresponding FURL file in the local
+:file:`~./ipython/security` directory. If it finds it, you are set. If you
+have put the FURL file in a different location or it has a different name,
+create the client like this::
 
     mec = client.MultiEngineClient('/path/to/my/ipcontroller-mec.furl')
 
@@ -192,7 +222,9 b' Same thing hold true of creating a task client::'
 
     tc = client.TaskClient('/path/to/my/ipcontroller-tc.furl')
 
-You are now ready to learn more about the :ref:`MultiEngine <parallelmultiengine>` and :ref:`Task <paralleltask>` interfaces to the controller.
+You are now ready to learn more about the :ref:`MultiEngine
+<parallelmultiengine>` and :ref:`Task <paralleltask>` interfaces to the
+controller.
 
 .. note:: 
 

docs/source/parallel/parallel_mpi.txt

@@ -4,9 +4,16 b''
 Using MPI with IPython
 =======================
 
-Often, a parallel algorithm will require moving data between the engines.  One way of accomplishing this is by doing a pull and then a push using the multiengine client.  However, this will be slow as all the data has to go through the controller to the client and then back through the controller, to its final destination.
+Often, a parallel algorithm will require moving data between the engines. One
+way of accomplishing this is by doing a pull and then a push using the
+multiengine client. However, this will be slow as all the data has to go
+through the controller to the client and then back through the controller, to
+its final destination.
 
-A much better way of moving data between engines is to use a message passing library, such as the Message Passing Interface (MPI) [MPI]_.  IPython's parallel computing architecture has been designed from the ground up to integrate with MPI.  This document describes how to use MPI with IPython.
+A much better way of moving data between engines is to use a message passing
+library, such as the Message Passing Interface (MPI) [MPI]_. IPython's
+parallel computing architecture has been designed from the ground up to
+integrate with MPI. This document describes how to use MPI with IPython.
 
 Additional installation requirements
 ====================================
@@ -35,12 +42,15 b' To use code that calls MPI, there are typically two things that MPI requires.'
    :command:`mpiexec` or a batch system (like PBS) that has MPI support.
 2. Once the process starts, it must call :func:`MPI_Init`.
 
-There are a couple of ways that you can start the IPython engines and get these things to happen.
+There are a couple of ways that you can start the IPython engines and get
+these things to happen.
 
 Automatic starting using :command:`mpiexec` and :command:`ipcluster`
 --------------------------------------------------------------------
 
-The easiest approach is to use the `mpiexec` mode of :command:`ipcluster`, which will first start a controller and then a set of engines using :command:`mpiexec`::
+The easiest approach is to use the `mpiexec` mode of :command:`ipcluster`,
+which will first start a controller and then a set of engines using
+:command:`mpiexec`::
 
     $ ipcluster mpiexec -n 4
 
@@ -50,7 +60,8 b' stop and clean up the controller and engines.'
 Manual starting using :command:`mpiexec`
 ----------------------------------------
 
-If you want to start the IPython engines using the :command:`mpiexec`, just do::
+If you want to start the IPython engines using the :command:`mpiexec`, just
+do::
 
     $ mpiexec -n 4 ipengine --mpi=mpi4py
 
@@ -64,14 +75,20 b' starting the engines with::'
 Automatic starting using PBS and :command:`ipcluster`
 -----------------------------------------------------
 
-The :command:`ipcluster` command also has built-in integration with PBS. For more information on this approach, see our documentation on :ref:`ipcluster <parallel_process>`.
+The :command:`ipcluster` command also has built-in integration with PBS. For
+more information on this approach, see our documentation on :ref:`ipcluster
+<parallel_process>`.
 
 Actually using MPI
 ==================
 
-Once the engines are running with MPI enabled, you are ready to go.  You can now call any code that uses MPI in the IPython engines.  And, all of this can be done interactively.  Here we show a simple example that uses mpi4py [mpi4py]_.
+Once the engines are running with MPI enabled, you are ready to go. You can
+now call any code that uses MPI in the IPython engines. And, all of this can
+be done interactively. Here we show a simple example that uses mpi4py
+[mpi4py]_.
 
-First, lets define a simply function that uses MPI to calculate the sum of a distributed array.  Save the following text in a file called :file:`psum.py`:
+First, lets define a simply function that uses MPI to calculate the sum of a
+distributed array. Save the following text in a file called :file:`psum.py`:
 
 .. sourcecode:: python
 
@@ -86,7 +103,9 b' Now, start an IPython cluster in the same directory as :file:`psum.py`::'
 
     $ ipcluster mpiexec -n 4
 
-Finally, connect to the cluster and use this function interactively.  In this case, we create a random array on each engine and sum up all the random arrays using our :func:`psum` function:
+Finally, connect to the cluster and use this function interactively. In this
+case, we create a random array on each engine and sum up all the random arrays
+using our :func:`psum` function:
 
 .. sourcecode:: ipython
 

docs/source/parallel/parallel_multiengine.txt

@@ -58,7 +58,10 b' Here we see that there are four engines ready to do work for us.'
 Quick and easy parallelism
 ==========================
 
-In many cases, you simply want to apply a Python function to a sequence of objects, but *in parallel*.  The multiengine interface provides two simple ways of accomplishing this:  a parallel version of :func:`map` and ``@parallel`` function decorator.
+In many cases, you simply want to apply a Python function to a sequence of
+objects, but *in parallel*. The multiengine interface provides two simple ways
+of accomplishing this: a parallel version of :func:`map` and ``@parallel``
+function decorator.
 
 Parallel map
 ------------
@@ -90,7 +93,9 b' parallel version of :meth:`map` that works just like its serial counterpart:'
 Parallel function decorator
 ---------------------------
 
-Parallel functions are just like normal function, but they can be called on sequences and *in parallel*.  The multiengine interface provides a decorator that turns any Python function into a parallel function:
+Parallel functions are just like normal function, but they can be called on
+sequences and *in parallel*. The multiengine interface provides a decorator
+that turns any Python function into a parallel function:
 
 .. sourcecode:: ipython
 
@@ -667,7 +672,9 b' more other types of exceptions. Here is how it works:'
 	[2:execute]: ZeroDivisionError: integer division or modulo by zero
 	[3:execute]: ZeroDivisionError: integer division or modulo by zero
 
-Notice how the error message printed when :exc:`CompositeError` is raised has information about the individual exceptions that were raised on each engine.  If you want, you can even raise one of these original exceptions:
+Notice how the error message printed when :exc:`CompositeError` is raised has
+information about the individual exceptions that were raised on each engine.
+If you want, you can even raise one of these original exceptions:
 
 .. sourcecode:: ipython
 

docs/source/parallel/parallel_process.txt

@@ -15,14 +15,20 b' Broadly speaking, there are two ways of going about starting a controller and en'
 * In a more manual way using the :command:`ipcontroller` and 
   :command:`ipengine` commands.
 
-This document describes both of these methods.  We recommend that new users start with the :command:`ipcluster` command as it simplifies many common usage cases.
+This document describes both of these methods. We recommend that new users
+start with the :command:`ipcluster` command as it simplifies many common usage
+cases.
 
 General considerations
 ======================
 
-Before delving into the details about how you can start a controller and engines using the various methods, we outline some of the general issues that come up when starting the controller and engines.  These things come up no matter which method you use to start your IPython cluster.
+Before delving into the details about how you can start a controller and
+engines using the various methods, we outline some of the general issues that
+come up when starting the controller and engines. These things come up no
+matter which method you use to start your IPython cluster.
 
-Let's say that you want to start the controller on ``host0`` and engines on hosts ``host1``-``hostn``.  The following steps are then required:
+Let's say that you want to start the controller on ``host0`` and engines on
+hosts ``host1``-``hostn``. The following steps are then required:
 
 1. Start the controller on ``host0`` by running :command:`ipcontroller` on
    ``host0``.
@@ -41,12 +47,15 b' at that location.'
 The final step required required to actually use the running controller from a
 client is to move the FURL files :file:`ipcontroller-mec.furl` and
 :file:`ipcontroller-tc.furl` from ``host0`` to the host where the clients will
-be run.  If these file are put into the :file:`~/.ipython/security` directory of the client's host, they will be found automatically.  Otherwise, the full path to them has to be passed to the client's constructor.
+be run. If these file are put into the :file:`~/.ipython/security` directory
+of the client's host, they will be found automatically. Otherwise, the full
+path to them has to be passed to the client's constructor.
 
 Using :command:`ipcluster`
 ==========================
 
-The :command:`ipcluster` command provides a simple way of starting a controller and engines in the following situations:
+The :command:`ipcluster` command provides a simple way of starting a
+controller and engines in the following situations:
 
 1. When the controller and engines are all run on localhost. This is useful
    for testing or running on a multicore computer.
@@ -110,11 +119,20 b' This does the following:'
 1. Starts the IPython controller on current host.
 2. Uses :command:`mpiexec` to start 4 engines.
 
-On newer MPI implementations (such as OpenMPI), this will work even if you don't make any calls to MPI or call :func:`MPI_Init`.  However, older MPI implementations actually require each process to call :func:`MPI_Init` upon starting.  The easiest way of having this done is to install the mpi4py [mpi4py]_ package and then call ipcluster with the ``--mpi`` option::
+On newer MPI implementations (such as OpenMPI), this will work even if you
+don't make any calls to MPI or call :func:`MPI_Init`. However, older MPI
+implementations actually require each process to call :func:`MPI_Init` upon
+starting. The easiest way of having this done is to install the mpi4py
+[mpi4py]_ package and then call ipcluster with the ``--mpi`` option::
 
     $ ipcluster mpiexec -n 4 --mpi=mpi4py
 
-Unfortunately, even this won't work for some MPI implementations.  If you are having problems with this, you will likely have to use a custom Python executable that itself calls :func:`MPI_Init` at the appropriate time.  Fortunately, mpi4py comes with such a custom Python executable that is easy to install and use.  However, this custom Python executable approach will not work with :command:`ipcluster` currently.
+Unfortunately, even this won't work for some MPI implementations. If you are
+having problems with this, you will likely have to use a custom Python
+executable that itself calls :func:`MPI_Init` at the appropriate time.
+Fortunately, mpi4py comes with such a custom Python executable that is easy to
+install and use. However, this custom Python executable approach will not work
+with :command:`ipcluster` currently.
 
 Additional command line options for this mode can be found by doing::
 
@@ -126,7 +144,9 b' More details on using MPI with IPython can be found :ref:`here <parallelmpi>`.'
 Using :command:`ipcluster` in PBS mode
 --------------------------------------
 
-The PBS mode uses the Portable Batch System [PBS]_ to start the engines.  To use this mode, you first need to create a PBS script template that will be used to start the engines.  Here is a sample PBS script template:
+The PBS mode uses the Portable Batch System [PBS]_ to start the engines. To
+use this mode, you first need to create a PBS script template that will be
+used to start the engines. Here is a sample PBS script template:
 
 .. sourcecode:: bash
 
@@ -161,7 +181,8 b' There are a few important points about this template:'
 5. Depending on the configuration of you system, you may have to set 
    environment variables in the script template.
 
-Once you have created such a script, save it with a name like :file:`pbs.template`.  Now you are ready to start your job::
+Once you have created such a script, save it with a name like
+:file:`pbs.template`. Now you are ready to start your job::
 
     $ ipcluster pbs -n 128 --pbs-script=pbs.template
 
@@ -175,9 +196,11 b' Using :command:`ipcluster` in SSH mode'
 The SSH mode uses :command:`ssh` to execute :command:`ipengine` on remote
 nodes and the :command:`ipcontroller` on localhost.
 
-When using using this mode it highly recommended that you have set up SSH keys and are using ssh-agent [SSH]_ for password-less logins.
+When using using this mode it highly recommended that you have set up SSH keys
+and are using ssh-agent [SSH]_ for password-less logins.
 
-To use this mode you need a python file describing the cluster, here is an example of such a "clusterfile":
+To use this mode you need a python file describing the cluster, here is an
+example of such a "clusterfile":
 
 .. sourcecode:: python
     
@@ -187,7 +210,8 b' To use this mode you need a python file describing the cluster, here is an examp'
                 'host3.example.com' : 1,
                 'host4.example.com' : 8 }
 
-Since this is a regular python file usual python syntax applies. Things to note:
+Since this is a regular python file usual python syntax applies. Things to
+note:
 
 * The `engines` dict, where the keys is the host we want to run engines on and
   the value is the number of engines to run on that host.
@@ -204,12 +228,16 b' start your cluster like so:'
    $ ipcluster ssh --clusterfile /path/to/my/clusterfile.py
 
 
-Two helper shell scripts are used to start and stop :command:`ipengine` on remote hosts:
+Two helper shell scripts are used to start and stop :command:`ipengine` on
+remote hosts:
 
 * sshx.sh 
 * engine_killer.sh
 
-Defaults for both of these are contained in the source code for :command:`ipcluster`. The default scripts are written to a local file in a tmep directory and then copied to a temp directory on the remote host and executed from there.  On most Unix, Linux and OS X systems this is /tmp.
+Defaults for both of these are contained in the source code for
+:command:`ipcluster`. The default scripts are written to a local file in a
+tmep directory and then copied to a temp directory on the remote host and
+executed from there. On most Unix, Linux and OS X systems this is /tmp.
 
 The default sshx.sh is the following:
 
@@ -241,7 +269,9 b' Current limitations of the SSH mode of :command:`ipcluster` are:'
 Using the :command:`ipcontroller` and :command:`ipengine` commands
 ==================================================================
 
-It is also possible to use the :command:`ipcontroller` and :command:`ipengine` commands to start your controller and engines.  This approach gives you full control over all aspects of the startup process.
+It is also possible to use the :command:`ipcontroller` and :command:`ipengine`
+commands to start your controller and engines. This approach gives you full
+control over all aspects of the startup process.
 
 Starting the controller and engine on your local machine
 --------------------------------------------------------
@@ -253,11 +283,14 b' First start the controller::'
 
 	$ ipcontroller
 	
-Next, start however many instances of the engine you want using (repeatedly) the command::
+Next, start however many instances of the engine you want using (repeatedly)
+the command::
 
 	$ ipengine
 
-The engines should start and automatically connect to the controller using the FURL files in :file:`~./ipython/security`. You are now ready to use the controller and engines from IPython.
+The engines should start and automatically connect to the controller using the
+FURL files in :file:`~./ipython/security`. You are now ready to use the
+controller and engines from IPython.
 
 .. warning:: 
 	
@@ -279,10 +312,13 b' When the controller and engines are running on different hosts, things are'
 slightly more complicated, but the underlying ideas are the same:
 
 1. Start the controller on a host using :command:`ipcontroller`.
-2. Copy :file:`ipcontroller-engine.furl` from :file:`~./ipython/security` on the controller's host to the host where the engines will run.
+2. Copy :file:`ipcontroller-engine.furl` from :file:`~./ipython/security` on
+   the controller's host to the host where the engines will run.
 3. Use :command:`ipengine` on the engine's hosts to start the engines.
 
-The only thing you have to be careful of is to tell :command:`ipengine` where the :file:`ipcontroller-engine.furl` file is located.  There are two ways you can do this:
+The only thing you have to be careful of is to tell :command:`ipengine` where
+the :file:`ipcontroller-engine.furl` file is located. There are two ways you
+can do this:
 
 * Put :file:`ipcontroller-engine.furl` in the :file:`~./ipython/security`
   directory on the engine's host, where it will be found automatically.

docs/source/parallel/parallel_security.txt

@@ -52,7 +52,8 b' When running the IPython kernel to perform a parallel computation, a user'
 utilizes the IPython client to send Python commands and data through the
 IPython controller to the IPython engines, where those commands are executed
 and the data processed. The design of IPython ensures that the client is the
-only access point for the capabilities of the engines.  That is, the only way of addressing the engines is through a client.
+only access point for the capabilities of the engines. That is, the only way
+of addressing the engines is through a client.
 
 A user can utilize the client to instruct the IPython engines to execute
 arbitrary Python commands. These Python commands can include calls to the
@@ -116,8 +117,8 b' controller creates a number of FURLs for different purposes:'
   to execute.
 
 Upon starting, the controller creates these different FURLS and writes them
-files in the user-read-only directory :file:`$HOME/.ipython/security`. Thus, only the
-user who starts the controller has access to the FURLs.
+files in the user-read-only directory :file:`$HOME/.ipython/security`. Thus,
+only the user who starts the controller has access to the FURLs.
 
 For an IPython client or engine to authenticate with a controller, it must
 present the appropriate FURL to the controller upon connecting. If the

docs/source/parallel/parallel_task.txt

@@ -4,9 +4,17 b''
 The IPython task interface
 ==========================
 
-The task interface to the controller presents the engines as a fault tolerant, dynamic load-balanced system or workers. Unlike the multiengine interface, in the task interface, the user have no direct access to individual engines. In some ways, this interface is simpler, but in other ways it is more powerful.
-
-Best of all the user can use both of these interfaces running at the same time to take advantage or both of their strengths.  When the user can break up the user's work into segments that do not depend on previous execution, the task interface is ideal.  But it also has more power and flexibility, allowing the user to guide the distribution of jobs, without having to assign tasks to engines explicitly.
+The task interface to the controller presents the engines as a fault tolerant,
+dynamic load-balanced system or workers. Unlike the multiengine interface, in
+the task interface, the user have no direct access to individual engines. In
+some ways, this interface is simpler, but in other ways it is more powerful.
+
+Best of all the user can use both of these interfaces running at the same time
+to take advantage or both of their strengths. When the user can break up the
+user's work into segments that do not depend on previous execution, the task
+interface is ideal. But it also has more power and flexibility, allowing the
+user to guide the distribution of jobs, without having to assign tasks to
+engines explicitly.
 
 Starting the IPython controller and engines
 ===========================================
@@ -44,12 +52,19 b' constructor:'
 Quick and easy parallelism
 ==========================
 
-In many cases, you simply want to apply a Python function to a sequence of objects, but *in parallel*.  Like the multiengine interface, the task interface provides two simple ways of accomplishing this:  a parallel version of :func:`map` and ``@parallel`` function decorator.  However, the verions in the task interface have one important difference:  they are dynamically load balanced.  Thus, if the execution time per item varies significantly, you should use the versions in the task interface.
+In many cases, you simply want to apply a Python function to a sequence of
+objects, but *in parallel*. Like the multiengine interface, the task interface
+provides two simple ways of accomplishing this: a parallel version of
+:func:`map` and ``@parallel`` function decorator. However, the verions in the
+task interface have one important difference: they are dynamically load
+balanced. Thus, if the execution time per item varies significantly, you
+should use the versions in the task interface.
 
 Parallel map
 ------------
 
-The parallel :meth:`map` in the task interface is similar to that in the multiengine interface:
+The parallel :meth:`map` in the task interface is similar to that in the
+multiengine interface:
 
 .. sourcecode:: ipython
 
@@ -63,7 +78,9 b' The parallel :meth:`map` in the task interface is similar to that in the multien'
 Parallel function decorator
 ---------------------------
 
-Parallel functions are just like normal function, but they can be called on sequences and *in parallel*.  The multiengine interface provides a decorator that turns any Python function into a parallel function:
+Parallel functions are just like normal function, but they can be called on
+sequences and *in parallel*. The multiengine interface provides a decorator
+that turns any Python function into a parallel function:
 
 .. sourcecode:: ipython
 
@@ -79,7 +96,9 b' Parallel functions are just like normal function, but they can be called on sequ'
 More details
 ============
 
-The :class:`TaskClient` has many more powerful features that allow quite a bit of flexibility in how tasks are defined and run.  The next places to look are in the following classes:
+The :class:`TaskClient` has many more powerful features that allow quite a bit
+of flexibility in how tasks are defined and run. The next places to look are
+in the following classes:
 
 * :class:`IPython.kernel.client.TaskClient`
 * :class:`IPython.kernel.client.StringTask`
@@ -95,5 +114,7 b' The following is an overview of how to use these classes together:'
 4. Use :meth:`TaskClient.get_task_result` to get the results of the
    tasks.
 
-We are in the process of developing more detailed information about the task interface.  For now, the docstrings of the :class:`TaskClient`, :class:`StringTask` and :class:`MapTask` classes should be consulted.
+We are in the process of developing more detailed information about the task
+interface. For now, the docstrings of the :class:`TaskClient`,
+:class:`StringTask` and :class:`MapTask` classes should be consulted.