merge from trunk

IPython/frontend/tests/test_asyncfrontendbase.py

@@ -0,0 +1,155 b''
+# encoding: utf-8
+
+"""This file contains unittests for the frontendbase module."""
+
+__docformat__ = "restructuredtext en"
+
+#---------------------------------------------------------------------------
+#  Copyright (C) 2008  The IPython Development Team                         
+#                                                                           
+#  Distributed under the terms of the BSD License.  The full license is in  
+#  the file COPYING, distributed as part of this software.                  
+#---------------------------------------------------------------------------
+                                                                            
+#---------------------------------------------------------------------------
+# Imports                                                                   
+#---------------------------------------------------------------------------
+
+import unittest
+
+try:
+    from IPython.frontend.asyncfrontendbase import AsyncFrontEndBase
+    from IPython.frontend import frontendbase 
+    from IPython.kernel.engineservice import EngineService
+except ImportError:
+    import nose
+    raise nose.SkipTest("This test requires zope.interface, Twisted and Foolscap")
+
+from IPython.testing.decorators import skip
+
+class FrontEndCallbackChecker(AsyncFrontEndBase):
+    """FrontEndBase subclass for checking callbacks"""
+    def __init__(self, engine=None, history=None):
+        super(FrontEndCallbackChecker, self).__init__(engine=engine, 
+                                                    history=history)
+        self.updateCalled = False
+        self.renderResultCalled = False
+        self.renderErrorCalled = False
+   
+    def update_cell_prompt(self, result, blockID=None):
+        self.updateCalled = True
+        return result
+   
+    def render_result(self, result):
+        self.renderResultCalled = True
+        return result
+    
+    
+    def render_error(self, failure):
+        self.renderErrorCalled = True
+        return failure
+    
+
+
+    
+class TestAsyncFrontendBase(unittest.TestCase):
+    def setUp(self):
+        """Setup the EngineService and FrontEndBase"""
+        
+        self.fb = FrontEndCallbackChecker(engine=EngineService())
+    
+    def test_implements_IFrontEnd(self):
+        assert(frontendbase.IFrontEnd.implementedBy(
+                                    AsyncFrontEndBase))
+    
+    def test_is_complete_returns_False_for_incomplete_block(self):
+        """"""
+        
+        block = """def test(a):"""
+        
+        assert(self.fb.is_complete(block) == False)
+    
+    def test_is_complete_returns_True_for_complete_block(self):
+        """"""
+        
+        block = """def test(a): pass"""
+            
+        assert(self.fb.is_complete(block))
+        
+        block = """a=3"""
+        
+        assert(self.fb.is_complete(block))
+    
+    def test_blockID_added_to_result(self):
+        block = """3+3"""
+        
+        d = self.fb.execute(block, blockID='TEST_ID')
+        
+        d.addCallback(self.checkBlockID, expected='TEST_ID')
+    
+    def test_blockID_added_to_failure(self):
+        block = "raise Exception()"
+        
+        d = self.fb.execute(block,blockID='TEST_ID')
+        d.addErrback(self.checkFailureID, expected='TEST_ID')
+    
+    def checkBlockID(self, result, expected=""):
+        assert(result['blockID'] == expected)
+    
+    
+    def checkFailureID(self, failure, expected=""):
+        assert(failure.blockID == expected)
+    
+    
+    def test_callbacks_added_to_execute(self):
+        """test that
+        update_cell_prompt
+        render_result
+        
+        are added to execute request
+        """
+        
+        d = self.fb.execute("10+10")
+        d.addCallback(self.checkCallbacks)
+    
+    def checkCallbacks(self, result):
+        assert(self.fb.updateCalled)
+        assert(self.fb.renderResultCalled)
+    
+    @skip("This test fails and lead to an unhandled error in a Deferred.")
+    def test_error_callback_added_to_execute(self):
+        """test that render_error called on execution error"""
+        
+        d = self.fb.execute("raise Exception()")
+        d.addCallback(self.checkRenderError)
+    
+    def checkRenderError(self, result):
+        assert(self.fb.renderErrorCalled)
+    
+    def test_history_returns_expected_block(self):
+        """Make sure history browsing doesn't fail"""
+        
+        blocks = ["a=1","a=2","a=3"]
+        for b in blocks:
+            d = self.fb.execute(b)
+        
+        # d is now the deferred for the last executed block
+        d.addCallback(self.historyTests, blocks)
+        
+    
+    def historyTests(self, result, blocks):
+        """historyTests"""
+        
+        assert(len(blocks) >= 3)
+        assert(self.fb.get_history_previous("") == blocks[-2])
+        assert(self.fb.get_history_previous("") == blocks[-3])
+        assert(self.fb.get_history_next() == blocks[-2])
+    
+    
+    def test_history_returns_none_at_startup(self):
+        """test_history_returns_none_at_startup"""
+        
+        assert(self.fb.get_history_previous("")==None)
+        assert(self.fb.get_history_next()==None)
+    
+    

IPython/kernel/core/tests/test_interpreter.py

@@ -0,0 +1,26 b''
+# encoding: utf-8
+
+"""This file contains unittests for the interpreter.py module."""
+
+__docformat__ = "restructuredtext en"
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008  The IPython Development Team                           
+#                                                                             
+#  Distributed under the terms of the BSD License.  The full license is in    
+#  the file COPYING, distributed as part of this software.                    
+#-----------------------------------------------------------------------------
+                                                                              
+#-----------------------------------------------------------------------------
+# Imports                                                                     
+#-----------------------------------------------------------------------------
+
+from IPython.kernel.core.interpreter import Interpreter
+
+def test_unicode():
+    """ Test unicode handling with the interpreter.
+    """
+    i = Interpreter()
+    i.execute_python(u'print "ù"')
+    i.execute_python('print "ù"')
+

IPython/testing/tests/test_decorators.py

@@ -0,0 +1,161 b''
+"""Tests for the decorators we've created for IPython.
+"""
+
+# Module imports
+# Std lib
+import inspect
+import sys
+
+# Third party
+import nose.tools as nt
+
+# Our own
+from IPython.testing import decorators as dec
+
+
+#-----------------------------------------------------------------------------
+# Utilities
+
+# Note: copied from OInspect, kept here so the testing stuff doesn't create
+# circular dependencies and is easier to reuse.
+def getargspec(obj):
+    """Get the names and default values of a function's arguments.
+
+    A tuple of four things is returned: (args, varargs, varkw, defaults).
+    'args' is a list of the argument names (it may contain nested lists).
+    'varargs' and 'varkw' are the names of the * and ** arguments or None.
+    'defaults' is an n-tuple of the default values of the last n arguments.
+
+    Modified version of inspect.getargspec from the Python Standard
+    Library."""
+
+    if inspect.isfunction(obj):
+        func_obj = obj
+    elif inspect.ismethod(obj):
+        func_obj = obj.im_func
+    else:
+        raise TypeError, 'arg is not a Python function'
+    args, varargs, varkw = inspect.getargs(func_obj.func_code)
+    return args, varargs, varkw, func_obj.func_defaults
+
+#-----------------------------------------------------------------------------
+# Testing functions
+
[e-mail]
+def test_deliberately_broken():
+    """A deliberately broken test - we want to skip this one."""
+    1/0
+
[e-mail]('foo')
+def test_deliberately_broken2():
+    """Another deliberately broken test - we want to skip this one."""
+    1/0
+
+
+# Verify that we can correctly skip the doctest for a function at will, but
+# that the docstring itself is NOT destroyed by the decorator.
[e-mail]_doctest
+def doctest_bad(x,y=1,**k):
+    """A function whose doctest we need to skip.
+
+    >>> 1+1
+    3
+    """
+    print 'x:',x
+    print 'y:',y
+    print 'k:',k
+
+
+def call_doctest_bad():
+    """Check that we can still call the decorated functions.
+    
+    >>> doctest_bad(3,y=4)
+    x: 3
+    y: 4
+    k: {}
+    """
+    pass
+
+
+def test_skip_dt_decorator():
+    """Doctest-skipping decorator should preserve the docstring.
+    """
+    # Careful: 'check' must be a *verbatim* copy of the doctest_bad docstring!
+    check = """A function whose doctest we need to skip.
+
+    >>> 1+1
+    3
+    """
+    # Fetch the docstring from doctest_bad after decoration.
+    val = doctest_bad.__doc__
+    
+    assert check==val,"doctest_bad docstrings don't match"
+
+# Doctest skipping should work for class methods too
+class foo(object):
+    """Foo
+
+    Example:
+
+    >>> 1+1
+    2
+    """
+
+    @dec.skip_doctest
+    def __init__(self,x):
+        """Make a foo.
+
+        Example:
+
+        >>> f = foo(3)
+        junk
+        """
+        print 'Making a foo.'
+        self.x = x
+        
+    @dec.skip_doctest
+    def bar(self,y):
+        """Example:
+
+        >>> f = foo(3)
+        >>> f.bar(0)
+        boom!
+        >>> 1/0
+        bam!
+        """
+        return 1/y
+
+    def baz(self,y):
+        """Example:
+
+        >>> f = foo(3)
+        Making a foo.
+        >>> f.baz(3)
+        True
+        """
+        return self.x==y
+
+
+
+def test_skip_dt_decorator2():
+    """Doctest-skipping decorator should preserve function signature.
+    """
+    # Hardcoded correct answer
+    dtargs = (['x', 'y'], None, 'k', (1,))
+    # Introspect out the value
+    dtargsr = getargspec(doctest_bad)
+    assert dtargsr==dtargs, \
+           "Incorrectly reconstructed args for doctest_bad: %s" % (dtargsr,)
+
+
[e-mail]_linux
+def test_linux():
+    nt.assert_not_equals(sys.platform,'linux2',"This test can't run under linux")
+
[e-mail]_win32
+def test_win32():
+    nt.assert_not_equals(sys.platform,'win32',"This test can't run under windows")
+
[e-mail]_osx
+def test_osx():
+    nt.assert_not_equals(sys.platform,'darwin',"This test can't run under osx")

IPython/tests/test_genutils.py

@@ -0,0 +1,32 b''
+# encoding: utf-8
+
+"""Tests for genutils.py"""
+
+__docformat__ = "restructuredtext en"
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+from IPython import genutils
+
+
+def test_get_home_dir():
+    """Make sure we can get the home directory."""
+    home_dir = genutils.get_home_dir()
+
+def test_get_ipython_dir():
+    """Make sure we can get the ipython directory."""
+    ipdir = genutils.get_ipython_dir()
+
+def test_get_security_dir():
+    """Make sure we can get the ipython/security directory."""
+    sdir = genutils.get_security_dir()
+    
  No newline at end of file

docs/source/attic/parallel_task_old.txt

@@ -0,0 +1,240 b''
+.. _paralleltask:
+
+==========================
+The IPython task interface
+==========================
+
+.. contents::
+
+The ``Task`` interface to the controller presents the engines as a fault tolerant, dynamic load-balanced system or workers. Unlike the ``MultiEngine`` interface, in the ``Task`` interface, the user have no direct access to individual engines. In some ways, this interface is simpler, but in other ways it is more powerful. Best of all the user can use both of these interfaces at the same time to take advantage or both of their strengths.  When the user can break up the user's work into segments that do not depend on previous execution, the ``Task`` interface is ideal.  But it also has more power and flexibility, allowing the user to guide the distribution of jobs, without having to assign Tasks to engines explicitly.
+
+Starting the IPython controller and engines
+===========================================
+
+To follow along with this tutorial, the user will need to start the IPython
+controller and four IPython engines. The simplest way of doing this is to
+use the ``ipcluster`` command::
+
+	$ ipcluster -n 4
+	
+For more detailed information about starting the controller and engines, see our :ref:`introduction <ip1par>` to using IPython for parallel computing.
+
+The magic here is that this single controller and set of engines is running both the MultiEngine and ``Task`` interfaces simultaneously.
+
+QuickStart Task Farming
+=======================
+
+First, a quick example of how to start running the most basic Tasks.
+The first step is to import the IPython ``client`` module and then create a ``TaskClient`` instance::
+    
+    In [1]: from IPython.kernel import client
+    
+    In [2]: tc = client.TaskClient()
+
+Then the user wrap the commands the user want to run in Tasks::
+
+    In [3]: tasklist = []
+    In [4]: for n in range(1000):
+        ...     tasklist.append(client.Task("a = %i"%n, pull="a"))
+
+The first argument of the ``Task`` constructor is a string, the command to be executed.  The most important optional keyword argument is ``pull``, which can be a string or list of strings, and it specifies the variable names to be saved as results of the ``Task``.
+
+Next, the user need to submit the Tasks to the ``TaskController`` with the ``TaskClient``::
+
+    In [5]: taskids = [ tc.run(t) for t in tasklist ]
+
+This will give the user a list of the TaskIDs used by the controller to keep track of the Tasks and their results.  Now at some point the user are going to want to get those results back.  The ``barrier`` method allows the user to wait for the Tasks to finish running::
+
+    In [6]: tc.barrier(taskids)
+
+This command will block until all the Tasks in ``taskids`` have finished.  Now, the user probably want to look at the user's results::
+
+    In [7]: task_results = [ tc.get_task_result(taskid) for taskid in taskids ]
+
+Now the user have a list of ``TaskResult`` objects, which have the actual result as a dictionary, but also keep track of some useful metadata about the ``Task``::
+
+    In [8]: tr = ``Task``_results[73]
+    
+    In [9]: tr
+    Out[9]: ``TaskResult``[ID:73]:{'a':73}
+    
+    In [10]: tr.engineid
+    Out[10]: 1
+    
+    In [11]: tr.submitted, tr.completed, tr.duration
+    Out[11]: ("2008/03/08 03:41:42", "2008/03/08 03:41:44", 2.12345)
+
+The actual results are stored in a dictionary, ``tr.results``, and a namespace object ``tr.ns`` which accesses the result keys by attribute::
+
+    In [12]: tr.results['a']
+    Out[12]: 73
+    
+    In [13]: tr.ns.a
+    Out[13]: 73
+
+That should cover the basics of running simple Tasks.  There are several more powerful things the user can do with Tasks covered later.  The most useful probably being using a ``MutiEngineClient`` interface to initialize all the engines with the import dependencies necessary to run the user's Tasks.
+
+There are many options for running and managing Tasks. The best way to learn further about the ``Task`` interface is to study the examples in ``docs/examples``. If the user do so and learn a lots about this interface, we encourage the user to expand this documentation about the ``Task`` system.
+
+Overview of the Task System
+===========================
+
+The user's view of the ``Task`` system has three basic objects:  The ``TaskClient``, the ``Task``, and the ``TaskResult``.  The names of these three objects well indicate their role.
+
+The ``TaskClient`` is the user's ``Task`` farming connection to the IPython cluster.  Unlike the ``MultiEngineClient``, the ``TaskControler`` handles all the scheduling and distribution of work, so the ``TaskClient`` has no notion of engines, it just submits Tasks and requests their results.  The Tasks are described as ``Task`` objects, and their results are wrapped in ``TaskResult`` objects.  Thus, there are very few necessary methods for the user to manage.
+
+Inside the task system is a Scheduler object, which assigns tasks to workers.  The default scheduler is a simple FIFO queue.  Subclassing the Scheduler should be easy, just implementing your own priority system.
+
+The TaskClient
+==============
+
+The ``TaskClient`` is the object the user use to connect to the ``Controller`` that is managing the user's Tasks.  It is the analog of the ``MultiEngineClient`` for the standard IPython multiplexing interface.  As with all client interfaces, the first step is to import the IPython Client Module::
+
+    In [1]: from IPython.kernel import client
+
+Just as with the ``MultiEngineClient``, the user create the ``TaskClient`` with a tuple, containing the ip-address and port of the ``Controller``.  the ``client`` module conveniently has the default address of the ``Task`` interface of the controller.  Creating a default ``TaskClient`` object would be done with this::
+
+    In [2]: tc = client.TaskClient(client.default_task_address)
+
+or, if the user want to specify a non default location of the ``Controller``, the user can specify explicitly::
+
+    In [3]: tc = client.TaskClient(("192.168.1.1", 10113))
+
+As discussed earlier, the ``TaskClient`` only has a few basic methods.
+
+ * ``tc.run(task)``
+    ``run`` is the method by which the user submits Tasks.  It takes exactly one argument, a ``Task`` object.  All the advanced control of ``Task`` behavior is handled by properties of the ``Task`` object, rather than the submission command, so they will be discussed later in the `Task`_ section.  ``run`` returns an integer, the ``Task``ID by which the ``Task`` and its results can be tracked and retrieved::
+    
+        In [4]: ``Task``ID = tc.run(``Task``)
+    
+ * ``tc.get_task_result(taskid, block=``False``)``
+    ``get_task_result`` is the method by which results are retrieved.  It takes a single integer argument, the ``Task``ID`` of the result the user wish to retrieve.  ``get_task_result`` also takes a keyword argument ``block``.  ``block`` specifies whether the user actually want to wait for the result.  If ``block`` is false, as it is by default, ``get_task_result`` will return immediately.  If the ``Task`` has completed, it will return the ``TaskResult`` object for that ``Task``.  But if the ``Task`` has not completed, it will return ``None``.  If the user specify ``block=``True``, then ``get_task_result`` will wait for the ``Task`` to complete, and always return the ``TaskResult`` for the requested ``Task``.
+ * ``tc.barrier(taskid(s))``
+    ``barrier`` is a synchronization method.  It takes exactly one argument, a ``Task``ID or list of taskIDs.  ``barrier`` will block until all the specified Tasks have completed.  In practice, a barrier is often called between the ``Task`` submission section of the code and the result gathering section::
+        
+        In [5]: taskIDs = [ tc.run(``Task``) for ``Task`` in myTasks ]
+        
+        In [6]: tc.get_task_result(taskIDs[-1]) is None
+        Out[6]: ``True``
+        
+        In [7]: tc.barrier(``Task``ID)
+        
+        In [8]: results = [ tc.get_task_result(tid) for tid in taskIDs ]
+        
+ * ``tc.queue_status(verbose=``False``)``
+    ``queue_status`` is a method for querying the state of the ``TaskControler``.  ``queue_status`` returns a dict of the form::
+        
+        {'scheduled': Tasks that have been submitted but yet run
+         'pending'  : Tasks that are currently running
+         'succeeded': Tasks that have completed successfully
+         'failed'   : Tasks that have finished with a failure
+        }
+    
+    if @verbose is not specified (or is ``False``), then the values of the dict are integers - the number of Tasks in each state.  if @verbose is ``True``, then each element in the dict is a list of the taskIDs in that state::
+    
+        In [8]: tc.queue_status()
+        Out[8]: {'scheduled': 4,
+                'pending'  : 2,
+                'succeeded': 5,
+                'failed'   : 1
+                }
+        
+        In [9]: tc.queue_status(verbose=True)
+        Out[9]: {'scheduled': [8,9,10,11],
+                'pending'  : [6,7],
+                'succeeded': [0,1,2,4,5],
+                'failed'   : [3]
+                }
+        
+ * ``tc.abort(taskid)``
+    ``abort`` allows the user to abort Tasks that have already been submitted.  ``abort`` will always return immediately.  If the ``Task`` has completed, ``abort`` will raise an ``IndexError ``Task`` Already Completed``.  An obvious case for ``abort`` would be where the user submits a long-running ``Task`` with a number of retries (see ``Task``_ section for how to specify retries) in an interactive session, but realizes there has been a typo.  The user can then abort the ``Task``, preventing certain failures from cluttering up the queue.  It can also be used for parallel search-type problems, where only one ``Task`` will give the solution, so once the user find the solution, the user would want to abort all remaining Tasks to prevent wasted work.
+ * ``tc.spin()``
+    ``spin`` simply triggers the scheduler in the ``TaskControler``.  Under most normal circumstances, this will do nothing.  The primary known usage case involves the ``Task`` dependency (see `Dependencies`_).  The dependency is a function of an Engine's ``properties``, but changing the ``properties`` via the ``MutliEngineClient`` does not trigger a reschedule event.  The main example case for this requires the following event sequence:
+     * ``engine`` is available, ``Task`` is submitted, but ``engine`` does not have ``Task``'s dependencies.
+     * ``engine`` gets necessary dependencies while no new Tasks are submitted or completed.
+     * now ``engine`` can run ``Task``, but a ``Task`` event is required for the ``TaskControler`` to try scheduling ``Task`` again.
+     
+    ``spin`` is just an empty ping method to ensure that the Controller has scheduled all available Tasks, and should not be needed under most normal circumstances.
+
+That covers the ``TaskClient``, a simple interface to the cluster.  With this, the user can submit jobs (and abort if necessary), request their results, synchronize on arbitrary subsets of jobs.
+
+.. _task: The Task Object
+
+The Task Object
+===============
+
+The ``Task`` is the basic object for describing a job.  It can be used in a very simple manner, where the user just specifies a command string to be executed as the ``Task``.  The usage of this first argument is exactly the same as the ``execute`` method of the ``MultiEngine`` (in fact, ``execute`` is called to run the code)::
+
+    In [1]: t = client.Task("a = str(id)")
+    
+This ``Task`` would run, and store the string representation of the ``id`` element in ``a`` in each worker's namespace, but it is fairly useless because the user does not know anything about the state of the ``worker`` on which it ran at the time of retrieving results.  It is important that each ``Task`` not expect the state of the ``worker`` to persist after the ``Task`` is completed.  
+There are many different situations for using ``Task`` Farming, and the ``Task`` object has many attributes for use in customizing the ``Task`` behavior.  All of a ``Task``'s attributes may be specified in the constructor, through keyword arguments, or after ``Task`` construction through attribute assignment.
+
+Data Attributes
+***************
+It is likely that the user may want to move data around before or after executing the ``Task``.  We provide methods of sending data to initialize the worker's namespace, and specifying what data to bring back as the ``Task``'s results.
+
+ * pull = []
+    The obvious case is as above, where ``t`` would execute and store the result of ``myfunc`` in ``a``, it is likely that the user would want to bring ``a`` back to their namespace.  This is done through the ``pull`` attribute.  ``pull`` can be a string or list of strings, and it specifies the names of variables to be retrieved.  The ``TaskResult`` object retrieved by ``get_task_result`` will have a dictionary of keys and values, and the ``Task``'s ``pull`` attribute determines what goes into it::
+        
+        In [2]: t = client.Task("a = str(id)", pull = "a")
+        
+        In [3]: t = client.Task("a = str(id)", pull = ["a", "id"])
+        
+ * push = {}
+    A user might also want to initialize some data into the namespace before the code part of the ``Task`` is run.  Enter ``push``.  ``push`` is a dictionary of key/value pairs to be loaded from the user's namespace into the worker's immediately before execution::
+
+        In [4]: t = client.Task("a = f(submitted)", push=dict(submitted=time.time()), pull="a")
+
+push and pull result directly in calling an ``engine``'s ``push`` and ``pull`` methods before and after ``Task`` execution respectively, and thus their api is the same.
+
+Namespace Cleaning
+******************
+When a user is running a large number of Tasks, it is likely that the namespace of the worker's could become cluttered.  Some Tasks might be sensitive to clutter, while others might be known to cause namespace pollution.  For these reasons, Tasks have two boolean attributes for cleaning up the namespace.
+
+ * ``clear_after``
+    if clear_after is specified ``True``, the worker on which the ``Task`` was run will be reset (via ``engine.reset``) upon completion of the ``Task``.  This can be useful for both Tasks that produce clutter or Tasks whose intermediate data one might wish to be kept private::
+    
+        In [5]: t = client.Task("a = range(1e10)", pull = "a",clear_after=True)
+        
+    
+ * ``clear_before``
+    as one might guess, clear_before is identical to ``clear_after``, but it takes place before the ``Task`` is run.  This ensures that the ``Task`` runs on a fresh worker::
+
+        In [6]: t = client.Task("a = globals()", pull = "a",clear_before=True)
+
+Of course, a user can both at the same time, ensuring that all workers are clear except when they are currently running a job.  Both of these default to ``False``.
+
+Fault Tolerance
+***************
+It is possible that Tasks might fail, and there are a variety of reasons this could happen.  One might be that the worker it was running on disconnected, and there was nothing wrong with the ``Task`` itself.  With the fault tolerance attributes of the ``Task``, the user can specify how many times to resubmit the ``Task``, and what to do if it never succeeds.
+
+ * ``retries``
+    ``retries`` is an integer, specifying the number of times a ``Task`` is to be retried.  It defaults to zero.  It is often a good idea for this number to be 1 or 2, to protect the ``Task`` from disconnecting engines, but not a large number.  If a ``Task`` is failing 100 times, there is probably something wrong with the ``Task``.  The canonical bad example:
+    
+        In [7]: t = client.Task("os.kill(os.getpid(), 9)", retries=99)
+        
+    This would actually take down 100 workers.
+    
+ * ``recovery_task``
+    ``recovery_task`` is another ``Task`` object, to be run in the event of the original ``Task`` still failing after running out of retries.  Since ``recovery_task`` is another ``Task`` object, it can have its own ``recovery_task``.  The chain of Tasks is limitless, except loops are not allowed (that would be bad!).
+
+Dependencies
+************
+Dependencies are the most powerful part of the ``Task`` farming system, because it allows the user to do some classification of the workers, and guide the ``Task`` distribution without meddling with the controller directly.  It makes use of two objects - the ``Task``'s ``depend`` attribute, and the engine's ``properties``.  See the `MultiEngine`_ reference for how to use engine properties.  The engine properties api exists for extending IPython, allowing conditional execution and new controllers that make decisions based on properties of its engines.  Currently the ``Task`` dependency is the only internal use of the properties api.
+
+.. _MultiEngine: ./parallel_multiengine
+
+The ``depend`` attribute of a ``Task`` must be a function of exactly one argument, the worker's properties dictionary, and it should return ``True`` if the ``Task`` should be allowed to run on the worker and ``False`` if not.  The usage in the controller is fault tolerant, so exceptions raised by ``Task.depend`` will be ignored and functionally equivalent to always returning ``False``.  Tasks`` with invalid ``depend`` functions will never be assigned to a worker::
+
+    In [8]: def dep(properties):
+        ...     return properties["RAM"] > 2**32 # have at least 4GB
+    In [9]: t = client.Task("a = bigfunc()", depend=dep)
+    
+It is important to note that assignment of values to the properties dict is done entirely by the user, either locally (in the engine) using the EngineAPI, or remotely, through the ``MultiEngineClient``'s get/set_properties methods.
+
+
+
+
+
+    

docs/source/install/install.txt

@@ -0,0 +1,191 b''
+Overview
+========
+
+This document describes the steps required to install IPython.  IPython is organized into a number of subpackages, each of which has its own dependencies.  All of the subpackages come with IPython, so you don't need to download and install them separately.  However, to use a given subpackage, you will need to install all of its dependencies.  
+
+
+Please let us know if you have problems installing IPython or any of its
+dependencies. IPython requires Python version 2.4 or greater.  We have not tested
+IPython with the upcoming 2.6 or 3.0 versions.
+
+.. warning::
+
+	IPython will not work with Python 2.3 or below.
+
+Some of the installation approaches use the :mod:`setuptools` package and its :command:`easy_install` command line program.  In many scenarios, this provides the most simple method of installing IPython and its dependencies.  It is not required though.  More information about :mod:`setuptools` can be found on its website.
+
+More general information about installing Python packages can be found in Python's documentation at http://www.python.org/doc/.
+
+Installing IPython itself
+=========================
+
+Given a properly built Python, the basic interactive IPython shell will work with no external dependencies.  However, some Python distributions (particularly on Windows and OS X), don't come with a working :mod:`readline` module.  The IPython shell will work without :mod:`readline`, but will lack many features that users depend on, such as tab completion and command line editing.  See below for details of how to make sure you have a working :mod:`readline`.
+
+Installation using easy_install
+-------------------------------
+
+If you have :mod:`setuptools` installed, the easiest way of getting IPython is to simple use :command:`easy_install`::
+
+	$ easy_install IPython
+
+That's it.
+
+Installation from source
+------------------------
+
+If you don't want to use :command:`easy_install`, or don't have it installed, just grab the latest stable build of IPython from `here <http://ipython.scipy.org/dist/>`_.  Then do the following::
+
+	$ tar -xzf ipython.tar.gz
+	$ cd ipython
+	$ python setup.py install
+
+If you are installing to a location (like ``/usr/local``) that requires higher permissions, you may need to run the last command with :command:`sudo`.
+
+Windows
+-------
+
+There are a few caveats for Windows users.  The main issue is that a basic ``python setup.py install`` approach won't create ``.bat`` file or Start Menu shortcuts, which most users want.  To get an installation with these, there are two choices:
+
+1.  Install using :command:`easy_install`.
+
+2.  Install using our binary ``.exe`` Windows installer, which can be found at `here <http://ipython.scipy.org/dist/>`_
+
+3.  Install from source, but using :mod:`setuptools` (``python setupegg.py install``).
+
+Installing the development version
+----------------------------------
+
+It is also possible to install the development version of IPython from our `Bazaar <http://bazaar-vcs.org/>`_ source code
+repository.  To do this you will need to have Bazaar installed on your system.  Then just do::
+
+	$ bzr branch lp:ipython
+	$ cd ipython
+	$ python setup.py install
+
+Again, this last step on Windows won't create ``.bat`` files or Start Menu shortcuts, so you will have to use one of the other approaches listed above.
+
+Some users want to be able to follow the development branch as it changes.  If you have :mod:`setuptools` installed, this is easy. Simply replace the last step by::
+
+	$ python setupegg.py develop
+
+This creates links in the right places and installs the command line script to the appropriate places.  Then, if you want to update your IPython at any time, just do::
+
+	$ bzr pull
+
+Basic optional dependencies
+===========================
+
+There are a number of basic optional dependencies that most users will want to get.  These are:
+
+* readline (for command line editing, tab completion, etc.)
+* nose (to run the IPython test suite)
+* pexpect (to use things like irunner)
+
+If you are comfortable installing these things yourself, have at it, otherwise read on for more details.
+
+readline
+--------
+
+In principle, all Python distributions should come with a working :mod:`readline` module.  But, reality is not quite that simple.  There are two common situations where you won't have a working :mod:`readline` module:
+
+* If you are using the built-in Python on Mac OS X.
+
+* If you are running Windows, which doesn't have a :mod:`readline` module.
+
+On OS X, the built-in Python doesn't not have :mod:`readline` because of license issues.  Starting with OS X 10.5 (Leopard), Apple's built-in Python has a BSD-licensed not-quite-compatible readline replacement. As of IPython 0.9, many of the issues related to the differences between readline and libedit have been resolved.  For many users, libedit may be sufficient.
+
+Most users on OS X will want to get the full :mod:`readline` module.  To get a working :mod:`readline` module, just do (with :mod:`setuptools` installed)::
+
+	$ easy_install readline
+
+.. note:
+
+    Other Python distributions on OS X (such as fink, MacPorts and the 
+    official python.org binaries) already have readline installed so 
+    you don't have to do this step.
+
+If needed, the readline egg can be build and installed from source (see the wiki page at http://ipython.scipy.org/moin/InstallationOSXLeopard).
+
+On Windows, you will need the PyReadline module.  PyReadline is a separate, Windows only implementation of readline that uses native Windows calls through :mod:`ctypes`.  The easiest way of installing PyReadline is you use the binary installer available `here <http://ipython.scipy.org/dist/>`_.  The :mod:`ctypes` module, which comes with Python 2.5 and greater, is required by PyReadline.  It is available for Python 2.4 at http://python.net/crew/theller/ctypes.
+
+nose
+----
+
+To run the IPython test suite you will need the :mod:`nose` package.  Nose provides a great way of sniffing out and running all of the IPython tests.  The simplest way of getting nose, is to use :command:`easy_install`::
+
+	$ easy_install nose
+
+Another way of getting this is to do::
+
+    $ easy_install IPython[test]
+
+For more installation options, see the `nose website <http://somethingaboutorange.com/mrl/projects/nose/>`_.  Once you have nose installed, you can run IPython's test suite using the iptest command::
+
+	$ iptest
+
+
+pexpect
+-------
+
+The `pexpect <http://www.noah.org/wiki/Pexpect>`_ package is used in IPython's :command:`irunner` script.  On Unix platforms (including OS X), just do::
+
+	$ easy_install pexpect
+
+Windows users are out of luck as pexpect does not run there.
+
+Dependencies for IPython.kernel (parallel computing)
+====================================================
+
+The IPython kernel provides a nice architecture for parallel computing.  The main focus of this architecture is on interactive parallel computing.  These features require a number of additional packages:
+
+* zope.interface (yep, we use interfaces)
+* Twisted (asynchronous networking framework)
+* Foolscap (a nice, secure network protocol)
+* pyOpenSSL (security for network connections)
+
+On a Unix style platform (including OS X), if you want to use :mod:`setuptools`, you can just do::
+
+    $ easy_install IPython[kernel]    # the first three
+    $ easy_install IPython[security]  # pyOpenSSL
+
+zope.interface and Twisted
+--------------------------
+
+On Unix style platforms (including OS X), the simplest way of getting the these is to use :command:`easy_install`::
+
+	$ easy_install zope.interface
+	$ easy_install Twisted
+
+Of course, you can also download the source tarballs from the `Twisted website <twistedmatrix.org>`_ and the `zope.interface page at PyPI <http://pypi.python.org/pypi/zope.interface>`_ and do the usual ``python setup.py install`` if you prefer.
+
+Windows is a bit different.  For zope.interface and Twisted, simply get the latest binary ``.exe`` installer from the Twisted website.  This installer includes both zope.interface and Twisted and should just work.
+
+Foolscap
+--------
+
+Foolscap uses Twisted to provide a very nice secure RPC protocol that we use to implement our parallel computing features.
+
+On all platforms a simple::
+
+    $ easy_install foolscap
+
+should work.  You can also download the source tarballs from the `Foolscap website <http://foolscap.lothar.com/trac>`_ and do ``python setup.py install`` if you prefer.
+
+pyOpenSSL
+---------
+
+IPython requires an older version of pyOpenSSL (0.6 rather than the current 0.7).  There are a couple of options for getting this:
+
+1.  Most Linux distributions have packages for pyOpenSSL.
+2.  The built-in Python 2.5 on OS X 10.5 already has it installed.
+3.  There are source tarballs on the pyOpenSSL website.  On Unix-like
+    platforms, these can be built using ``python seutp.py install``.
+4.  There is also a binary ``.exe`` Windows installer on the `pyOpenSSL website <http://pyopenssl.sourceforge.net/>`_.
+
+Dependencies for IPython.frontend (the IPython GUI)
+===================================================
+
+wxPython
+--------
+
+Starting with IPython 0.9, IPython has a new IPython.frontend package that has a nice wxPython based IPython GUI.  As you would expect, this GUI requires wxPython.  Most Linux distributions have wxPython packages available and the built-in Python on OS X comes with wxPython preinstalled.  For Windows, a binary installer is available on the `wxPython website <http://www.wxpython.org/>`_. 
  No newline at end of file

docs/sphinxext/inheritance_diagram.py

@@ -0,0 +1,423 b''
+"""
+Defines a docutils directive for inserting inheritance diagrams.
+
+Provide the directive with one or more classes or modules (separated
+by whitespace).  For modules, all of the classes in that module will
+be used.
+
+Example::
+
+   Given the following classes:
+
+   class A: pass
+   class B(A): pass
+   class C(A): pass
+   class D(B, C): pass
+   class E(B): pass
+
+   .. inheritance-diagram: D E
+
+   Produces a graph like the following:
+
+               A
+              / \
+             B   C
+            / \ /
+           E   D
+
+The graph is inserted as a PNG+image map into HTML and a PDF in
+LaTeX.
+"""
+
+import inspect
+import os
+import re
+import subprocess
+try:
+    from hashlib import md5
+except ImportError:
+    from md5 import md5
+
+from docutils.nodes import Body, Element
+from docutils.writers.html4css1 import HTMLTranslator
+from sphinx.latexwriter import LaTeXTranslator
+from docutils.parsers.rst import directives
+from sphinx.roles import xfileref_role
+
+class DotException(Exception):
+    pass
+
+class InheritanceGraph(object):
+    """
+    Given a list of classes, determines the set of classes that
+    they inherit from all the way to the root "object", and then
+    is able to generate a graphviz dot graph from them.
+    """
+    def __init__(self, class_names, show_builtins=False):
+        """
+        *class_names* is a list of child classes to show bases from.
+
+        If *show_builtins* is True, then Python builtins will be shown
+        in the graph.
+        """
+        self.class_names = class_names
+        self.classes = self._import_classes(class_names)
+        self.all_classes = self._all_classes(self.classes)
+        if len(self.all_classes) == 0:
+            raise ValueError("No classes found for inheritance diagram")
+        self.show_builtins = show_builtins
+
+    py_sig_re = re.compile(r'''^([\w.]*\.)?    # class names
+                           (\w+)  \s* $        # optionally arguments
+                           ''', re.VERBOSE)
+
+    def _import_class_or_module(self, name):
+        """
+        Import a class using its fully-qualified *name*.
+        """
+        try:
+            path, base = self.py_sig_re.match(name).groups()
+        except:
+            raise ValueError(
+                "Invalid class or module '%s' specified for inheritance diagram" % name)
+        fullname = (path or '') + base
+        path = (path and path.rstrip('.'))
+        if not path:
+            path = base
+        if not path:
+            raise ValueError(
+                "Invalid class or module '%s' specified for inheritance diagram" % name)
+        try:
+            module = __import__(path, None, None, [])
+        except ImportError:
+            raise ValueError(
+                "Could not import class or module '%s' specified for inheritance diagram" % name)
+
+        try:
+            todoc = module
+            for comp in fullname.split('.')[1:]:
+                todoc = getattr(todoc, comp)
+        except AttributeError:
+            raise ValueError(
+                "Could not find class or module '%s' specified for inheritance diagram" % name)
+
+        # If a class, just return it
+        if inspect.isclass(todoc):
+            return [todoc]
+        elif inspect.ismodule(todoc):
+            classes = []
+            for cls in todoc.__dict__.values():
+                if inspect.isclass(cls) and cls.__module__ == todoc.__name__:
+                    classes.append(cls)
+            return classes
+        raise ValueError(
+            "'%s' does not resolve to a class or module" % name)
+
+    def _import_classes(self, class_names):
+        """
+        Import a list of classes.
+        """
+        classes = []
+        for name in class_names:
+            classes.extend(self._import_class_or_module(name))
+        return classes
+
+    def _all_classes(self, classes):
+        """
+        Return a list of all classes that are ancestors of *classes*.
+        """
+        all_classes = {}
+
+        def recurse(cls):
+            all_classes[cls] = None
+            for c in cls.__bases__:
+                if c not in all_classes:
+                    recurse(c)
+
+        for cls in classes:
+            recurse(cls)
+
+        return all_classes.keys()
+
+    def class_name(self, cls, parts=0):
+        """
+        Given a class object, return a fully-qualified name.  This
+        works for things I've tested in matplotlib so far, but may not
+        be completely general.
+        """
+        module = cls.__module__
+        if module == '__builtin__':
+            fullname = cls.__name__
+        else:
+            fullname = "%s.%s" % (module, cls.__name__)
+        if parts == 0:
+            return fullname
+        name_parts = fullname.split('.')
+        return '.'.join(name_parts[-parts:])
+
+    def get_all_class_names(self):
+        """
+        Get all of the class names involved in the graph.
+        """
+        return [self.class_name(x) for x in self.all_classes]
+
+    # These are the default options for graphviz
+    default_graph_options = {
+        "rankdir": "LR",
+        "size": '"8.0, 12.0"'
+        }
+    default_node_options = {
+        "shape": "box",
+        "fontsize": 10,
+        "height": 0.25,
+        "fontname": "Vera Sans, DejaVu Sans, Liberation Sans, Arial, Helvetica, sans",
+        "style": '"setlinewidth(0.5)"'
+        }
+    default_edge_options = {
+        "arrowsize": 0.5,
+        "style": '"setlinewidth(0.5)"'
+        }
+
+    def _format_node_options(self, options):
+        return ','.join(["%s=%s" % x for x in options.items()])
+    def _format_graph_options(self, options):
+        return ''.join(["%s=%s;\n" % x for x in options.items()])
+
+    def generate_dot(self, fd, name, parts=0, urls={},
+                     graph_options={}, node_options={},
+                     edge_options={}):
+        """
+        Generate a graphviz dot graph from the classes that
+        were passed in to __init__.
+
+        *fd* is a Python file-like object to write to.
+
+        *name* is the name of the graph
+
+        *urls* is a dictionary mapping class names to http urls
+
+        *graph_options*, *node_options*, *edge_options* are
+        dictionaries containing key/value pairs to pass on as graphviz
+        properties.
+        """
+        g_options = self.default_graph_options.copy()
+        g_options.update(graph_options)
+        n_options = self.default_node_options.copy()
+        n_options.update(node_options)
+        e_options = self.default_edge_options.copy()
+        e_options.update(edge_options)
+
+        fd.write('digraph %s {\n' % name)
+        fd.write(self._format_graph_options(g_options))
+
+        for cls in self.all_classes:
+            if not self.show_builtins and cls in __builtins__.values():
+                continue
+
+            name = self.class_name(cls, parts)
+
+            # Write the node
+            this_node_options = n_options.copy()
+            url = urls.get(self.class_name(cls))
+            if url is not None:
+                this_node_options['URL'] = '"%s"' % url
+            fd.write('  "%s" [%s];\n' %
+                     (name, self._format_node_options(this_node_options)))
+
+            # Write the edges
+            for base in cls.__bases__:
+                if not self.show_builtins and base in __builtins__.values():
+                    continue
+
+                base_name = self.class_name(base, parts)
+                fd.write('  "%s" -> "%s" [%s];\n' %
+                         (base_name, name,
+                          self._format_node_options(e_options)))
+        fd.write('}\n')
+
+    def run_dot(self, args, name, parts=0, urls={},
+                graph_options={}, node_options={}, edge_options={}):
+        """
+        Run graphviz 'dot' over this graph, returning whatever 'dot'
+        writes to stdout.
+
+        *args* will be passed along as commandline arguments.
+
+        *name* is the name of the graph
+
+        *urls* is a dictionary mapping class names to http urls
+
+        Raises DotException for any of the many os and
+        installation-related errors that may occur.
+        """
+        try:
+            dot = subprocess.Popen(['dot'] + list(args),
+                                   stdin=subprocess.PIPE, stdout=subprocess.PIPE,
+                                   close_fds=True)
+        except OSError:
+            raise DotException("Could not execute 'dot'.  Are you sure you have 'graphviz' installed?")
+        except ValueError:
+            raise DotException("'dot' called with invalid arguments")
+        except:
+            raise DotException("Unexpected error calling 'dot'")
+
+        self.generate_dot(dot.stdin, name, parts, urls, graph_options,
+                          node_options, edge_options)
+        dot.stdin.close()
+        result = dot.stdout.read()
+        returncode = dot.wait()
+        if returncode != 0:
+            raise DotException("'dot' returned the errorcode %d" % returncode)
+        return result
+
+class inheritance_diagram(Body, Element):
+    """
+    A docutils node to use as a placeholder for the inheritance
+    diagram.
+    """
+    pass
+
+def inheritance_diagram_directive_run(class_names, options, state):
+    """
+    Run when the inheritance_diagram directive is first encountered.
+    """
+    node = inheritance_diagram()
+
+    # Create a graph starting with the list of classes
+    graph = InheritanceGraph(class_names)
+
+    # Create xref nodes for each target of the graph's image map and
+    # add them to the doc tree so that Sphinx can resolve the
+    # references to real URLs later.  These nodes will eventually be
+    # removed from the doctree after we're done with them.
+    for name in graph.get_all_class_names():
+        refnodes, x = xfileref_role(
+            'class', ':class:`%s`' % name, name, 0, state)
+        node.extend(refnodes)
+    # Store the graph object so we can use it to generate the
+    # dot file later
+    node['graph'] = graph
+    # Store the original content for use as a hash
+    node['parts'] = options.get('parts', 0)
+    node['content'] = " ".join(class_names)
+    return [node]
+
+def get_graph_hash(node):
+    return md5(node['content'] + str(node['parts'])).hexdigest()[-10:]
+
+def html_output_graph(self, node):
+    """
+    Output the graph for HTML.  This will insert a PNG with clickable
+    image map.
+    """
+    graph = node['graph']
+    parts = node['parts']
+
+    graph_hash = get_graph_hash(node)
+    name = "inheritance%s" % graph_hash
+    png_path = os.path.join('_static', name + ".png")
+
+    path = '_static'
+    source = self.document.attributes['source']
+    count = source.split('/doc/')[-1].count('/')
+    for i in range(count):
+        if os.path.exists(path): break
+        path = '../'+path
+    path = '../'+path #specifically added for matplotlib
+
+    # Create a mapping from fully-qualified class names to URLs.
+    urls = {}
+    for child in node:
+        if child.get('refuri') is not None:
+            urls[child['reftitle']] = child.get('refuri')
+        elif child.get('refid') is not None:
+            urls[child['reftitle']] = '#' + child.get('refid')
+
+    # These arguments to dot will save a PNG file to disk and write
+    # an HTML image map to stdout.
+    image_map = graph.run_dot(['-Tpng', '-o%s' % png_path, '-Tcmapx'],
+                              name, parts, urls)
+    return ('<img src="%s/%s.png" usemap="#%s" class="inheritance"/>%s' %
+            (path, name, name, image_map))
+
+def latex_output_graph(self, node):
+    """
+    Output the graph for LaTeX.  This will insert a PDF.
+    """
+    graph = node['graph']
+    parts = node['parts']
+
+    graph_hash = get_graph_hash(node)
+    name = "inheritance%s" % graph_hash
+    pdf_path = os.path.join('_static', name + ".pdf")
+
+    graph.run_dot(['-Tpdf', '-o%s' % pdf_path],
+                  name, parts, graph_options={'size': '"6.0,6.0"'})
+    return '\\includegraphics{../../%s}' % pdf_path
+
+def visit_inheritance_diagram(inner_func):
+    """
+    This is just a wrapper around html/latex_output_graph to make it
+    easier to handle errors and insert warnings.
+    """
+    def visitor(self, node):
+        try:
+            content = inner_func(self, node)
+        except DotException, e:
+            # Insert the exception as a warning in the document
+            warning = self.document.reporter.warning(str(e), line=node.line)
+            warning.parent = node
+            node.children = [warning]
+        else:
+            source = self.document.attributes['source']
+            self.body.append(content)
+            node.children = []
+    return visitor
+
+def do_nothing(self, node):
+    pass
+
+options_spec = {
+    'parts': directives.nonnegative_int
+    }
+
+# Deal with the old and new way of registering directives
+try:
+    from docutils.parsers.rst import Directive
+except ImportError:
+    from docutils.parsers.rst.directives import _directives
+    def inheritance_diagram_directive(name, arguments, options, content, lineno,
+                                      content_offset, block_text, state,
+                                      state_machine):
+        return inheritance_diagram_directive_run(arguments, options, state)
+    inheritance_diagram_directive.__doc__ = __doc__
+    inheritance_diagram_directive.arguments = (1, 100, 0)
+    inheritance_diagram_directive.options = options_spec
+    inheritance_diagram_directive.content = 0
+    _directives['inheritance-diagram'] = inheritance_diagram_directive
+else:
+    class inheritance_diagram_directive(Directive):
+        has_content = False
+        required_arguments = 1
+        optional_arguments = 100
+        final_argument_whitespace = False
+        option_spec = options_spec
+
+        def run(self):
+            return inheritance_diagram_directive_run(
+                self.arguments, self.options, self.state)
+    inheritance_diagram_directive.__doc__ = __doc__
+
+    directives.register_directive('inheritance-diagram',
+                                  inheritance_diagram_directive)
+
+def setup(app):
+    app.add_node(inheritance_diagram)
+
+    HTMLTranslator.visit_inheritance_diagram = \
+        visit_inheritance_diagram(html_output_graph)
+    HTMLTranslator.depart_inheritance_diagram = do_nothing
+
+    LaTeXTranslator.visit_inheritance_diagram = \
+        visit_inheritance_diagram(latex_output_graph)
+    LaTeXTranslator.depart_inheritance_diagram = do_nothing

docs/sphinxext/ipython_console_highlighting.py

@@ -0,0 +1,75 b''
+from pygments.lexer import Lexer, do_insertions
+from pygments.lexers.agile import PythonConsoleLexer, PythonLexer, \
+    PythonTracebackLexer
+from pygments.token import Comment, Generic
+from sphinx import highlighting
+import re
+
+line_re = re.compile('.*?\n')
+
+class IPythonConsoleLexer(Lexer):
+    """
+    For IPython console output or doctests, such as:
+
+    Tracebacks are not currently supported.
+
+    .. sourcecode:: ipython
+
+      In [1]: a = 'foo'
+
+      In [2]: a
+      Out[2]: 'foo'
+
+      In [3]: print a
+      foo
+
+      In [4]: 1 / 0
+    """
+    name = 'IPython console session'
+    aliases = ['ipython']
+    mimetypes = ['text/x-ipython-console']
+    input_prompt = re.compile("(In \[[0-9]+\]: )|(   \.\.\.+:)")
+    output_prompt = re.compile("(Out\[[0-9]+\]: )|(   \.\.\.+:)")
+    continue_prompt = re.compile("   \.\.\.+:")
+    tb_start = re.compile("\-+")
+
+    def get_tokens_unprocessed(self, text):
+        pylexer = PythonLexer(**self.options)
+        tblexer = PythonTracebackLexer(**self.options)
+
+        curcode = ''
+        insertions = []
+        for match in line_re.finditer(text):
+            line = match.group()
+            input_prompt = self.input_prompt.match(line)
+            continue_prompt = self.continue_prompt.match(line.rstrip())
+            output_prompt = self.output_prompt.match(line)
+            if line.startswith("#"):
+                insertions.append((len(curcode),
+                                   [(0, Comment, line)]))
+            elif input_prompt is not None:
+                insertions.append((len(curcode),
+                                   [(0, Generic.Prompt, input_prompt.group())]))
+                curcode += line[input_prompt.end():]
+            elif continue_prompt is not None:
+                insertions.append((len(curcode),
+                                   [(0, Generic.Prompt, continue_prompt.group())]))
+                curcode += line[continue_prompt.end():]
+            elif output_prompt is not None:
+                insertions.append((len(curcode),
+                                   [(0, Generic.Output, output_prompt.group())]))
+                curcode += line[output_prompt.end():]
+            else:
+                if curcode:
+                    for item in do_insertions(insertions,
+                                              pylexer.get_tokens_unprocessed(curcode)):
+                        yield item
+                        curcode = ''
+                        insertions = []
+                yield match.start(), Generic.Output, line
+        if curcode:
+            for item in do_insertions(insertions,
+                                      pylexer.get_tokens_unprocessed(curcode)):
+                yield item
+
+highlighting.lexers['ipython'] = IPythonConsoleLexer()

docs/sphinxext/only_directives.py

@@ -0,0 +1,87 b''
+#
+# A pair of directives for inserting content that will only appear in
+# either html or latex.
+#
+
+from docutils.nodes import Body, Element
+from docutils.writers.html4css1 import HTMLTranslator
+from sphinx.latexwriter import LaTeXTranslator
+from docutils.parsers.rst import directives
+
+class html_only(Body, Element):
+    pass
+
+class latex_only(Body, Element):
+    pass
+
+def run(content, node_class, state, content_offset):
+    text = '\n'.join(content)
+    node = node_class(text)
+    state.nested_parse(content, content_offset, node)
+    return [node]
+
+try:
+    from docutils.parsers.rst import Directive
+except ImportError:
+    from docutils.parsers.rst.directives import _directives
+
+    def html_only_directive(name, arguments, options, content, lineno,
+                            content_offset, block_text, state, state_machine):
+        return run(content, html_only, state, content_offset)
+
+    def latex_only_directive(name, arguments, options, content, lineno,
+                             content_offset, block_text, state, state_machine):
+        return run(content, latex_only, state, content_offset)
+
+    for func in (html_only_directive, latex_only_directive):
+        func.content = 1
+        func.options = {}
+        func.arguments = None
+
+    _directives['htmlonly'] = html_only_directive
+    _directives['latexonly'] = latex_only_directive
+else:
+    class OnlyDirective(Directive):
+        has_content = True
+        required_arguments = 0
+        optional_arguments = 0
+        final_argument_whitespace = True
+        option_spec = {}
+
+        def run(self):
+            self.assert_has_content()
+            return run(self.content, self.node_class,
+                       self.state, self.content_offset)
+
+    class HtmlOnlyDirective(OnlyDirective):
+        node_class = html_only
+
+    class LatexOnlyDirective(OnlyDirective):
+        node_class = latex_only
+
+    directives.register_directive('htmlonly', HtmlOnlyDirective)
+    directives.register_directive('latexonly', LatexOnlyDirective)
+
+def setup(app):
+    app.add_node(html_only)
+    app.add_node(latex_only)
+
+    # Add visit/depart methods to HTML-Translator:
+    def visit_perform(self, node):
+        pass
+    def depart_perform(self, node):
+        pass
+    def visit_ignore(self, node):
+        node.children = []
+    def depart_ignore(self, node):
+        node.children = []
+
+    HTMLTranslator.visit_html_only = visit_perform
+    HTMLTranslator.depart_html_only = depart_perform
+    HTMLTranslator.visit_latex_only = visit_ignore
+    HTMLTranslator.depart_latex_only = depart_ignore
+
+    LaTeXTranslator.visit_html_only = visit_ignore
+    LaTeXTranslator.depart_html_only = depart_ignore
+    LaTeXTranslator.visit_latex_only = visit_perform
+    LaTeXTranslator.depart_latex_only = depart_perform

docs/sphinxext/plot_directive.py

@@ -0,0 +1,155 b''
+"""A special directive for including a matplotlib plot.
+
+Given a path to a .py file, it includes the source code inline, then:
+
+- On HTML, will include a .png with a link to a high-res .png.
+
+- On LaTeX, will include a .pdf
+
+This directive supports all of the options of the `image` directive,
+except for `target` (since plot will add its own target).
+
+Additionally, if the :include-source: option is provided, the literal
+source will be included inline, as well as a link to the source.
+"""
+
+import sys, os, glob, shutil
+from docutils.parsers.rst import directives
+
+try:
+    # docutils 0.4
+    from docutils.parsers.rst.directives.images import align
+except ImportError:
+    # docutils 0.5
+    from docutils.parsers.rst.directives.images import Image
+    align = Image.align
+
+
+import matplotlib
+import IPython.Shell
+matplotlib.use('Agg')
+import matplotlib.pyplot as plt
+
+mplshell = IPython.Shell.MatplotlibShell('mpl')
+
+options = {'alt': directives.unchanged,
+           'height': directives.length_or_unitless,
+           'width': directives.length_or_percentage_or_unitless,
+           'scale': directives.nonnegative_int,
+           'align': align,
+           'class': directives.class_option,
+           'include-source': directives.flag }
+
+template = """
+.. htmlonly::
+
+   [`source code <../%(srcdir)s/%(basename)s.py>`__,
+   `png <../%(srcdir)s/%(basename)s.hires.png>`__,
+   `pdf <../%(srcdir)s/%(basename)s.pdf>`__]
+
+   .. image:: ../%(srcdir)s/%(basename)s.png
+%(options)s
+
+.. latexonly::
+   .. image:: ../%(srcdir)s/%(basename)s.pdf
+%(options)s
+
+"""
+
+def makefig(fullpath, outdir):
+    """
+    run a pyplot script and save the low and high res PNGs and a PDF in _static
+    """
+
+    fullpath = str(fullpath)  # todo, why is unicode breaking this
+    formats = [('png', 100),
+               ('hires.png', 200),
+               ('pdf', 72),
+               ]
+
+    basedir, fname = os.path.split(fullpath)
+    basename, ext = os.path.splitext(fname)
+    all_exists = True
+
+    if basedir != outdir:
+        shutil.copyfile(fullpath, os.path.join(outdir, fname))
+
+    for format, dpi in formats:
+        outname = os.path.join(outdir, '%s.%s' % (basename, format))
+        if not os.path.exists(outname):
+            all_exists = False
+            break
+
+    if all_exists:
+        print '    already have %s'%fullpath
+        return
+
+    print '    building %s'%fullpath
+    plt.close('all')    # we need to clear between runs
+    matplotlib.rcdefaults()
+
+    mplshell.magic_run(fullpath)
+    for format, dpi in formats:
+        outname = os.path.join(outdir, '%s.%s' % (basename, format))
+        if os.path.exists(outname): continue
+        plt.savefig(outname, dpi=dpi)
+
+def run(arguments, options, state_machine, lineno):
+    reference = directives.uri(arguments[0])
+    basedir, fname = os.path.split(reference)
+    basename, ext = os.path.splitext(fname)
+
+    # todo - should we be using the _static dir for the outdir, I am
+    # not sure we want to corrupt that dir with autogenerated files
+    # since it also has permanent files in it which makes it difficult
+    # to clean (save an rm -rf followed by and svn up)
+    srcdir = 'pyplots'
+
+    makefig(os.path.join(srcdir, reference), srcdir)
+
+    # todo: it is not great design to assume the makefile is putting
+    # the figs into the right place, so we may want to do that here instead.
+
+    if options.has_key('include-source'):
+        lines = ['.. literalinclude:: ../pyplots/%(reference)s' % locals()]
+        del options['include-source']
+    else:
+        lines = []
+
+    options = ['      :%s: %s' % (key, val) for key, val in
+               options.items()]
+    options = "\n".join(options)
+
+    lines.extend((template % locals()).split('\n'))
+
+    state_machine.insert_input(
+        lines, state_machine.input_lines.source(0))
+    return []
+
+
+try:
+    from docutils.parsers.rst import Directive
+except ImportError:
+    from docutils.parsers.rst.directives import _directives
+
+    def plot_directive(name, arguments, options, content, lineno,
+                       content_offset, block_text, state, state_machine):
+        return run(arguments, options, state_machine, lineno)
+    plot_directive.__doc__ = __doc__
+    plot_directive.arguments = (1, 0, 1)
+    plot_directive.options = options
+
+    _directives['plot'] = plot_directive
+else:
+    class plot_directive(Directive):
+        required_arguments = 1
+        optional_arguments = 0
+        final_argument_whitespace = True
+        option_spec = options
+        def run(self):
+            return run(self.arguments, self.options,
+                       self.state_machine, self.lineno)
+    plot_directive.__doc__ = __doc__
+
+    directives.register_directive('plot', plot_directive)
+

tools/compile.py

@@ -0,0 +1,20 b''
+#!/usr/bin/env python
+"""Call the compile script to check that all code we ship compiles correctly.
+"""
+
+import os
+import sys
+
+
+vstr = '.'.join(map(str,sys.version_info[:2]))
+
+stat = os.system('python %s/lib/python%s/compileall.py .' % (sys.prefix,vstr))
+
+print
+if stat:
+    print '*** THERE WAS AN ERROR! ***'
+    print 'See messages above for the actual file that produced it.'
+else:
+    print 'OK'
+
+sys.exit(stat)

IPython/Release.py

@@ -1,7 +1,5 b''
 # -*- coding: utf-8 -*-
-"""Release data for the IPython project.
-
-$Id: Release.py 3002 2008-02-01 07:17:00Z fperez $"""
+"""Release data for the IPython project."""
 
 #*****************************************************************************
 #       Copyright (C) 2001-2006 Fernando Perez <[e-mail]>
@@ -23,9 +21,9 b" name = 'ipython'"
 # bdist_deb does not accept underscores (a Debian convention).
 
 development = False    # change this to False to do a release
-version_base = '0.9.rc1'
+version_base = '0.9.1'
 branch = 'ipython'
-revision = '1124'
+revision = '1143'
 
 if development:
     if branch == 'ipython':
@@ -36,45 +34,69 b' else:'
     version = version_base
 
 
-description = "Tools for interactive development in Python."
+description = "An interactive computing environment for Python"
 
 long_description = \
 """
-IPython provides a replacement for the interactive Python interpreter with
-extra functionality.
+The goal of IPython is to create a comprehensive environment for
+interactive and exploratory computing.  To support this goal, IPython
+has two main components:
+
+* An enhanced interactive Python shell.
+
+* An architecture for interactive parallel computing.
+
+The enhanced interactive Python shell has the following main features:
+
+* Comprehensive object introspection.
+
+* Input history, persistent across sessions.
 
-Main features:
+* Caching of output results during a session with automatically generated
+  references.
 
- * Comprehensive object introspection.
+* Readline based name completion.
 
- * Input history, persistent across sessions.
+* Extensible system of 'magic' commands for controlling the environment and
+  performing many tasks related either to IPython or the operating system.
 
- * Caching of output results during a session with automatically generated
-   references.
+* Configuration system with easy switching between different setups (simpler
+  than changing $PYTHONSTARTUP environment variables every time).
 
- * Readline based name completion.
+* Session logging and reloading.
 
- * Extensible system of 'magic' commands for controlling the environment and
-   performing many tasks related either to IPython or the operating system.
+* Extensible syntax processing for special purpose situations.
 
- * Configuration system with easy switching between different setups (simpler
-   than changing $PYTHONSTARTUP environment variables every time).
+* Access to the system shell with user-extensible alias system.
 
- * Session logging and reloading.
+* Easily embeddable in other Python programs and wxPython GUIs.
 
- * Extensible syntax processing for special purpose situations.
+* Integrated access to the pdb debugger and the Python profiler. 
 
- * Access to the system shell with user-extensible alias system.
+The parallel computing architecture has the following main features:
 
- * Easily embeddable in other Python programs.
+* Quickly parallelize Python code from an interactive Python/IPython session.
 
- * Integrated access to the pdb debugger and the Python profiler. 
+* A flexible and dynamic process model that be deployed on anything from 
+  multicore workstations to supercomputers.
 
- The latest development version is always available at the IPython subversion
- repository_.
+* An architecture that supports many different styles of parallelism, from
+  message passing to task farming.
 
-.. _repository: http://ipython.scipy.org/svn/ipython/ipython/trunk#egg=ipython-dev
- """
+* Both blocking and fully asynchronous interfaces.
+
+* High level APIs that enable many things to be parallelized in a few lines
+  of code.
+
+* Share live parallel jobs with other users securely.  
+
+* Dynamically load balanced task farming system.
+
+* Robust error handling in parallel code.
+
+The latest development version is always available from IPython's `Launchpad 
+site <http://launchpad.net/ipython>`_.
+"""
 
 license = 'BSD'
 

IPython/config/api.py

@@ -16,13 +16,11 b' __docformat__ = "restructuredtext en"'
 #-------------------------------------------------------------------------------
 
 import os
-from IPython.config.cutils import get_home_dir, get_ipython_dir
+from os.path import join as pjoin
+
+from IPython.genutils import get_home_dir, get_ipython_dir
 from IPython.external.configobj import ConfigObj
 
-# Traitlets config imports
-from IPython.config import traitlets
-from IPython.config.config import *
-from traitlets import *
 
 class ConfigObjManager(object):
     
@@ -53,7 +51,7 b' class ConfigObjManager(object):'
 
     def write_default_config_file(self):
         ipdir = get_ipython_dir()
-        fname = ipdir + '/' + self.filename
+        fname = pjoin(ipdir, self.filename)
         if not os.path.isfile(fname):
             print "Writing the configuration file to: " + fname
             self.write_config_obj_to_file(fname)
@@ -87,11 +85,11 b' class ConfigObjManager(object):'
 
         # In ipythondir if it is set
         if ipythondir is not None:
-            trythis = ipythondir + '/' + filename
+            trythis = pjoin(ipythondir, filename)
             if os.path.isfile(trythis):
                 return trythis        
 
-        trythis = get_ipython_dir() + '/' + filename
+        trythis = pjoin(get_ipython_dir(), filename)
         if os.path.isfile(trythis):
             return trythis
 

IPython/config/cutils.py

@@ -22,71 +22,6 b' import sys'
 # Normal code begins
 #---------------------------------------------------------------------------
 
-class HomeDirError(Exception):
-    pass
-
-def get_home_dir():
-    """Return the closest possible equivalent to a 'home' directory.
-
-    We first try $HOME.  Absent that, on NT it's $HOMEDRIVE\$HOMEPATH.
-
-    Currently only Posix and NT are implemented, a HomeDirError exception is
-    raised for all other OSes. """
-
-    isdir = os.path.isdir
-    env = os.environ
-    try:
-        homedir = env['HOME']
-        if not isdir(homedir):
-            # in case a user stuck some string which does NOT resolve to a
-            # valid path, it's as good as if we hadn't foud it
-            raise KeyError
-        return homedir
-    except KeyError:
-        if os.name == 'posix':
-            raise HomeDirError,'undefined $HOME, IPython can not proceed.'
-        elif os.name == 'nt':
-            # For some strange reason, win9x returns 'nt' for os.name.
-            try:
-                homedir = os.path.join(env['HOMEDRIVE'],env['HOMEPATH'])
-                if not isdir(homedir):
-                    homedir = os.path.join(env['USERPROFILE'])
-                    if not isdir(homedir):
-                        raise HomeDirError
-                return homedir
-            except:
-                try:
-                    # Use the registry to get the 'My Documents' folder.
-                    import _winreg as wreg
-                    key = wreg.OpenKey(wreg.HKEY_CURRENT_USER,
-                                       "Software\Microsoft\Windows\CurrentVersion\Explorer\Shell Folders")
-                    homedir = wreg.QueryValueEx(key,'Personal')[0]
-                    key.Close()
-                    if not isdir(homedir):
-                        e = ('Invalid "Personal" folder registry key '
-                             'typically "My Documents".\n'
-                             'Value: %s\n'
-                             'This is not a valid directory on your system.' %
-                             homedir)
-                        raise HomeDirError(e)
-                    return homedir
-                except HomeDirError:
-                    raise
-                except:
-                    return 'C:\\'
-        elif os.name == 'dos':
-            # Desperate, may do absurd things in classic MacOS. May work under DOS.
-            return 'C:\\'
-        else:
-            raise HomeDirError,'support for your operating system not implemented.'
-
-def get_ipython_dir():
-    ipdir_def = '.ipython'
-    home_dir = get_home_dir()
-    ipdir = os.path.abspath(os.environ.get('IPYTHONDIR',
-                                           os.path.join(home_dir,ipdir_def)))
-    return ipdir
-
 def import_item(key):
     """
     Import and return bar given the string foo.bar.

IPython/frontend/_process/killableprocess.py

@@ -50,7 +50,6 b' import subprocess'
 from subprocess import PIPE
 import sys
 import os
-import time
 import types
 
 try:
@@ -69,8 +68,16 b' except ImportError:'
 
 mswindows = (sys.platform == "win32")
 
+skip = False
+
 if mswindows:
-    import winprocess
+    import platform
+    if platform.uname()[3] == '' or platform.uname()[3] > '6.0.6000':
+        # Killable process does not work under vista when starting for
+        # something else than cmd.
+        skip = True
+    else:
+        import winprocess
 else:
     import signal
 
@@ -78,7 +85,11 b' if not mswindows:'
     def DoNothing(*args):
         pass
 
-class Popen(subprocess.Popen):
+
+if skip:
+ Popen = subprocess.Popen
+else:
+ class Popen(subprocess.Popen):
     if not mswindows:
         # Override __init__ to set a preexec_fn
         def __init__(self, *args, **kwargs):

IPython/frontend/_process/pipedprocess.py

@@ -50,7 +50,7 b' class PipedProcess(Thread):'
         """
         env = os.environ
         env['TERM'] = 'xterm'
-        process = Popen((self.command_string + ' 2>&1', ), shell=True,
+        process = Popen(self.command_string + ' 2>&1', shell=True,
                                 env=env,
                                 universal_newlines=True,
                                 stdout=PIPE, stdin=PIPE, )

IPython/frontend/asyncfrontendbase.py

@@ -14,7 +14,7 b' __docformat__ = "restructuredtext en"'
 #-------------------------------------------------------------------------------
 # Imports
 #-------------------------------------------------------------------------------
-import uuid
+from IPython.external import guid
 
 
 from zope.interface import Interface, Attribute, implements, classProvides
@@ -59,7 +59,7 b' class AsyncFrontEndBase(FrontEndBase):'
             return Failure(Exception("Block is not compilable"))
         
         if(blockID == None):
-            blockID = uuid.uuid4() #random UUID
+            blockID = guid.generate() 
         
         d = self.engine.execute(block)
         d.addCallback(self._add_history, block=block)

IPython/frontend/cocoa/cocoa_frontend.py

@@ -26,7 +26,7 b' __docformat__ = "restructuredtext en"'
 
 import sys
 import objc
-import uuid
+from IPython.external import guid
 
 from Foundation import NSObject, NSMutableArray, NSMutableDictionary,\
                         NSLog, NSNotificationCenter, NSMakeRange,\
@@ -361,7 +361,7 b' class IPythonCocoaController(NSObject, AsyncFrontEndBase):'
     
     def next_block_ID(self):
         
-        return uuid.uuid4()
+        return guid.generate() 
     
     def new_cell_block(self):
         """A new CellBlock at the end of self.textView.textStorage()"""

IPython/frontend/frontendbase.py

@@ -21,14 +21,13 b' __docformat__ = "restructuredtext en"'
 # Imports
 #-------------------------------------------------------------------------------
 import string
-import uuid
-import _ast
+import codeop
+from IPython.external import guid
+
 
 from IPython.frontend.zopeinterface import (
     Interface, 
     Attribute, 
-    implements, 
-    classProvides
 )
 from IPython.kernel.core.history import FrontEndHistory
 from IPython.kernel.core.util import Bunch
@@ -134,11 +133,7 b' class IFrontEnd(Interface):'
         
         pass
     
-    def compile_ast(block):
-        """Compiles block to an _ast.AST"""
-        
-        pass
-    
+   
     def get_history_previous(current_block):
         """Returns the block previous in  the history. Saves currentBlock if
         the history_cursor is currently at the end of the input history"""
@@ -220,28 +215,14 b' class FrontEndBase(object):'
         """
         
         try:
-            ast = self.compile_ast(block)
+            is_complete = codeop.compile_command(block.rstrip() + '\n\n',
+                            "<string>", "exec") 
         except:
             return False
         
         lines = block.split('\n')
-        return (len(lines)==1 or str(lines[-1])=='')
-    
-    
-    def compile_ast(self, block):
-        """Compile block to an AST
-        
-        Parameters:
-            block : str
-        
-        Result:
-            AST
-        
-        Throws:
-            Exception if block cannot be compiled
-        """
-        
-        return compile(block, "<string>", "exec", _ast.PyCF_ONLY_AST)
+        return ((is_complete is not None) 
+                    and (len(lines)==1 or str(lines[-1])==''))
     
     
     def execute(self, block, blockID=None):
@@ -261,7 +242,7 b' class FrontEndBase(object):'
             raise Exception("Block is not compilable")
         
         if(blockID == None):
-            blockID = uuid.uuid4() #random UUID
+            blockID = guid.generate()
         
         try:
             result = self.shell.execute(block)

IPython/frontend/linefrontendbase.py

@@ -20,6 +20,8 b' import re'
 
 import IPython
 import sys
+import codeop
+import traceback
 
 from frontendbase import FrontEndBase
 from IPython.kernel.core.interpreter import Interpreter
@@ -76,6 +78,11 b' class LineFrontEndBase(FrontEndBase):'
 
         if banner is not None:
             self.banner = banner
+
+    def start(self):
+        """ Put the frontend in a state where it is ready for user
+            interaction.
+        """
         if self.banner is not None:
             self.write(self.banner, refresh=False)
 
@@ -141,9 +148,18 b' class LineFrontEndBase(FrontEndBase):'
                         and not re.findall(r"\n[\t ]*\n[\t ]*$", string)):
             return False
         else:
-            # Add line returns here, to make sure that the statement is
-            # complete.
-            return FrontEndBase.is_complete(self, string.rstrip() + '\n\n')
+            self.capture_output()
+            try:
+                # Add line returns here, to make sure that the statement is
+                # complete.
+                is_complete = codeop.compile_command(string.rstrip() + '\n\n',
+                            "<string>", "exec")
+                self.release_output()
+            except Exception, e:
+                # XXX: Hack: return True so that the
+                # code gets executed and the error captured.
+                is_complete = True
+            return is_complete
 
 
     def write(self, string, refresh=True):
@@ -166,22 +182,35 b' class LineFrontEndBase(FrontEndBase):'
             raw_string = python_string
         # Create a false result, in case there is an exception
         self.last_result = dict(number=self.prompt_number)
+
+        ## try:
+        ##     self.history.input_cache[-1] = raw_string.rstrip()
+        ##     result = self.shell.execute(python_string)
+        ##     self.last_result = result
+        ##     self.render_result(result)
+        ## except:
+        ##     self.show_traceback()
+        ## finally:
+        ##     self.after_execute()
+
         try:
-            self.history.input_cache[-1] = raw_string.rstrip()
-            result = self.shell.execute(python_string)
-            self.last_result = result
-            self.render_result(result)
-        except:
-            self.show_traceback()
+            try:
+                self.history.input_cache[-1] = raw_string.rstrip()
+                result = self.shell.execute(python_string)
+                self.last_result = result
+                self.render_result(result)
+            except:
+                self.show_traceback()
         finally:
             self.after_execute()
 
+
     #--------------------------------------------------------------------------
     # LineFrontEndBase interface
     #--------------------------------------------------------------------------
 
     def prefilter_input(self, string):
-        """ Priflter the input to turn it in valid python.
+        """ Prefilter the input to turn it in valid python.
         """
         string = string.replace('\r\n', '\n')
         string = string.replace('\t', 4*' ')
@@ -210,9 +239,12 b' class LineFrontEndBase(FrontEndBase):'
         line = self.input_buffer
         new_line, completions = self.complete(line)
         if len(completions)>1:
-            self.write_completion(completions)
-        self.input_buffer = new_line
+            self.write_completion(completions, new_line=new_line)
+        elif not line == new_line:
+            self.input_buffer = new_line
         if self.debug:
+            print >>sys.__stdout__, 'line', line
+            print >>sys.__stdout__, 'new_line', new_line
             print >>sys.__stdout__, completions 
 
 
@@ -222,10 +254,15 b' class LineFrontEndBase(FrontEndBase):'
         return 80
 
 
-    def write_completion(self, possibilities):
+    def write_completion(self, possibilities, new_line=None):
         """ Write the list of possible completions.
+
+            new_line is the completed input line that should be displayed
+            after the completion are writen. If None, the input_buffer
+            before the completion is used.
         """
-        current_buffer = self.input_buffer
+        if new_line is None:
+            new_line = self.input_buffer
         
         self.write('\n')
         max_len = len(max(possibilities, key=len)) + 1
@@ -246,7 +283,7 b' class LineFrontEndBase(FrontEndBase):'
         self.write(''.join(buf))
         self.new_prompt(self.input_prompt_template.substitute(
                             number=self.last_result['number'] + 1))
-        self.input_buffer = current_buffer
+        self.input_buffer = new_line
 
 
     def new_prompt(self, prompt):
@@ -275,6 +312,8 b' class LineFrontEndBase(FrontEndBase):'
         else:
             self.input_buffer += self._get_indent_string(
                                                 current_buffer[:-1])
+            if len(current_buffer.split('\n')) == 2:
+                self.input_buffer += '\t\t'
             if current_buffer[:-1].split('\n')[-1].rstrip().endswith(':'):
                 self.input_buffer += '\t'
 

IPython/frontend/prefilterfrontend.py

@@ -24,6 +24,7 b' __docformat__ = "restructuredtext en"'
 import sys
 
 from linefrontendbase import LineFrontEndBase, common_prefix
+from frontendbase import FrontEndBase
 
 from IPython.ipmaker import make_IPython
 from IPython.ipapi import IPApi
@@ -34,6 +35,7 b' from IPython.kernel.core.sync_traceback_trap import SyncTracebackTrap'
 from IPython.genutils import Term
 import pydoc
 import os
+import sys
 
 
 def mk_system_call(system_call_function, command):
@@ -57,6 +59,8 b' class PrefilterFrontEnd(LineFrontEndBase):'
     to execute the statements and the ipython0 used for code
     completion...
     """
+
+    debug = False
     
     def __init__(self, ipython0=None, *args, **kwargs):
         """ Parameters:
@@ -65,12 +69,24 b' class PrefilterFrontEnd(LineFrontEndBase):'
             ipython0: an optional ipython0 instance to use for command
             prefiltering and completion.
         """
+        LineFrontEndBase.__init__(self, *args, **kwargs)
+        self.shell.output_trap = RedirectorOutputTrap(
+                            out_callback=self.write,
+                            err_callback=self.write,
+                                            )
+        self.shell.traceback_trap = SyncTracebackTrap(
+                        formatters=self.shell.traceback_trap.formatters,
+                            )
+
+        # Start the ipython0 instance:
         self.save_output_hooks()
         if ipython0 is None:
             # Instanciate an IPython0 interpreter to be able to use the
             # prefiltering.
             # XXX: argv=[] is a bit bold.
-            ipython0 = make_IPython(argv=[])
+            ipython0 = make_IPython(argv=[], 
+                                    user_ns=self.shell.user_ns,
+                                    user_global_ns=self.shell.user_global_ns)
         self.ipython0 = ipython0
         # Set the pager:
         self.ipython0.set_hook('show_in_pager', 
@@ -86,24 +102,13 b' class PrefilterFrontEnd(LineFrontEndBase):'
                                                             'ls -CF')
         # And now clean up the mess created by ipython0
         self.release_output()
+
+
         if not 'banner' in kwargs and self.banner is None:
-            kwargs['banner'] = self.ipython0.BANNER + """
+            self.banner = self.ipython0.BANNER + """
 This is the wx frontend, by Gael Varoquaux. This is EXPERIMENTAL code."""
 
-        LineFrontEndBase.__init__(self, *args, **kwargs)
-        # XXX: Hack: mix the two namespaces
-        self.shell.user_ns.update(self.ipython0.user_ns)
-        self.ipython0.user_ns = self.shell.user_ns 
-        self.shell.user_global_ns.update(self.ipython0.user_global_ns)
-        self.ipython0.user_global_ns = self.shell.user_global_ns
-
-        self.shell.output_trap = RedirectorOutputTrap(
-                            out_callback=self.write,
-                            err_callback=self.write,
-                                            )
-        self.shell.traceback_trap = SyncTracebackTrap(
-                        formatters=self.shell.traceback_trap.formatters,
-                            )
+        self.start()
 
     #--------------------------------------------------------------------------
     # FrontEndBase interface 
@@ -113,7 +118,7 b' This is the wx frontend, by Gael Varoquaux. This is EXPERIMENTAL code."""'
         """ Use ipython0 to capture the last traceback and display it.
         """
         self.capture_output()
-        self.ipython0.showtraceback()
+        self.ipython0.showtraceback(tb_offset=-1)
         self.release_output()
 
 
@@ -164,6 +169,8 b' This is the wx frontend, by Gael Varoquaux. This is EXPERIMENTAL code."""'
 
 
     def complete(self, line):
+        # FIXME: This should be factored out in the linefrontendbase
+        # method.
         word = line.split('\n')[-1].split(' ')[-1]
         completions = self.ipython0.complete(word)
         # FIXME: The proper sort should be done in the complete method.
@@ -189,17 +196,33 b' This is the wx frontend, by Gael Varoquaux. This is EXPERIMENTAL code."""'
         # capture it.
         self.capture_output()
         self.last_result = dict(number=self.prompt_number)
+        
+        ## try:
+        ##     for line in input_string.split('\n'):
+        ##         filtered_lines.append(
+        ##                 self.ipython0.prefilter(line, False).rstrip())
+        ## except:
+        ##     # XXX: probably not the right thing to do.
+        ##     self.ipython0.showsyntaxerror()
+        ##     self.after_execute()
+        ## finally:
+        ##     self.release_output()
+
+
         try:
-            for line in input_string.split('\n'):
-                filtered_lines.append(
-                        self.ipython0.prefilter(line, False).rstrip())
-        except:
-            # XXX: probably not the right thing to do.
-            self.ipython0.showsyntaxerror()
-            self.after_execute()
+            try:
+                for line in input_string.split('\n'):
+                    filtered_lines.append(
+                            self.ipython0.prefilter(line, False).rstrip())
+            except:
+                # XXX: probably not the right thing to do.
+                self.ipython0.showsyntaxerror()
+                self.after_execute()
         finally:
             self.release_output()
 
+
+
         # Clean up the trailing whitespace, to avoid indentation errors
         filtered_string = '\n'.join(filtered_lines)
         return filtered_string

IPython/frontend/tests/test_frontendbase.py

@@ -1,157 +1,32 b''
 # encoding: utf-8
-
-"""This file contains unittests for the frontendbase module."""
+"""
+Test the basic functionality of frontendbase. 
+"""
 
 __docformat__ = "restructuredtext en"
 
-#---------------------------------------------------------------------------
-#  Copyright (C) 2008  The IPython Development Team                         
-#                                                                           
-#  Distributed under the terms of the BSD License.  The full license is in  
-#  the file COPYING, distributed as part of this software.                  
-#---------------------------------------------------------------------------
-                                                                            
-#---------------------------------------------------------------------------
-# Imports                                                                   
-#---------------------------------------------------------------------------
-
-import unittest
-
-try:
-    from IPython.frontend.asyncfrontendbase import AsyncFrontEndBase
-    from IPython.frontend import frontendbase 
-    from IPython.kernel.engineservice import EngineService
-except ImportError:
-    import nose
-    raise nose.SkipTest("This test requires zope.interface, Twisted and Foolscap")
-
-class FrontEndCallbackChecker(AsyncFrontEndBase):
-    """FrontEndBase subclass for checking callbacks"""
-    def __init__(self, engine=None, history=None):
-        super(FrontEndCallbackChecker, self).__init__(engine=engine, 
-                                                    history=history)
-        self.updateCalled = False
-        self.renderResultCalled = False
-        self.renderErrorCalled = False
-   
-    def update_cell_prompt(self, result, blockID=None):
-        self.updateCalled = True
-        return result
-   
-    def render_result(self, result):
-        self.renderResultCalled = True
-        return result
-    
-    
-    def render_error(self, failure):
-        self.renderErrorCalled = True
-        return failure
-    
-
+#-------------------------------------------------------------------------------
+#  Copyright (C) 2008  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is
+#  in the file COPYING, distributed as part of this software.
+#-------------------------------------------------------------------------------
+
+from IPython.frontend.frontendbase import FrontEndBase
+
+def test_iscomplete():
+    """ Check that is_complete works. 
+    """
+    f = FrontEndBase()
+    assert f.is_complete('(a + a)')
+    assert not f.is_complete('(a + a')
+    assert f.is_complete('1')
+    assert not f.is_complete('1 + ')
+    assert not f.is_complete('1 + \n\n')
+    assert f.is_complete('if True:\n  print 1\n')
+    assert not f.is_complete('if True:\n  print 1')
+    assert f.is_complete('def f():\n  print 1\n')
+
+if __name__ == '__main__':
+    test_iscomplete()
 
-    
-class TestAsyncFrontendBase(unittest.TestCase):
-    def setUp(self):
-        """Setup the EngineService and FrontEndBase"""
-        
-        self.fb = FrontEndCallbackChecker(engine=EngineService())
-    
-    
-    def test_implements_IFrontEnd(self):
-        assert(frontendbase.IFrontEnd.implementedBy(
-                                    AsyncFrontEndBase))
-    
-    
-    def test_is_complete_returns_False_for_incomplete_block(self):
-        """"""
-        
-        block = """def test(a):"""
-        
-        assert(self.fb.is_complete(block) == False)
-    
-    def test_is_complete_returns_True_for_complete_block(self):
-        """"""
-        
-        block = """def test(a): pass"""
-            
-        assert(self.fb.is_complete(block))
-        
-        block = """a=3"""
-        
-        assert(self.fb.is_complete(block))
-    
-    
-    def test_blockID_added_to_result(self):
-        block = """3+3"""
-        
-        d = self.fb.execute(block, blockID='TEST_ID')
-        
-        d.addCallback(self.checkBlockID, expected='TEST_ID')
-    
-    def test_blockID_added_to_failure(self):
-        block = "raise Exception()"
-        
-        d = self.fb.execute(block,blockID='TEST_ID')
-        d.addErrback(self.checkFailureID, expected='TEST_ID')
-    
-    def checkBlockID(self, result, expected=""):
-        assert(result['blockID'] == expected)
-    
-    
-    def checkFailureID(self, failure, expected=""):
-        assert(failure.blockID == expected)
-    
-    
-    def test_callbacks_added_to_execute(self):
-        """test that
-        update_cell_prompt
-        render_result
-        
-        are added to execute request
-        """
-        
-        d = self.fb.execute("10+10")
-        d.addCallback(self.checkCallbacks)
-    
-    
-    def checkCallbacks(self, result):
-        assert(self.fb.updateCalled)
-        assert(self.fb.renderResultCalled)
-    
-    
-    def test_error_callback_added_to_execute(self):
-        """test that render_error called on execution error"""
-        
-        d = self.fb.execute("raise Exception()")
-        d.addCallback(self.checkRenderError)
-    
-    def checkRenderError(self, result):
-        assert(self.fb.renderErrorCalled)
-    
-    def test_history_returns_expected_block(self):
-        """Make sure history browsing doesn't fail"""
-        
-        blocks = ["a=1","a=2","a=3"]
-        for b in blocks:
-            d = self.fb.execute(b)
-        
-        # d is now the deferred for the last executed block
-        d.addCallback(self.historyTests, blocks)
-        
-    
-    def historyTests(self, result, blocks):
-        """historyTests"""
-        
-        assert(len(blocks) >= 3)
-        assert(self.fb.get_history_previous("") == blocks[-2])
-        assert(self.fb.get_history_previous("") == blocks[-3])
-        assert(self.fb.get_history_next() == blocks[-2])
-    
-    
-    def test_history_returns_none_at_startup(self):
-        """test_history_returns_none_at_startup"""
-        
-        assert(self.fb.get_history_previous("")==None)
-        assert(self.fb.get_history_next()==None)
-    
-    

IPython/frontend/tests/test_process.py

@@ -19,6 +19,7 b' import sys'
 from IPython.frontend._process import PipedProcess
 from IPython.testing import decorators as testdec
 
+
 def test_capture_out():
     """ A simple test to see if we can execute a process and get the output.
     """
@@ -29,8 +30,7 b' def test_capture_out():'
     result = s.getvalue().rstrip()
     assert result == '1'
 
-# FIXME
[e-mail]("This doesn't work under Windows")
+
 def test_io():
     """ Checks that we can send characters on stdin to the process.
     """

IPython/frontend/wx/console_widget.py

@@ -23,6 +23,7 b' import wx'
 import wx.stc  as  stc
 
 from wx.py import editwindow
+import time
 import sys
 LINESEP = '\n'
 if sys.platform == 'win32':
@@ -35,7 +36,7 b' import re'
 
 _DEFAULT_SIZE = 10
 if sys.platform == 'darwin':
-    _DEFAULT_STYLE = 12
+    _DEFAULT_SIZE = 12
 
 _DEFAULT_STYLE = {
     'stdout'      : 'fore:#0000FF',
@@ -115,12 +116,15 b' class ConsoleWidget(editwindow.EditWindow):'
     # The color of the carret (call _apply_style() after setting)
     carret_color = 'BLACK'
     
+    # Store the last time a refresh was done
+    _last_refresh_time = 0
+
     #--------------------------------------------------------------------------
     # Public API
     #--------------------------------------------------------------------------
     
     def __init__(self, parent, id=wx.ID_ANY, pos=wx.DefaultPosition, 
-                        size=wx.DefaultSize, style=0, ):
+                        size=wx.DefaultSize, style=wx.WANTS_CHARS, ):
         editwindow.EditWindow.__init__(self, parent, id, pos, size, style)
         self._configure_scintilla()
 
@@ -168,9 +172,14 b' class ConsoleWidget(editwindow.EditWindow):'
                 
         self.GotoPos(self.GetLength())
         if refresh:
-            # Maybe this is faster than wx.Yield()
-            self.ProcessEvent(wx.PaintEvent())
-            #wx.Yield()
+            current_time = time.time()
+            if current_time - self._last_refresh_time > 0.03:
+                if sys.platform == 'win32':
+                    wx.SafeYield()
+                else:
+                    wx.Yield()
+                #    self.ProcessEvent(wx.PaintEvent())
+                self._last_refresh_time = current_time 
 
    
     def new_prompt(self, prompt):
@@ -183,7 +192,6 b' class ConsoleWidget(editwindow.EditWindow):'
         # now we update our cursor giving end of prompt
         self.current_prompt_pos = self.GetLength()
         self.current_prompt_line = self.GetCurrentLine()
-        wx.Yield()
         self.EnsureCaretVisible()
 
 

IPython/frontend/wx/wx_frontend.py

@@ -128,6 +128,7 b' class WxController(ConsoleWidget, PrefilterFrontEnd):'
     # while it is being swapped
     _out_buffer_lock = Lock()
 
+    # The different line markers used to higlight the prompts.
     _markers = dict()
 
     #--------------------------------------------------------------------------
@@ -135,12 +136,16 b' class WxController(ConsoleWidget, PrefilterFrontEnd):'
     #--------------------------------------------------------------------------
  
     def __init__(self, parent, id=wx.ID_ANY, pos=wx.DefaultPosition,
-                 size=wx.DefaultSize, style=wx.CLIP_CHILDREN,
+                 size=wx.DefaultSize,
+                 style=wx.CLIP_CHILDREN|wx.WANTS_CHARS,
                  *args, **kwds):
         """ Create Shell instance.
         """
         ConsoleWidget.__init__(self, parent, id, pos, size, style)
         PrefilterFrontEnd.__init__(self, **kwds)
+        
+        # Stick in our own raw_input:
+        self.ipython0.raw_input = self.raw_input
 
         # Marker for complete buffer.
         self.MarkerDefine(_COMPLETE_BUFFER_MARKER, stc.STC_MARK_BACKGROUND,
@@ -164,9 +169,11 b' class WxController(ConsoleWidget, PrefilterFrontEnd):'
         # Inject self in namespace, for debug
         if self.debug:
             self.shell.user_ns['self'] = self
+        # Inject our own raw_input in namespace
+        self.shell.user_ns['raw_input'] = self.raw_input
 
 
-    def raw_input(self, prompt):
+    def raw_input(self, prompt=''):
         """ A replacement from python's raw_input.
         """
         self.new_prompt(prompt)
@@ -174,15 +181,13 b' class WxController(ConsoleWidget, PrefilterFrontEnd):'
         if hasattr(self, '_cursor'):
             del self._cursor 
         self.SetCursor(wx.StockCursor(wx.CURSOR_CROSS))
-        self.waiting = True
         self.__old_on_enter = self._on_enter
+        event_loop = wx.EventLoop()
         def my_on_enter():
-            self.waiting = False
+            event_loop.Exit()
         self._on_enter = my_on_enter
-        # XXX: Busy waiting, ugly.
-        while self.waiting:
-            wx.Yield()
-            sleep(0.1)
+        # XXX: Running a separate event_loop. Ugly.
+        event_loop.Run() 
         self._on_enter = self.__old_on_enter
         self._input_state = 'buffering'
         self._cursor = wx.BusyCursor()
@@ -191,16 +196,18 b' class WxController(ConsoleWidget, PrefilterFrontEnd):'
 
     def system_call(self, command_string):
         self._input_state = 'subprocess'
+        event_loop = wx.EventLoop()
+        def _end_system_call():
+            self._input_state = 'buffering'
+            self._running_process = False
+            event_loop.Exit()
+
         self._running_process = PipedProcess(command_string, 
                     out_callback=self.buffered_write,
-                    end_callback = self._end_system_call)
+                    end_callback = _end_system_call)
         self._running_process.start()
-        # XXX: another one of these polling loops to have a blocking
-        # call
-        wx.Yield()
-        while self._running_process:
-            wx.Yield()
-            sleep(0.1)
+        # XXX: Running a separate event_loop. Ugly.
+        event_loop.Run() 
         # Be sure to flush the buffer.
         self._buffer_flush(event=None)
 
@@ -226,8 +233,9 b' class WxController(ConsoleWidget, PrefilterFrontEnd):'
             for name in symbol_string.split('.')[1:] + ['__doc__']:
                 symbol = getattr(symbol, name)
             self.AutoCompCancel()
-            wx.Yield()
-            self.CallTipShow(self.GetCurrentPos(), symbol)
+            # Check that the symbol can indeed be converted to a string:
+            symbol += ''
+            wx.CallAfter(self.CallTipShow, self.GetCurrentPos(), symbol)
         except:
             # The retrieve symbol couldn't be converted to a string
             pass
@@ -238,9 +246,9 b' class WxController(ConsoleWidget, PrefilterFrontEnd):'
             true, open the menu.
         """
         if self.debug:
-            print >>sys.__stdout__, "_popup_completion", 
+            print >>sys.__stdout__, "_popup_completion" 
         line = self.input_buffer
-        if (self.AutoCompActive() and not line[-1] == '.') \
+        if (self.AutoCompActive() and line and not line[-1] == '.') \
                     or create==True:
             suggestion, completions = self.complete(line)
             offset=0
@@ -284,19 +292,21 b' class WxController(ConsoleWidget, PrefilterFrontEnd):'
             if i in self._markers:
                 self.MarkerDeleteHandle(self._markers[i])
             self._markers[i] = self.MarkerAdd(i, _COMPLETE_BUFFER_MARKER)
-        # Update the display:
-        wx.Yield()
-        self.GotoPos(self.GetLength())
-        PrefilterFrontEnd.execute(self, python_string, raw_string=raw_string)
+        # Use a callafter to update the display robustly under windows
+        def callback():
+            self.GotoPos(self.GetLength())
+            PrefilterFrontEnd.execute(self, python_string, 
+                                            raw_string=raw_string)
+        wx.CallAfter(callback)
 
     def save_output_hooks(self):    
         self.__old_raw_input = __builtin__.raw_input
         PrefilterFrontEnd.save_output_hooks(self)
 
     def capture_output(self):
-        __builtin__.raw_input = self.raw_input
         self.SetLexer(stc.STC_LEX_NULL)
         PrefilterFrontEnd.capture_output(self)
+        __builtin__.raw_input = self.raw_input
         
     
     def release_output(self):
@@ -316,12 +326,24 b' class WxController(ConsoleWidget, PrefilterFrontEnd):'
     def show_traceback(self):
         start_line = self.GetCurrentLine()
         PrefilterFrontEnd.show_traceback(self)
-        wx.Yield()
+        self.ProcessEvent(wx.PaintEvent())
+        #wx.Yield()
         for i in range(start_line, self.GetCurrentLine()):
             self._markers[i] = self.MarkerAdd(i, _ERROR_MARKER)
 
     
     #--------------------------------------------------------------------------
+    # FrontEndBase interface 
+    #--------------------------------------------------------------------------
+    
+    def render_error(self, e):
+        start_line = self.GetCurrentLine()
+        self.write('\n' + e + '\n')
+        for i in range(start_line, self.GetCurrentLine()):
+            self._markers[i] = self.MarkerAdd(i, _ERROR_MARKER)
+
+
+    #--------------------------------------------------------------------------
     # ConsoleWidget interface 
     #--------------------------------------------------------------------------
 
@@ -351,7 +373,8 b' class WxController(ConsoleWidget, PrefilterFrontEnd):'
             if self._input_state == 'subprocess':
                 if self.debug:
                     print >>sys.__stderr__, 'Killing running process'
-                self._running_process.process.kill()
+                if hasattr(self._running_process, 'process'):
+                    self._running_process.process.kill()
             elif self._input_state == 'buffering':
                 if self.debug:
                     print >>sys.__stderr__, 'Raising KeyboardInterrupt'
@@ -376,7 +399,7 b' class WxController(ConsoleWidget, PrefilterFrontEnd):'
                 char = '\04'
             self._running_process.process.stdin.write(char)
             self._running_process.process.stdin.flush()
-        elif event.KeyCode in (ord('('), 57):
+        elif event.KeyCode in (ord('('), 57, 53):
             # Calltips
             event.Skip()
             self.do_calltip()
@@ -410,8 +433,8 b' class WxController(ConsoleWidget, PrefilterFrontEnd):'
                     self.input_buffer = new_buffer
             # Tab-completion
             elif event.KeyCode == ord('\t'):
-                last_line = self.input_buffer.split('\n')[-1]
-                if not re.match(r'^\s*$', last_line):
+                current_line, current_line_number = self.CurLine
+                if not re.match(r'^\s*$', current_line):
                     self.complete_current_input()
                     if self.AutoCompActive():
                         wx.CallAfter(self._popup_completion, create=True)
@@ -427,7 +450,7 b' class WxController(ConsoleWidget, PrefilterFrontEnd):'
         if event.KeyCode in (59, ord('.')):
             # Intercepting '.'
             event.Skip()
-            self._popup_completion(create=True)
+            wx.CallAfter(self._popup_completion, create=True)
         else:
             ConsoleWidget._on_key_up(self, event, skip=skip)
 
@@ -456,13 +479,6 b' class WxController(ConsoleWidget, PrefilterFrontEnd):'
     # Private API
     #--------------------------------------------------------------------------
  
-    def _end_system_call(self):
-        """ Called at the end of a system call.
-        """
-        self._input_state = 'buffering'
-        self._running_process = False
-
-
     def _buffer_flush(self, event):
         """ Called by the timer to flush the write buffer.
             

IPython/frontend/zopeinterface.py

@@ -16,13 +16,6 b' __docformat__ = "restructuredtext en"'
 #  the file COPYING, distributed as part of this software.
 #-------------------------------------------------------------------------------
 
-#-------------------------------------------------------------------------------
-# Imports
-#-------------------------------------------------------------------------------
-import string
-import uuid
-import _ast
-
 try:
     from zope.interface import Interface, Attribute, implements, classProvides
 except ImportError:

IPython/genutils.py

@@ -979,6 +979,38 b' def get_home_dir():'
         else:
             raise HomeDirError,'support for your operating system not implemented.'
 
+
+def get_ipython_dir():
+    """Get the IPython directory for this platform and user.
+    
+    This uses the logic in `get_home_dir` to find the home directory
+    and the adds either .ipython or _ipython to the end of the path.
+    """
+    if os.name == 'posix':
+         ipdir_def = '.ipython'
+    else:
+         ipdir_def = '_ipython'
+    home_dir = get_home_dir()
+    ipdir = os.path.abspath(os.environ.get('IPYTHONDIR',
+                                           os.path.join(home_dir,ipdir_def)))
+    return ipdir
+
+def get_security_dir():
+    """Get the IPython security directory.
+    
+    This directory is the default location for all security related files,
+    including SSL/TLS certificates and FURL files.
+    
+    If the directory does not exist, it is created with 0700 permissions.
+    If it exists, permissions are set to 0700.
+    """
+    security_dir = os.path.join(get_ipython_dir(), 'security')
+    if not os.path.isdir(security_dir):
+        os.mkdir(security_dir, 0700)
+    else:
+        os.chmod(security_dir, 0700)
+    return security_dir
+        
 #****************************************************************************
 # strings and text
 

IPython/kernel/config/__init__.py

@@ -15,17 +15,15 b' __docformat__ = "restructuredtext en"'
 # Imports
 #-------------------------------------------------------------------------------
 
+from os.path import join as pjoin
+
 from IPython.external.configobj import ConfigObj
 from IPython.config.api import ConfigObjManager
-from IPython.config.cutils import get_ipython_dir
+from IPython.genutils import get_ipython_dir, get_security_dir
 
 default_kernel_config = ConfigObj()
 
-try:
-    ipython_dir = get_ipython_dir() + '/'
-except:
-    # This will defaults to the cwd
-    ipython_dir = ''
+security_dir = get_security_dir()
 
 #-------------------------------------------------------------------------------
 # Engine Configuration
@@ -33,7 +31,7 b' except:'
 
 engine_config = dict(
     logfile = '',    # Empty means log to stdout
-    furl_file = ipython_dir + 'ipcontroller-engine.furl'
+    furl_file = pjoin(security_dir, 'ipcontroller-engine.furl')
 )
 
 #-------------------------------------------------------------------------------
@@ -69,10 +67,10 b' controller_config = dict('
         port = 0,        # 0 means pick a port for me
         location = '',    # Empty string means try to set automatically
         secure = True,
-        cert_file = ipython_dir + 'ipcontroller-engine.pem',
+        cert_file = pjoin(security_dir, 'ipcontroller-engine.pem'),
     ),
     engine_fc_interface = 'IPython.kernel.enginefc.IFCControllerBase',
-    engine_furl_file = ipython_dir + 'ipcontroller-engine.furl',
+    engine_furl_file = pjoin(security_dir, 'ipcontroller-engine.furl'),
     
     controller_interfaces = dict(
         # multiengine = dict(
@@ -83,12 +81,12 b' controller_config = dict('
         task = dict(
             controller_interface = 'IPython.kernel.task.ITaskController',
             fc_interface = 'IPython.kernel.taskfc.IFCTaskController',
-            furl_file = ipython_dir + 'ipcontroller-tc.furl'
+            furl_file = pjoin(security_dir, 'ipcontroller-tc.furl')
         ),
         multiengine = dict(
             controller_interface = 'IPython.kernel.multiengine.IMultiEngine',
             fc_interface = 'IPython.kernel.multienginefc.IFCSynchronousMultiEngine',
-            furl_file = ipython_dir + 'ipcontroller-mec.furl'
+            furl_file = pjoin(security_dir, 'ipcontroller-mec.furl')
         )
     ),
 
@@ -97,7 +95,7 b' controller_config = dict('
         port = 0,        # 0 means pick a port for me
         location = '',    # Empty string means try to set automatically
         secure = True,
-        cert_file = ipython_dir + 'ipcontroller-client.pem'
+        cert_file = pjoin(security_dir, 'ipcontroller-client.pem')
     )
 )
 
@@ -108,10 +106,10 b' controller_config = dict('
 client_config = dict(
     client_interfaces = dict(
         task = dict(
-            furl_file = ipython_dir + 'ipcontroller-tc.furl'
+            furl_file = pjoin(security_dir, 'ipcontroller-tc.furl')
         ),
         multiengine = dict(
-            furl_file = ipython_dir + 'ipcontroller-mec.furl'
+            furl_file = pjoin(security_dir, 'ipcontroller-mec.furl')
         )
     )
 )

IPython/kernel/contexts.py

@@ -8,8 +8,6 b' which can also be useful as templates for writing new, application-specific'
 managers.
 """
 
-from __future__ import with_statement
-
 __docformat__ = "restructuredtext en"
 
 #-------------------------------------------------------------------------------

IPython/kernel/controllerservice.py

@@ -50,7 +50,7 b' from IPython.kernel.engineservice import \\'
     IEngineSerialized, \
     IEngineQueued
     
-from IPython.config import cutils
+from IPython.genutils import get_ipython_dir
 from IPython.kernel import codeutil
 
 #-------------------------------------------------------------------------------
@@ -170,7 +170,7 b' class ControllerService(object, service.Service):'
     
     def _getEngineInfoLogFile(self):
         # Store all logs inside the ipython directory
-        ipdir = cutils.get_ipython_dir()
+        ipdir = get_ipython_dir()
         pjoin = os.path.join
         logdir_base = pjoin(ipdir,'log')
         if not os.path.isdir(logdir_base):

IPython/kernel/core/interpreter.py

@@ -680,6 +680,13 b' class Interpreter(object):'
         # how trailing whitespace is handled, but this seems to work.
         python = python.strip()
 
+        # The compiler module does not like unicode. We need to convert
+        # it encode it:
+        if isinstance(python, unicode):
+            # Use the utf-8-sig BOM so the compiler detects this a UTF-8
+            # encode string.
+            python = '\xef\xbb\xbf' + python.encode('utf-8')
+
         # The compiler module will parse the code into an abstract syntax tree.
         ast = compiler.parse(python)
 

IPython/kernel/core/tests/test_redirectors.py

@@ -13,13 +13,17 b' __docformat__ = "restructuredtext en"'
 #-------------------------------------------------------------------------------
 
 
+# Stdlib imports
 import os
 from cStringIO import StringIO
 
-from IPython.testing import decorators as testdec
+# Our own imports
+from IPython.testing import decorators as dec
 
-# FIXME
[e-mail]("This doesn't work under Windows")
+#-----------------------------------------------------------------------------
+# Test functions
+
[e-mail]_win32
 def test_redirector():
     """ Checks that the redirector can be used to do synchronous capture.
     """
@@ -40,8 +44,8 b' def test_redirector():'
     result2 = "".join("%ic\n%i\n" %(i, i) for i in range(10))
     assert result1 == result2
 
-# FIXME
[e-mail]("This doesn't work under Windows")
+
[e-mail]_win32
 def test_redirector_output_trap():
     """ This test check not only that the redirector_output_trap does
         trap the output, but also that it does it in a gready way, that
@@ -63,6 +67,4 b' def test_redirector_output_trap():'
     result1 = out.getvalue()
     result2 = "".join("%ic\n%ip\n%i\n" %(i, i, i) for i in range(10))
     assert result1 == result2
-
     
-

IPython/kernel/scripts/ipcluster.py

@@ -93,7 +93,7 b' from subprocess import Popen,call'
 # IPython imports
 #---------------------------------------------------------------------------
 from IPython.tools import utils
-from IPython.config import cutils
+from IPython.genutils import get_ipython_dir
 
 #---------------------------------------------------------------------------
 # Normal code begins
@@ -180,7 +180,7 b' def clusterLocal(opt,arg):'
     """Start a cluster on the local machine."""
     
     # Store all logs inside the ipython directory
-    ipdir = cutils.get_ipython_dir()
+    ipdir = get_ipython_dir()
     pjoin = os.path.join
 
     logfile = opt.logfile
@@ -256,6 +256,24 b' def clusterLocal(opt,arg):'
 def clusterRemote(opt,arg):
     """Start a remote cluster over SSH"""
 
+    # B. Granger, 9/3/08
+    # The launching of a remote cluster using SSH and a clusterfile
+    # is broken.  Because it won't be fixed before the 0.9 release, 
+    # we are removing it.  For now, we just print a message to the 
+    # user and abort.
+    
+    print """The launching of a remote IPython cluster using SSL
+and a clusterfile has been removed in this release.  
+It has been broken for a while and we are in the process 
+of building a new process management system that will be 
+used to provide a more robust way of starting an IPython
+cluster.
+
+For now remote clusters have to be launched using ipcontroller
+and ipengine separately.
+    """
+    sys.exit(1)
+
     # Load the remote cluster configuration
     clConfig = {}
     execfile(opt.clusterfile,clConfig)
@@ -265,7 +283,7 b' def clusterRemote(opt,arg):'
     sshx = clConfig.get('sshx',os.environ.get('IPYTHON_SSHX','sshx'))
     
     # Store all logs inside the ipython directory
-    ipdir = cutils.get_ipython_dir()
+    ipdir = get_ipython_dir()
     pjoin = os.path.join
 
     logfile = opt.logfile
@@ -311,6 +329,11 b' def clusterRemote(opt,arg):'
 def main():
     """Main driver for the two big options: local or remote cluster."""
     
+    if sys.platform=='win32':
+        print """ipcluster does not work on Microsoft Windows.  Please start
+your IPython cluster using the ipcontroller and ipengine scripts."""
+        sys.exit(1)
+    
     opt,arg = parse_args()
 
     clusterfile = opt.clusterfile

IPython/kernel/scripts/ipengine.py

@@ -105,6 +105,7 b' def start_engine():'
     # register_engine to tell the controller we are ready to do work
     engine_connector = EngineConnector(tub_service)
     furl_file = kernel_config['engine']['furl_file']
+    log.msg("Using furl file: %s" % furl_file)
     d = engine_connector.connect_to_controller(engine_service, furl_file)
     d.addErrback(lambda _: reactor.stop())
     

IPython/kernel/tests/engineservicetest.py

@@ -245,7 +245,7 b' class IEngineSerializedTestCase(object):'
         self.assert_(es.IEngineSerialized.providedBy(self.engine))
         
     def testIEngineSerializedInterfaceMethods(self):
-        """Does self.engine have the methods and attributes in IEngireCore."""
+        """Does self.engine have the methods and attributes in IEngineCore."""
         for m in list(es.IEngineSerialized):
             self.assert_(hasattr(self.engine, m))
        
@@ -288,7 +288,7 b' class IEngineQueuedTestCase(object):'
         self.assert_(es.IEngineQueued.providedBy(self.engine))
         
     def testIEngineQueuedInterfaceMethods(self):
-        """Does self.engine have the methods and attributes in IEngireQueued."""
+        """Does self.engine have the methods and attributes in IEngineQueued."""
         for m in list(es.IEngineQueued):
             self.assert_(hasattr(self.engine, m))
             
@@ -326,7 +326,7 b' class IEnginePropertiesTestCase(object):'
         self.assert_(es.IEngineProperties.providedBy(self.engine))
     
     def testIEnginePropertiesInterfaceMethods(self):
-        """Does self.engine have the methods and attributes in IEngireProperties."""
+        """Does self.engine have the methods and attributes in IEngineProperties."""
         for m in list(es.IEngineProperties):
             self.assert_(hasattr(self.engine, m))
     

IPython/kernel/tests/test_contexts.py

@@ -1,4 +1,6 b''
-from __future__ import with_statement
+#from __future__ import with_statement
+
+# XXX This file is currently disabled to preserve 2.4 compatibility.
 
 #def test_simple():
 if 0:
@@ -25,17 +27,17 b' if 0:'
 
     mec.pushAll()
 
-    with parallel as pr:
-        # A comment
-        remote()  # this means the code below only runs remotely
-        print 'Hello remote world'
-        x = range(10)
-        # Comments are OK
-        # Even misindented.
-        y = x+1
+    ## with parallel as pr:
+    ##     # A comment
+    ##     remote()  # this means the code below only runs remotely
+    ##     print 'Hello remote world'
+    ##     x = range(10)
+    ##     # Comments are OK
+    ##     # Even misindented.
+    ##     y = x+1
 
 
-    with pfor('i',sequence) as pr:
-        print x[i]
+    ## with pfor('i',sequence) as pr:
+    ##     print x[i]
 
     print pr.x + pr.y

IPython/testing/decorators.py

@@ -8,6 +8,10 b' the decorator, in order to preserve metadata such as function name,'
 setup and teardown functions and so on - see nose.tools for more
 information.
 
+This module provides a set of useful decorators meant to be ready to use in
+your own tests.  See the bottom of the file for the ready-made ones, and if you
+find yourself writing a new one that may be of generic use, add it here.
+
 NOTE: This file contains IPython-specific decorators and imports the
 numpy.testing.decorators file, which we've copied verbatim.  Any of our own
 code will be added at the bottom if we end up extending this.
@@ -15,6 +19,7 b' code will be added at the bottom if we end up extending this.'
 
 # Stdlib imports
 import inspect
+import sys
 
 # Third-party imports
 
@@ -123,6 +128,9 b" skip_doctest = make_label_dec('skip_doctest',"
 def skip(msg=''):
     """Decorator - mark a test function for skipping from test suite.
 
+    This function *is* already a decorator, it is not a factory like
+    make_label_dec or some of those in decorators_numpy.
+
     :Parameters:
 
       func : function
@@ -145,3 +153,8 b" def skip(msg=''):"
         return apply_wrapper(wrapper,func)
 
     return inner
+
+# Decorators to skip certain tests on specific platforms.
+skip_win32 = skipif(sys.platform=='win32',"This test does not run under Windows")
+skip_linux = skipif(sys.platform=='linux2',"This test does not run under Linux")
+skip_osx = skipif(sys.platform=='darwin',"This test does not run under OSX")

IPython/testing/plugin/Makefile

@@ -1,6 +1,5 b''
 # Set this prefix to where you want to install the plugin
-PREFIX=~/usr/local
-PREFIX=~/tmp/local
+PREFIX=/usr/local
 
 NOSE0=nosetests -vs --with-doctest --doctest-tests --detailed-errors
 NOSE=nosetests -vvs --with-ipdoctest --doctest-tests --doctest-extension=txt \

IPython/testing/plugin/test_refs.py

@@ -1,152 +1,19 b''
+"""Some simple tests for the plugin while running scripts.
+"""
 # Module imports
 # Std lib
 import inspect
 
-# Third party
-
 # Our own
 from IPython.testing import decorators as dec
 
 #-----------------------------------------------------------------------------
-# Utilities
-
-# Note: copied from OInspect, kept here so the testing stuff doesn't create
-# circular dependencies and is easier to reuse.
-def getargspec(obj):
-    """Get the names and default values of a function's arguments.
-
-    A tuple of four things is returned: (args, varargs, varkw, defaults).
-    'args' is a list of the argument names (it may contain nested lists).
-    'varargs' and 'varkw' are the names of the * and ** arguments or None.
-    'defaults' is an n-tuple of the default values of the last n arguments.
-
-    Modified version of inspect.getargspec from the Python Standard
-    Library."""
-
-    if inspect.isfunction(obj):
-        func_obj = obj
-    elif inspect.ismethod(obj):
-        func_obj = obj.im_func
-    else:
-        raise TypeError, 'arg is not a Python function'
-    args, varargs, varkw = inspect.getargs(func_obj.func_code)
-    return args, varargs, varkw, func_obj.func_defaults
-
-#-----------------------------------------------------------------------------
 # Testing functions
 
 def test_trivial():
     """A trivial passing test."""
     pass
 
-
[e-mail]
-def test_deliberately_broken():
-    """A deliberately broken test - we want to skip this one."""
-    1/0
-
[e-mail]('foo')
-def test_deliberately_broken2():
-    """Another deliberately broken test - we want to skip this one."""
-    1/0
-
-
-# Verify that we can correctly skip the doctest for a function at will, but
-# that the docstring itself is NOT destroyed by the decorator.
[e-mail]_doctest
-def doctest_bad(x,y=1,**k):
-    """A function whose doctest we need to skip.
-
-    >>> 1+1
-    3
-    """
-    print 'x:',x
-    print 'y:',y
-    print 'k:',k
-
-
-def call_doctest_bad():
-    """Check that we can still call the decorated functions.
-    
-    >>> doctest_bad(3,y=4)
-    x: 3
-    y: 4
-    k: {}
-    """
-    pass
-
-
-# Doctest skipping should work for class methods too
-class foo(object):
-    """Foo
-
-    Example:
-
-    >>> 1+1
-    2
-    """
-
-    @dec.skip_doctest
-    def __init__(self,x):
-        """Make a foo.
-
-        Example:
-
-        >>> f = foo(3)
-        junk
-        """
-        print 'Making a foo.'
-        self.x = x
-        
-    @dec.skip_doctest
-    def bar(self,y):
-        """Example:
-
-        >>> f = foo(3)
-        >>> f.bar(0)
-        boom!
-        >>> 1/0
-        bam!
-        """
-        return 1/y
-
-    def baz(self,y):
-        """Example:
-
-        >>> f = foo(3)
-        Making a foo.
-        >>> f.baz(3)
-        True
-        """
-        return self.x==y
-
-
-def test_skip_dt_decorator():
-    """Doctest-skipping decorator should preserve the docstring.
-    """
-    # Careful: 'check' must be a *verbatim* copy of the doctest_bad docstring!
-    check = """A function whose doctest we need to skip.
-
-    >>> 1+1
-    3
-    """
-    # Fetch the docstring from doctest_bad after decoration.
-    val = doctest_bad.__doc__
-    
-    assert check==val,"doctest_bad docstrings don't match"
-
-
-def test_skip_dt_decorator2():
-    """Doctest-skipping decorator should preserve function signature.
-    """
-    # Hardcoded correct answer
-    dtargs = (['x', 'y'], None, 'k', (1,))
-    # Introspect out the value
-    dtargsr = getargspec(doctest_bad)
-    assert dtargsr==dtargs, \
-           "Incorrectly reconstructed args for doctest_bad: %s" % (dtargsr,)
-
-
 def doctest_run():
     """Test running a trivial script.
 
@@ -154,7 +21,6 b' def doctest_run():'
     x is: 1
     """
     
-#@dec.skip_doctest
 def doctest_runvars():
     """Test that variables defined in scripts get loaded correcly via %run.
 

README.txt

@@ -1,11 +1,11 b''
-===============
-IPython1 README
-===============
- 
-.. contents::
+==============
+IPython README
+==============
 
 Overview
 ========
 
-Welcome to IPython.  New users should consult our documentation, which can be found
-in the docs/source subdirectory.
+Welcome to IPython. Our documentation can be found in the docs/source
+subdirectory. We also have ``.html`` and ``.pdf`` versions of this
+documentation available on the IPython `website <http://ipython.scipy.org>`_.
+

docs/examples/kernel/nwmerge.py

@@ -45,7 +45,10 b' def mergesort(list_of_lists, key=None):'
     for i, itr in enumerate(iter(pl) for pl in list_of_lists):
         try:
             item = itr.next()
-            toadd = (key(item), i, item, itr) if key else (item, i, itr)
+            if key:
+                toadd = (key(item), i, item, itr)
+            else:
+                toadd = (item, i, itr)
             heap.append(toadd)
         except StopIteration:
             pass

docs/source/conf.py

@@ -1,7 +1,6 b''
 # -*- coding: utf-8 -*-
 #
-# IPython documentation build configuration file, created by
-# sphinx-quickstart on Thu May  8 16:45:02 2008.
+# IPython documentation build configuration file.
 
 # NOTE: This file has been edited manually from the auto-generated one from
 # sphinx.  Do NOT delete and re-generate.  If any changes from sphinx are
@@ -21,7 +20,11 b' import sys, os'
 # If your extensions are in another directory, add it here. If the directory
 # is relative to the documentation root, use os.path.abspath to make it
 # absolute, like shown here.
-#sys.path.append(os.path.abspath('some/directory'))
+sys.path.append(os.path.abspath('../sphinxext'))
+
+# Import support for ipython console session syntax highlighting (lives
+# in the sphinxext directory defined above)
+import ipython_console_highlighting
 
 # We load the ipython release info into a dict by explicit execution
 iprelease = {}
@@ -32,7 +35,11 b" execfile('../../IPython/Release.py',iprelease)"
 
 # Add any Sphinx extension module names here, as strings. They can be extensions
 # coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
-#extensions = []
+extensions = ['sphinx.ext.autodoc',
+              'inheritance_diagram', 'only_directives',
+              'ipython_console_highlighting', 
+              # 'plot_directive', # disabled for now, needs matplotlib
+              ]
 
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ['_templates']
@@ -67,7 +74,7 b" today_fmt = '%B %d, %Y'"
 
 # List of directories, relative to source directories, that shouldn't be searched
 # for source files.
-#exclude_dirs = []
+exclude_dirs = ['attic']
 
 # If true, '()' will be appended to :func: etc. cross-reference text.
 #add_function_parentheses = True
@@ -135,7 +142,7 b" html_last_updated_fmt = '%b %d, %Y'"
 #html_file_suffix = ''
 
 # Output file base name for HTML help builder.
-htmlhelp_basename = 'IPythondoc'
+htmlhelp_basename = 'ipythondoc'
 
 
 # Options for LaTeX output
@@ -151,10 +158,7 b" latex_font_size = '11pt'"
 # (source start file, target name, title, author, document class [howto/manual]).
 
 latex_documents = [ ('index', 'ipython.tex', 'IPython Documentation',
-                     ur"""Brian Granger, Fernando Pérez and Ville Vainio\\
-                     \ \\
-                     With contributions from:\\
-                     Benjamin Ragan-Kelley and Barry Wark.""",
+                     ur"""The IPython Development Team""",
                      'manual'),
                     ]
 
@@ -176,5 +180,8 b" latex_documents = [ ('index', 'ipython.tex', 'IPython Documentation',"
 #latex_use_modindex = True
 
 
-# Cleanup: delete release info to avoid pickling errors from sphinx
+# Cleanup
+# -------
+# delete release info to avoid pickling errors from sphinx
+
 del iprelease

docs/source/config/customization.txt

@@ -18,6 +18,8 b' time. A hybrid approach of specifying a few options in ipythonrc and'
 doing the more advanced configuration in ipy_user_conf.py is also
 possible.
 
+.. _ipythonrc:
+
 The ipythonrc approach
 ======================
 
@@ -36,11 +38,11 b' fairly primitive). Note that these are not python files, and this is'
 deliberate, because it allows us to do some things which would be quite
 tricky to implement if they were normal python files.
 
-First, an rcfile can contain permanent default values for almost all
-command line options (except things like -help or -Version). Sec
-`command line options`_ contains a description of all command-line
-options. However, values you explicitly specify at the command line
-override the values defined in the rcfile.
+First, an rcfile can contain permanent default values for almost all command
+line options (except things like -help or -Version). :ref:`This section
+<command_line_options>` contains a description of all command-line
+options. However, values you explicitly specify at the command line override
+the values defined in the rcfile.
 
 Besides command line option values, the rcfile can specify values for
 certain extra special options which are not available at the command
@@ -266,13 +268,13 b' which look like this::'
 IPython profiles
 ================
 
-As we already mentioned, IPython supports the -profile command-line
-option (see sec. `command line options`_). A profile is nothing more
-than a particular configuration file like your basic ipythonrc one,
-but with particular customizations for a specific purpose. When you
-start IPython with 'ipython -profile <name>', it assumes that in your
-IPYTHONDIR there is a file called ipythonrc-<name> or
-ipy_profile_<name>.py, and loads it instead of the normal ipythonrc.
+As we already mentioned, IPython supports the -profile command-line option (see
+:ref:`here <command_line_options>`).  A profile is nothing more than a
+particular configuration file like your basic ipythonrc one, but with
+particular customizations for a specific purpose. When you start IPython with
+'ipython -profile <name>', it assumes that in your IPYTHONDIR there is a file
+called ipythonrc-<name> or ipy_profile_<name>.py, and loads it instead of the
+normal ipythonrc.
 
 This system allows you to maintain multiple configurations which load
 modules, set options, define functions, etc. suitable for different

docs/source/config/initial_config.txt

@@ -11,28 +11,27 b' in a directory named by default $HOME/.ipython. You can change this by'
 defining the environment variable IPYTHONDIR, or at runtime with the
 command line option -ipythondir.
 
-If all goes well, the first time you run IPython it should
-automatically create a user copy of the config directory for you,
-based on its builtin defaults. You can look at the files it creates to
-learn more about configuring the system. The main file you will modify
-to configure IPython's behavior is called ipythonrc (with a .ini
-extension under Windows), included for reference in `ipythonrc`_
-section. This file is very commented and has many variables you can
-change to suit your taste, you can find more details in
-Sec. customization_. Here we discuss the basic things you will want to
-make sure things are working properly from the beginning.
+If all goes well, the first time you run IPython it should automatically create
+a user copy of the config directory for you, based on its builtin defaults. You
+can look at the files it creates to learn more about configuring the
+system. The main file you will modify to configure IPython's behavior is called
+ipythonrc (with a .ini extension under Windows), included for reference
+:ref:`here <ipythonrc>`. This file is very commented and has many variables you
+can change to suit your taste, you can find more details :ref:`here
+<customization>`. Here we discuss the basic things you will want to make sure
+things are working properly from the beginning.
 
 
-.. _Accessing help:
+.. _accessing_help:
 
 Access to the Python help system
 ================================
 
-This is true for Python in general (not just for IPython): you should
-have an environment variable called PYTHONDOCS pointing to the directory
-where your HTML Python documentation lives. In my system it's
-/usr/share/doc/python-docs-2.3.4/html, check your local details or ask
-your systems administrator.
+This is true for Python in general (not just for IPython): you should have an
+environment variable called PYTHONDOCS pointing to the directory where your
+HTML Python documentation lives. In my system it's
+:file:`/usr/share/doc/python-doc/html`, check your local details or ask your
+systems administrator.
 
 This is the directory which holds the HTML version of the Python
 manuals. Unfortunately it seems that different Linux distributions
@@ -40,8 +39,9 b' package these files differently, so you may have to look around a bit.'
 Below I show the contents of this directory on my system for reference::
 
     [html]> ls
-    about.dat acks.html dist/ ext/ index.html lib/ modindex.html
-    stdabout.dat tut/ about.html api/ doc/ icons/ inst/ mac/ ref/ style.css
+    about.html  dist/  icons/      lib/           python2.5.devhelp.gz  whatsnew/
+    acks.html   doc/   index.html  mac/           ref/
+    api/        ext/   inst/       modindex.html  tut/
 
 You should really make sure this variable is correctly set so that
 Python's pydoc-based help system works. It is a powerful and convenient
@@ -108,6 +108,8 b' The following terminals seem to handle the color sequences fine:'
       support under cygwin, please post to the IPython mailing list so
       this issue can be resolved for all users.
 
+.. _pyreadline: https://code.launchpad.net/pyreadline
+      
 These have shown problems:
 
     * Windows command prompt in WinXP/2k logged into a Linux machine via
@@ -157,13 +159,12 b' $HOME/.ipython/ipythonrc and set the colors option to the desired value.'
 Object details (types, docstrings, source code, etc.)
 =====================================================
 
-IPython has a set of special functions for studying the objects you
-are working with, discussed in detail in Sec. `dynamic object
-information`_. But this system relies on passing information which is
-longer than your screen through a data pager, such as the common Unix
-less and more programs. In order to be able to see this information in
-color, your pager needs to be properly configured. I strongly
-recommend using less instead of more, as it seems that more simply can
+IPython has a set of special functions for studying the objects you are working
+with, discussed in detail :ref:`here <dynamic_object_info>`. But this system
+relies on passing information which is longer than your screen through a data
+pager, such as the common Unix less and more programs. In order to be able to
+see this information in color, your pager needs to be properly configured. I
+strongly recommend using less instead of more, as it seems that more simply can
 not understand colored text correctly.
 
 In order to configure less as your default pager, do the following:

docs/source/credits.txt

@@ -4,24 +4,39 b''
 Credits
 =======
 
-IPython is mainly developed by Fernando Pérez
-<[e-mail]>, but the project was born from mixing in
-Fernando's code with the IPP project by Janko Hauser
-<jhauser-AT-zscout.de> and LazyPython by Nathan Gray
-<n8gray-AT-caltech.edu>. For all IPython-related requests, please
-contact Fernando.
-
-As of early 2006, the following developers have joined the core team:
-
-    * [Robert Kern] <rkern-AT-enthought.com>: co-mentored the 2005
-      Google Summer of Code project to develop python interactive
-      notebooks (XML documents) and graphical interface. This project
-      was awarded to the students Tzanko Matev <tsanko-AT-gmail.com> and
-      Toni Alatalo <antont-AT-an.org>
-    * [Brian Granger] <bgranger-AT-scu.edu>: extending IPython to allow
-      support for interactive parallel computing.
-    * [Ville Vainio] <vivainio-AT-gmail.com>: Ville is the new
-      maintainer for the main trunk of IPython after version 0.7.1.
+IPython is led by Fernando Pérez.
+
+As of this writing, the following developers have joined the core team:
+
+* [Robert Kern] <rkern-AT-enthought.com>: co-mentored the 2005
+  Google Summer of Code project to develop python interactive
+  notebooks (XML documents) and graphical interface. This project
+  was awarded to the students Tzanko Matev <tsanko-AT-gmail.com> and
+  Toni Alatalo <antont-AT-an.org>.
+
+* [Brian Granger] <ellisonbg-AT-gmail.com>: extending IPython to allow
+  support for interactive parallel computing.
+
+* [Benjamin (Min) Ragan-Kelley]: key work on IPython's parallel
+  computing infrastructure.
+
+* [Ville Vainio] <vivainio-AT-gmail.com>: Ville has made many improvements
+  to the core of IPython and was the maintainer of the main IPython
+  trunk from version 0.7.1 to 0.8.4.
+
+* [Gael Varoquaux] <gael.varoquaux-AT-normalesup.org>: work on the merged
+  architecture for the interpreter as of version 0.9, implementing a new WX GUI
+  based on this system.
+
+* [Barry Wark] <barrywark-AT-gmail.com>: implementing a new Cocoa GUI, as well
+  as work on the new interpreter architecture and Twisted support.
+
+* [Laurent Dufrechou] <laurent.dufrechou-AT-gmail.com>: development of the WX
+  GUI support.
+
+* [Jörgen Stenarson] <jorgen.stenarson-AT-bostream.nu>: maintainer of the
+  PyReadline project, necessary for IPython under windows.
+
 
 The IPython project is also very grateful to:
 
@@ -50,90 +65,143 b" an O'Reilly Python editor. His Oct/11/2001 article about IPP and"
 LazyPython, was what got this project started. You can read it at:
 http://www.onlamp.com/pub/a/python/2001/10/11/pythonnews.html.
 
-And last but not least, all the kind IPython users who have emailed new
-code, bug reports, fixes, comments and ideas. A brief list follows,
-please let me know if I have ommitted your name by accident:
-
-    * [Jack Moffit] <jack-AT-xiph.org> Bug fixes, including the infamous
-      color problem. This bug alone caused many lost hours and
-      frustration, many thanks to him for the fix. I've always been a
-      fan of Ogg & friends, now I have one more reason to like these folks.
-      Jack is also contributing with Debian packaging and many other
-      things.
-    * [Alexander Schmolck] <a.schmolck-AT-gmx.net> Emacs work, bug
-      reports, bug fixes, ideas, lots more. The ipython.el mode for
-      (X)Emacs is Alex's code, providing full support for IPython under
-      (X)Emacs.
-    * [Andrea Riciputi] <andrea.riciputi-AT-libero.it> Mac OSX
-      information, Fink package management.
-    * [Gary Bishop] <gb-AT-cs.unc.edu> Bug reports, and patches to work
-      around the exception handling idiosyncracies of WxPython. Readline
-      and color support for Windows.
-    * [Jeffrey Collins] <Jeff.Collins-AT-vexcel.com> Bug reports. Much
-      improved readline support, including fixes for Python 2.3.
-    * [Dryice Liu] <dryice-AT-liu.com.cn> FreeBSD port.
-    * [Mike Heeter] <korora-AT-SDF.LONESTAR.ORG>
-    * [Christopher Hart] <hart-AT-caltech.edu> PDB integration.
-    * [Milan Zamazal] <pdm-AT-zamazal.org> Emacs info.
-    * [Philip Hisley] <compsys-AT-starpower.net>
-    * [Holger Krekel] <pyth-AT-devel.trillke.net> Tab completion, lots
-      more.
-    * [Robin Siebler] <robinsiebler-AT-starband.net>
-    * [Ralf Ahlbrink] <ralf_ahlbrink-AT-web.de>
-    * [Thorsten Kampe] <thorsten-AT-thorstenkampe.de>
-    * [Fredrik Kant] <fredrik.kant-AT-front.com> Windows setup.
-    * [Syver Enstad] <syver-en-AT-online.no> Windows setup.
-    * [Richard] <rxe-AT-renre-europe.com> Global embedding.
-    * [Hayden Callow] <h.callow-AT-elec.canterbury.ac.nz> Gnuplot.py 1.6
-      compatibility.
-    * [Leonardo Santagada] <retype-AT-terra.com.br> Fixes for Windows
-      installation.
-    * [Christopher Armstrong] <radix-AT-twistedmatrix.com> Bugfixes.
-    * [Francois Pinard] <pinard-AT-iro.umontreal.ca> Code and
-      documentation fixes.
-    * [Cory Dodt] <cdodt-AT-fcoe.k12.ca.us> Bug reports and Windows
-      ideas. Patches for Windows installer.
-    * [Olivier Aubert] <oaubert-AT-bat710.univ-lyon1.fr> New magics.
-    * [King C. Shu] <kingshu-AT-myrealbox.com> Autoindent patch.
-    * [Chris Drexler] <chris-AT-ac-drexler.de> Readline packages for
-      Win32/CygWin.
-    * [Gustavo Cordova Avila] <gcordova-AT-sismex.com> EvalDict code for
-      nice, lightweight string interpolation.
-    * [Kasper Souren] <Kasper.Souren-AT-ircam.fr> Bug reports, ideas.
-    * [Gever Tulley] <gever-AT-helium.com> Code contributions.
-    * [Ralf Schmitt] <ralf-AT-brainbot.com> Bug reports & fixes.
-    * [Oliver Sander] <osander-AT-gmx.de> Bug reports.
-    * [Rod Holland] <rhh-AT-structurelabs.com> Bug reports and fixes to
-      logging module.
-    * [Daniel 'Dang' Griffith] <pythondev-dang-AT-lazytwinacres.net>
-      Fixes, enhancement suggestions for system shell use.
-    * [Viktor Ransmayr] <viktor.ransmayr-AT-t-online.de> Tests and
-      reports on Windows installation issues. Contributed a true Windows
-      binary installer.
-    * [Mike Salib] <msalib-AT-mit.edu> Help fixing a subtle bug related
-      to traceback printing.
-    * [W.J. van der Laan] <gnufnork-AT-hetdigitalegat.nl> Bash-like
-      prompt specials.
-    * [Antoon Pardon] <Antoon.Pardon-AT-rece.vub.ac.be> Critical fix for
-      the multithreaded IPython.
-    * [John Hunter] <jdhunter-AT-nitace.bsd.uchicago.edu> Matplotlib
-      author, helped with all the development of support for matplotlib
-      in IPyhton, including making necessary changes to matplotlib itself.
-    * [Matthew Arnison] <maffew-AT-cat.org.au> Bug reports, '%run -d' idea.
-    * [Prabhu Ramachandran] <prabhu_r-AT-users.sourceforge.net> Help
-      with (X)Emacs support, threading patches, ideas...
-    * [Norbert Tretkowski] <tretkowski-AT-inittab.de> help with Debian
-      packaging and distribution.
-    * [George Sakkis] <gsakkis-AT-eden.rutgers.edu> New matcher for
-      tab-completing named arguments of user-defined functions.
-    * [Jörgen Stenarson] <jorgen.stenarson-AT-bostream.nu> Wildcard
-      support implementation for searching namespaces.
-    * [Vivian De Smedt] <vivian-AT-vdesmedt.com> Debugger enhancements,
-      so that when pdb is activated from within IPython, coloring, tab
-      completion and other features continue to work seamlessly.
-    * [Scott Tsai] <scottt958-AT-yahoo.com.tw> Support for automatic
-      editor invocation on syntax errors (see
-      http://www.scipy.net/roundup/ipython/issue36).
-    * [Alexander Belchenko] <bialix-AT-ukr.net> Improvements for win32
-      paging system.
-    * [Will Maier] <willmaier-AT-ml1.net> Official OpenBSD port.
  No newline at end of file
+And last but not least, all the kind IPython users who have emailed new code,
+bug reports, fixes, comments and ideas. A brief list follows, please let us
+know if we have ommitted your name by accident:
+
+* Dan Milstein <danmil-AT-comcast.net>.  A bold refactoring of the 
+  core prefilter stuff in the IPython interpreter.
+
+* [Jack Moffit] <jack-AT-xiph.org> Bug fixes, including the infamous
+  color problem. This bug alone caused many lost hours and
+  frustration, many thanks to him for the fix. I've always been a
+  fan of Ogg & friends, now I have one more reason to like these folks.
+  Jack is also contributing with Debian packaging and many other
+  things.
+ 
+* [Alexander Schmolck] <a.schmolck-AT-gmx.net> Emacs work, bug
+  reports, bug fixes, ideas, lots more. The ipython.el mode for
+  (X)Emacs is Alex's code, providing full support for IPython under
+  (X)Emacs.
+
+* [Andrea Riciputi] <andrea.riciputi-AT-libero.it> Mac OSX
+  information, Fink package management.
+
+* [Gary Bishop] <gb-AT-cs.unc.edu> Bug reports, and patches to work
+  around the exception handling idiosyncracies of WxPython. Readline
+  and color support for Windows.
+
+* [Jeffrey Collins] <Jeff.Collins-AT-vexcel.com> Bug reports. Much
+  improved readline support, including fixes for Python 2.3.
+
+* [Dryice Liu] <dryice-AT-liu.com.cn> FreeBSD port.
+
+* [Mike Heeter] <korora-AT-SDF.LONESTAR.ORG>
+
+* [Christopher Hart] <hart-AT-caltech.edu> PDB integration.
+
+* [Milan Zamazal] <pdm-AT-zamazal.org> Emacs info.
+
+* [Philip Hisley] <compsys-AT-starpower.net>
+
+* [Holger Krekel] <pyth-AT-devel.trillke.net> Tab completion, lots
+  more.
+
+* [Robin Siebler] <robinsiebler-AT-starband.net>
+
+* [Ralf Ahlbrink] <ralf_ahlbrink-AT-web.de>
+
+* [Thorsten Kampe] <thorsten-AT-thorstenkampe.de>
+
+* [Fredrik Kant] <fredrik.kant-AT-front.com> Windows setup.
+
+* [Syver Enstad] <syver-en-AT-online.no> Windows setup.
+
+* [Richard] <rxe-AT-renre-europe.com> Global embedding.
+
+* [Hayden Callow] <h.callow-AT-elec.canterbury.ac.nz> Gnuplot.py 1.6
+  compatibility.
+  
+* [Leonardo Santagada] <retype-AT-terra.com.br> Fixes for Windows
+  installation.
+  
+* [Christopher Armstrong] <radix-AT-twistedmatrix.com> Bugfixes.
+
+* [Francois Pinard] <pinard-AT-iro.umontreal.ca> Code and
+  documentation fixes.
+  
+* [Cory Dodt] <cdodt-AT-fcoe.k12.ca.us> Bug reports and Windows
+  ideas. Patches for Windows installer.
+  
+* [Olivier Aubert] <oaubert-AT-bat710.univ-lyon1.fr> New magics.
+
+* [King C. Shu] <kingshu-AT-myrealbox.com> Autoindent patch.
+
+* [Chris Drexler] <chris-AT-ac-drexler.de> Readline packages for
+  Win32/CygWin.
+  
+* [Gustavo Cordova Avila] <gcordova-AT-sismex.com> EvalDict code for
+  nice, lightweight string interpolation.
+  
+* [Kasper Souren] <Kasper.Souren-AT-ircam.fr> Bug reports, ideas.
+
+* [Gever Tulley] <gever-AT-helium.com> Code contributions.
+
+* [Ralf Schmitt] <ralf-AT-brainbot.com> Bug reports & fixes.
+
+* [Oliver Sander] <osander-AT-gmx.de> Bug reports.
+
+* [Rod Holland] <rhh-AT-structurelabs.com> Bug reports and fixes to
+  logging module.
+
+* [Daniel 'Dang' Griffith] <pythondev-dang-AT-lazytwinacres.net>
+  Fixes, enhancement suggestions for system shell use.
+
+* [Viktor Ransmayr] <viktor.ransmayr-AT-t-online.de> Tests and
+  reports on Windows installation issues. Contributed a true Windows
+  binary installer.
+
+* [Mike Salib] <msalib-AT-mit.edu> Help fixing a subtle bug related
+  to traceback printing.
+
+* [W.J. van der Laan] <gnufnork-AT-hetdigitalegat.nl> Bash-like
+  prompt specials.
+
+* [Antoon Pardon] <Antoon.Pardon-AT-rece.vub.ac.be> Critical fix for
+  the multithreaded IPython.
+
+* [John Hunter] <jdhunter-AT-nitace.bsd.uchicago.edu> Matplotlib
+  author, helped with all the development of support for matplotlib
+  in IPyhton, including making necessary changes to matplotlib itself.
+
+* [Matthew Arnison] <maffew-AT-cat.org.au> Bug reports, '%run -d' idea.
+
+* [Prabhu Ramachandran] <prabhu_r-AT-users.sourceforge.net> Help
+  with (X)Emacs support, threading patches, ideas...
+
+* [Norbert Tretkowski] <tretkowski-AT-inittab.de> help with Debian
+  packaging and distribution.
+
+* [George Sakkis] <gsakkis-AT-eden.rutgers.edu> New matcher for
+  tab-completing named arguments of user-defined functions.
+
+* [Jörgen Stenarson] <jorgen.stenarson-AT-bostream.nu> Wildcard
+  support implementation for searching namespaces.
+
+* [Vivian De Smedt] <vivian-AT-vdesmedt.com> Debugger enhancements,
+  so that when pdb is activated from within IPython, coloring, tab
+  completion and other features continue to work seamlessly.
+
+* [Scott Tsai] <scottt958-AT-yahoo.com.tw> Support for automatic
+  editor invocation on syntax errors (see
+  http://www.scipy.net/roundup/ipython/issue36).
+
+* [Alexander Belchenko] <bialix-AT-ukr.net> Improvements for win32
+  paging system.
+
+* [Will Maier] <willmaier-AT-ml1.net> Official OpenBSD port.
+
+* [Ondrej Certik] <ondrej-AT-certik.cz>: set up the IPython docs to use the new
+  Sphinx system used by Python, Matplotlib and many more projects.
+
+* [Stefan van der Walt] <stefan-AT-sun.ac.za>: support for the new config system.

docs/source/development/development.txt

@@ -1,10 +1,8 b''
 .. _development:
 
-==================================
+==============================
 IPython development guidelines
-==================================
-
-.. contents::
+==============================
 
 
 Overview
@@ -24,11 +22,11 b' There are two, no three, main goals of the IPython effort:'
    to be used from within a variety of GUI applications.
 3. Implement a system for interactive parallel computing.
 
-While the third goal may seem a bit unrelated to the main focus of IPython, it turns
-out that the technologies required for this goal are nearly identical with those
-required for goal two. This is the main reason the interactive parallel computing
-capabilities are being put into IPython proper. Currently the third of these goals is
-furthest along.
+While the third goal may seem a bit unrelated to the main focus of IPython, it
+turns out that the technologies required for this goal are nearly identical
+with those required for goal two. This is the main reason the interactive
+parallel computing capabilities are being put into IPython proper. Currently
+the third of these goals is furthest along.
 
 This document describes IPython from the perspective of developers.  
 
@@ -39,51 +37,59 b' Project organization'
 Subpackages
 -----------
 
-IPython is organized into semi self-contained subpackages.  Each of the subpackages will have its own:
+IPython is organized into semi self-contained subpackages.  Each of the
+subpackages will have its own:
 
 - **Dependencies**.  One of the most important things to keep in mind in
   partitioning code amongst subpackages, is that they should be used to cleanly
-  encapsulate dependencies.  
+  encapsulate dependencies.
+  
 - **Tests**.  Each subpackage shoud have its own ``tests`` subdirectory that
-  contains all of the tests for that package.  For information about writing tests
-  for IPython, see the `Testing System`_ section of this document.
-- **Configuration**.  Each subpackage should have its own ``config`` subdirectory
-  that contains the configuration information for the components of the 
-  subpackage.  For information about how the IPython configuration system
-  works, see the `Configuration System`_ section of this document.
-- **Scripts**.  Each subpackage should have its own ``scripts`` subdirectory that
-  contains all of the command line scripts associated with the subpackage.
+  contains all of the tests for that package.  For information about writing
+  tests for IPython, see the `Testing System`_ section of this document.
+  
+- **Configuration**.  Each subpackage should have its own ``config``
+  subdirectory that contains the configuration information for the components
+  of the subpackage.  For information about how the IPython configuration
+  system works, see the `Configuration System`_ section of this document.
+  
+- **Scripts**.  Each subpackage should have its own ``scripts`` subdirectory
+  that contains all of the command line scripts associated with the subpackage.
 
 Installation and dependencies
 -----------------------------
    
-IPython will not use `setuptools`_ for installation. Instead, we will use standard
-``setup.py`` scripts that use `distutils`_. While there are a number a extremely nice
-features that `setuptools`_ has (like namespace packages), the current implementation
-of `setuptools`_ has performance problems, particularly on shared file systems. In
-particular, when Python packages are installed on NSF file systems, import times
-become much too long (up towards 10 seconds). 
+IPython will not use `setuptools`_ for installation. Instead, we will use
+standard ``setup.py`` scripts that use `distutils`_. While there are a number a
+extremely nice features that `setuptools`_ has (like namespace packages), the
+current implementation of `setuptools`_ has performance problems, particularly
+on shared file systems. In particular, when Python packages are installed on
+NSF file systems, import times become much too long (up towards 10 seconds).
 
 Because IPython is being used extensively in the context of high performance
-computing, where performance is critical but shared file systems are common, we feel
-these performance hits are not acceptable. Thus, until the performance problems
-associated with `setuptools`_ are addressed, we will stick with plain `distutils`_. We
-are hopeful that these problems will be addressed and that we will eventually begin
-using `setuptools`_. Because of this, we are trying to organize IPython in a way that
-will make the eventual transition to `setuptools`_ as painless as possible.
-
-Because we will be using `distutils`_, there will be no method for automatically installing dependencies.  Instead, we are following the approach of `Matplotlib`_ which can be summarized as follows:
+computing, where performance is critical but shared file systems are common, we
+feel these performance hits are not acceptable. Thus, until the performance
+problems associated with `setuptools`_ are addressed, we will stick with plain
+`distutils`_. We are hopeful that these problems will be addressed and that we
+will eventually begin using `setuptools`_. Because of this, we are trying to
+organize IPython in a way that will make the eventual transition to
+`setuptools`_ as painless as possible.
+
+Because we will be using `distutils`_, there will be no method for
+automatically installing dependencies.  Instead, we are following the approach
+of `Matplotlib`_ which can be summarized as follows:
 
 - Distinguish between required and optional dependencies.  However, the required
   dependencies for IPython should be only the Python standard library.
-- Upon installation check to see which optional dependencies are present and tell
-  the user which parts of IPython need which optional dependencies.
+  
+- Upon installation check to see which optional dependencies are present and
+  tell the user which parts of IPython need which optional dependencies.
 
-It is absolutely critical that each subpackage of IPython has a clearly specified set
-of dependencies and that dependencies are not carelessly inherited from other IPython
-subpackages. Furthermore, tests that have certain dependencies should not fail if
-those dependencies are not present. Instead they should be skipped and print a
-message.
+It is absolutely critical that each subpackage of IPython has a clearly
+specified set of dependencies and that dependencies are not carelessly
+inherited from other IPython subpackages. Furthermore, tests that have certain
+dependencies should not fail if those dependencies are not present. Instead
+they should be skipped and print a message.
 
 .. _setuptools: http://peak.telecommunity.com/DevCenter/setuptools
 .. _distutils: http://docs.python.org/lib/module-distutils.html
@@ -106,9 +112,10 b' Specific subpackages'
 
 ``frontends``
     The various frontends for IPython.  A frontend is the end-user application
-    that exposes the capabilities of IPython to the user.  The most basic frontend
-    will simply be a terminal based application that looks just like today 's 
-    IPython.  Other frontends will likely be more powerful and based on GUI toolkits.
+    that exposes the capabilities of IPython to the user.  The most basic
+    frontend will simply be a terminal based application that looks just like
+    today 's IPython.  Other frontends will likely be more powerful and based
+    on GUI toolkits.
 
 ``notebook``
     An application that allows users to work with IPython notebooks.
@@ -120,10 +127,12 b' Specific subpackages'
 Version control
 ===============
 
-In the past, IPython development has been done using `Subversion`__.  Recently, we made the transition to using `Bazaar`__ and `Launchpad`__.  This makes it much easier for people
-to contribute code to IPython.  Here is a sketch of how to use Bazaar for IPython
-development.  First, you should install Bazaar.  After you have done that, make
-sure that it is working by getting the latest main branch of IPython::
+In the past, IPython development has been done using `Subversion`__.  Recently,
+we made the transition to using `Bazaar`__ and `Launchpad`__.  This makes it
+much easier for people to contribute code to IPython.  Here is a sketch of how
+to use Bazaar for IPython development.  First, you should install Bazaar.
+After you have done that, make sure that it is working by getting the latest
+main branch of IPython::
 
     $ bzr branch lp:ipython
 
@@ -131,17 +140,17 b' Now you can create a new branch for you to do your work in::'
 
     $ bzr branch ipython ipython-mybranch
 
-The typical work cycle in this branch will be to make changes in `ipython-mybranch`
-and then commit those changes using the commit command::
+The typical work cycle in this branch will be to make changes in
+``ipython-mybranch`` and then commit those changes using the commit command::
 
     $ ...do work in ipython-mybranch...
     $ bzr ci -m "the commit message goes here"
 
-Please note that since we now don't use an old-style linear ChangeLog
-(that tends to cause problems with distributed version control
-systems), you should ensure that your log messages are reasonably
-detailed.  Use a docstring-like approach in the commit messages
-(including the second line being left *blank*)::
+Please note that since we now don't use an old-style linear ChangeLog (that
+tends to cause problems with distributed version control systems), you should
+ensure that your log messages are reasonably detailed.  Use a docstring-like
+approach in the commit messages (including the second line being left
+*blank*)::
 
   Single line summary of  changes being committed.
 
@@ -149,27 +158,27 b' detailed. Use a docstring-like approach in the commit messages'
   - including crediting outside contributors if they sent the
     code/bug/idea!
 
-If we couple this with a policy of making single commits for each
-reasonably atomic change, the bzr log should give an excellent view of
-the project, and the `--short` log option becomes a nice summary.
+If we couple this with a policy of making single commits for each reasonably
+atomic change, the bzr log should give an excellent view of the project, and
+the `--short` log option becomes a nice summary.
 
-While working with this branch, it is a good idea to merge in changes that have been
-made upstream in the parent branch.  This can be done by doing::
+While working with this branch, it is a good idea to merge in changes that have
+been made upstream in the parent branch.  This can be done by doing::
 
     $ bzr pull
     
-If this command shows that the branches have diverged, then you should do a merge
-instead::
+If this command shows that the branches have diverged, then you should do a
+merge instead::
 
     $ bzr merge lp:ipython
 
-If you want others to be able to see your branch, you can create an account with
-launchpad and push the branch to your own workspace::
+If you want others to be able to see your branch, you can create an account
+with launchpad and push the branch to your own workspace::
 
     $ bzr push bzr+ssh://<me>@bazaar.launchpad.net/~<me>/+junk/ipython-mybranch
 
-Finally, once the work in your branch is done, you can merge your changes back into
-the `ipython` branch by using merge::
+Finally, once the work in your branch is done, you can merge your changes back
+into the `ipython` branch by using merge::
 
     $ cd ipython
     $ merge ../ipython-mybranch
@@ -177,8 +186,9 b' the `ipython` branch by using merge::'
     $ bzr ci -m "Fixing that bug"
     $ bzr push
 
-But this will require you to have write permissions to the `ipython` branch.  It you don't
-you can tell one of the IPython devs about your branch and they can do the merge for you.
+But this will require you to have write permissions to the `ipython` branch.
+It you don't you can tell one of the IPython devs about your branch and they
+can do the merge for you.
 
 More information about Bazaar workflows can be found `here`__.
 
@@ -193,27 +203,29 b' Documentation'
 Standalone documentation
 ------------------------
 
-All standalone documentation should be written in plain text (``.txt``) files using
-`reStructuredText`_ for markup and formatting. All such documentation should be placed
-in the top level directory ``docs`` of the IPython source tree. Or, when appropriate,
-a suitably named subdirectory should be used. The documentation in this location will
-serve as the main source for IPython documentation and all existing documentation
-should be converted to this format.
+All standalone documentation should be written in plain text (``.txt``) files
+using `reStructuredText`_ for markup and formatting. All such documentation
+should be placed in the top level directory ``docs`` of the IPython source
+tree. Or, when appropriate, a suitably named subdirectory should be used. The
+documentation in this location will serve as the main source for IPython
+documentation and all existing documentation should be converted to this
+format.
 
-In the future, the text files in the ``docs`` directory will be used to generate all
-forms of documentation for IPython. This include documentation on the IPython website
-as well as *pdf* documentation.
+In the future, the text files in the ``docs`` directory will be used to
+generate all forms of documentation for IPython. This include documentation on
+the IPython website as well as *pdf* documentation.
 
 .. _reStructuredText: http://docutils.sourceforge.net/rst.html
 
 Docstring format
 ----------------
 
-Good docstrings are very important. All new code will use `Epydoc`_ for generating API
-docs, so we will follow the `Epydoc`_ conventions. More specifically, we will use
-`reStructuredText`_ for markup and formatting, since it is understood by a wide
-variety of tools. This means that if in the future we have any reason to change from
-`Epydoc`_ to something else, we'll have fewer transition pains.
+Good docstrings are very important. All new code will use `Epydoc`_ for
+generating API docs, so we will follow the `Epydoc`_ conventions. More
+specifically, we will use `reStructuredText`_ for markup and formatting, since
+it is understood by a wide variety of tools. This means that if in the future
+we have any reason to change from `Epydoc`_ to something else, we'll have fewer
+transition pains.
 
 Details about using `reStructuredText`_ for docstrings can be found `here
 <http://epydoc.sourceforge.net/manual-othermarkup.html>`_.
@@ -233,7 +245,8 b' Coding conventions'
 General
 -------
 
-In general, we'll try to follow the standard Python style conventions as described here:
+In general, we'll try to follow the standard Python style conventions as
+described here:
 
 - `Style Guide for Python Code  <http://www.python.org/peps/pep-0008.html>`_  
 
@@ -250,8 +263,8 b' Other comments:'
 Naming conventions
 ------------------
 
-In terms of naming conventions, we'll follow the guidelines from the `Style Guide for
-Python Code`_.
+In terms of naming conventions, we'll follow the guidelines from the `Style
+Guide for Python Code`_.
 
 For all new IPython code (and much existing code is being refactored), we'll use:
 
@@ -259,67 +272,81 b" For all new IPython code (and much existing code is being refactored), we'll use"
 
 - ``CamelCase`` for class names.
 
-- ``lowercase_with_underscores`` for methods, functions, variables and attributes.
+- ``lowercase_with_underscores`` for methods, functions, variables and
+  attributes.
 
-This may be confusing as most of the existing IPython codebase uses a different convention (``lowerCamelCase`` for methods and attributes).  Slowly, we will move IPython over to the new
-convention, providing shadow names for backward compatibility in public interfaces. 
+This may be confusing as most of the existing IPython codebase uses a different
+convention (``lowerCamelCase`` for methods and attributes).  Slowly, we will
+move IPython over to the new convention, providing shadow names for backward
+compatibility in public interfaces.
 
-There are, however, some important exceptions to these rules.  In some cases, IPython
-code will interface with packages (Twisted, Wx, Qt) that use other conventions.  At some level this makes it impossible to adhere to our own standards at all times.  In particular, when subclassing classes that use other naming conventions, you must follow their naming conventions.  To deal with cases like this, we propose the following policy:
+There are, however, some important exceptions to these rules.  In some cases,
+IPython code will interface with packages (Twisted, Wx, Qt) that use other
+conventions.  At some level this makes it impossible to adhere to our own
+standards at all times.  In particular, when subclassing classes that use other
+naming conventions, you must follow their naming conventions.  To deal with
+cases like this, we propose the following policy:
 
 - If you are subclassing a class that uses different conventions, use its
-  naming conventions throughout your subclass.  Thus, if you are creating a 
-  Twisted Protocol class, used Twisted's ``namingSchemeForMethodsAndAttributes.``
-
-- All IPython's official interfaces should use our conventions.  In some cases 
-  this will mean that you need to provide shadow names (first implement ``fooBar``
-  and then ``foo_bar = fooBar``).  We want to avoid this at all costs, but it 
-  will probably be necessary at times.  But, please use this sparingly!
-
-Implementation-specific *private* methods will use ``_single_underscore_prefix``.
-Names with a leading double underscore will *only* be used in special cases, as they
-makes subclassing difficult (such names are not easily seen by child classes).
-
-Occasionally some run-in lowercase names are used, but mostly for very short names or
-where we are implementing methods very similar to existing ones in a base class (like
-``runlines()`` where ``runsource()`` and ``runcode()`` had established precedent).
+  naming conventions throughout your subclass.  Thus, if you are creating a
+  Twisted Protocol class, used Twisted's
+  ``namingSchemeForMethodsAndAttributes.``
+
+- All IPython's official interfaces should use our conventions.  In some cases
+  this will mean that you need to provide shadow names (first implement
+  ``fooBar`` and then ``foo_bar = fooBar``).  We want to avoid this at all
+  costs, but it will probably be necessary at times.  But, please use this
+  sparingly!
+
+Implementation-specific *private* methods will use
+``_single_underscore_prefix``.  Names with a leading double underscore will
+*only* be used in special cases, as they makes subclassing difficult (such
+names are not easily seen by child classes).
+
+Occasionally some run-in lowercase names are used, but mostly for very short
+names or where we are implementing methods very similar to existing ones in a
+base class (like ``runlines()`` where ``runsource()`` and ``runcode()`` had
+established precedent).
 
 The old IPython codebase has a big mix of classes and modules prefixed with an
-explicit ``IP``. In Python this is mostly unnecessary, redundant and frowned upon, as
-namespaces offer cleaner prefixing. The only case where this approach is justified is
-for classes which are expected to be imported into external namespaces and a very
-generic name (like Shell) is too likely to clash with something else. We'll need to
-revisit this issue as we clean up and refactor the code, but in general we should
-remove as many unnecessary ``IP``/``ip`` prefixes as possible. However, if a prefix
-seems absolutely necessary the more specific ``IPY`` or ``ipy`` are preferred.
+explicit ``IP``. In Python this is mostly unnecessary, redundant and frowned
+upon, as namespaces offer cleaner prefixing. The only case where this approach
+is justified is for classes which are expected to be imported into external
+namespaces and a very generic name (like Shell) is too likely to clash with
+something else. We'll need to revisit this issue as we clean up and refactor
+the code, but in general we should remove as many unnecessary ``IP``/``ip``
+prefixes as possible. However, if a prefix seems absolutely necessary the more
+specific ``IPY`` or ``ipy`` are preferred.
 
 .. _devel_testing:
 
 Testing system
 ==============
 
-It is extremely important that all code contributed to IPython has tests. Tests should
-be written as unittests, doctests or as entities that the `Nose`_ testing package will
-find. Regardless of how the tests are written, we will use `Nose`_ for discovering and
-running the tests. `Nose`_ will be required to run the IPython test suite, but will
-not be required to simply use IPython.
+It is extremely important that all code contributed to IPython has tests. Tests
+should be written as unittests, doctests or as entities that the `Nose`_
+testing package will find. Regardless of how the tests are written, we will use
+`Nose`_ for discovering and running the tests. `Nose`_ will be required to run
+the IPython test suite, but will not be required to simply use IPython.
 
 .. _Nose: http://code.google.com/p/python-nose/
 
-Tests of `Twisted`__ using code should be written by subclassing the ``TestCase`` class
-that comes with ``twisted.trial.unittest``. When this is done, `Nose`_ will be able to
-run the tests and the twisted reactor will be handled correctly.
+Tests of `Twisted`__ using code should be written by subclassing the
+``TestCase`` class that comes with ``twisted.trial.unittest``. When this is
+done, `Nose`_ will be able to run the tests and the twisted reactor will be
+handled correctly.
 
 .. __: http://www.twistedmatrix.com
 
-Each subpackage in IPython should have its own ``tests`` directory that contains all
-of the tests for that subpackage. This allows each subpackage to be self-contained. If
-a subpackage has any dependencies beyond the Python standard library, the tests for
-that subpackage should be skipped if the dependencies are not found. This is very
-important so users don't get tests failing simply because they don't have dependencies.
+Each subpackage in IPython should have its own ``tests`` directory that
+contains all of the tests for that subpackage. This allows each subpackage to
+be self-contained. If a subpackage has any dependencies beyond the Python
+standard library, the tests for that subpackage should be skipped if the
+dependencies are not found. This is very important so users don't get tests
+failing simply because they don't have dependencies.
 
-We also need to look into use Noses ability to tag tests to allow a more modular
-approach of running tests.
+We also need to look into use Noses ability to tag tests to allow a more
+modular approach of running tests.
 
 .. _devel_config:
 
@@ -327,45 +354,54 b' Configuration system'
 ====================
 
 IPython uses `.ini`_ files for configuration purposes. This represents a huge
-improvement over the configuration system used in IPython. IPython works with these
-files using the `ConfigObj`_ package, which IPython includes as
+improvement over the configuration system used in IPython. IPython works with
+these files using the `ConfigObj`_ package, which IPython includes as
 ``ipython1/external/configobj.py``.
 
-Currently, we are using raw `ConfigObj`_ objects themselves. Each subpackage of IPython
-should contain a ``config`` subdirectory that contains all of the configuration
-information for the subpackage. To see how configuration information is defined (along
-with defaults) see at the examples in ``ipython1/kernel/config`` and
-``ipython1/core/config``. Likewise, to see how the configuration information is used,
-see examples in ``ipython1/kernel/scripts/ipengine.py``.
-
-Eventually, we will add a new layer on top of the raw `ConfigObj`_ objects. We are
-calling this new layer, ``tconfig``, as it will use a `Traits`_-like validation model.
-We won't actually use `Traits`_, but will implement something similar in pure Python.
-But, even in this new system, we will still use `ConfigObj`_ and `.ini`_ files
-underneath the hood. Talk to Fernando if you are interested in working on this part of
-IPython. The current prototype of ``tconfig`` is located in the IPython sandbox.
+Currently, we are using raw `ConfigObj`_ objects themselves. Each subpackage of
+IPython should contain a ``config`` subdirectory that contains all of the
+configuration information for the subpackage. To see how configuration
+information is defined (along with defaults) see at the examples in
+``ipython1/kernel/config`` and ``ipython1/core/config``. Likewise, to see how
+the configuration information is used, see examples in
+``ipython1/kernel/scripts/ipengine.py``.
+
+Eventually, we will add a new layer on top of the raw `ConfigObj`_ objects. We
+are calling this new layer, ``tconfig``, as it will use a `Traits`_-like
+validation model.  We won't actually use `Traits`_, but will implement
+something similar in pure Python.  But, even in this new system, we will still
+use `ConfigObj`_ and `.ini`_ files underneath the hood. Talk to Fernando if you
+are interested in working on this part of IPython. The current prototype of
+``tconfig`` is located in the IPython sandbox.
 
 .. _.ini: http://docs.python.org/lib/module-ConfigParser.html
 .. _ConfigObj: http://www.voidspace.org.uk/python/configobj.html
 .. _Traits: http://code.enthought.com/traits/
 
+
 Installation and testing scenarios
 ==================================
 
-This section outlines the various scenarios that we need to test before we release an IPython version.  These scenarios represent different ways of installing IPython and its dependencies.
+This section outlines the various scenarios that we need to test before we
+release an IPython version.  These scenarios represent different ways of
+installing IPython and its dependencies.
 
 Installation scenarios under Linux and OS X
 -------------------------------------------
 
-    1.  Install from tarball using `python setup.py install`.
+    1.  Install from tarball using ``python setup.py install``.
         a.  With only readline+nose dependencies installed.
-        b.  With all dependencies installed (readline, zope.interface,
-            Twisted, foolscap, Sphinx, nose, pyOpenSSL).
+        b.  With all dependencies installed (readline, zope.interface, Twisted,
+        foolscap, Sphinx, nose, pyOpenSSL).
+	    
     2.  Install using easy_install.
+
         a.  With only readline+nose dependencies installed.
-            i.  Default dependencies: `easy_install ipython-0.9.beta3-py2.5.egg`
-            ii. Optional dependency sets: `easy_install -f ipython-0.9.beta3-py2.5.egg IPython[kernel,doc,test,security]`
+            i.  Default dependencies: ``easy_install ipython-0.9.beta3-py2.5.egg``
+            ii. Optional dependency sets: ``easy_install -f ipython-0.9.beta3-py2.5.egg IPython[kernel,doc,test,security]``
+
         b.  With all dependencies already installed.
+	
 
 Installation scenarios under Win32
 ----------------------------------
@@ -381,6 +417,7 b' Tests to run for these scenarios'
     2.  Start a controller and engines and try a few things by hand.
         a.  Using ipcluster.
         b.  Using ipcontroller/ipengine by hand.
+
     3.  Run a few of the parallel examples.
     4.  Try the kernel with and without security with and without PyOpenSSL
         installed.
@@ -388,10 +425,22 b' Tests to run for these scenarios'
     6.  Make sure that furl files are being put in proper locations.
 
 
+Release checklist
+=================
 
+Most of the release process is automated by the :file:`release` script in the
+:file:`tools` directory.  This is just a handy reminder for the release manager.
 
+#. Run the release script, which makes the tar.gz, eggs and Win32 .exe
+   installer.  It posts them to the site and registers the release with PyPI.
 
+#. Updating the website with announcements and links to the updated changes.txt
+   in html form.  Remember to put a short note both on the news page of the site
+   and on launcphad.
 
+#. Drafting a short release announcement with i) highlights and ii) a link to
+   the html changes.txt.
 
+#. Make sure that the released version of the docs is live on the site.
 
-
+#. Celebrate!

docs/source/development/index.txt

@@ -7,3 +7,4 b' Development'
 
    development.txt
    roadmap.txt
+   notification_blueprint.txt

docs/source/development/notification_blueprint.txt

@@ -1,4 +1,4 b''
-.. Notification:
+.. _notification:
 
 ==========================================
 IPython.kernel.core.notification blueprint
@@ -11,37 +11,39 b' The :mod:`IPython.kernel.core.notification` module will provide a simple impleme'
 Functional Requirements
 =======================
 The notification center must:
-	* Provide synchronous notification of events to all registered observers.
-	* Provide typed or labeled notification types
-	* Allow observers to register callbacks for individual or all notification types
-	* Allow observers to register callbacks for events from individual or all notifying objects
-	* Notification to the observer consists of the notification type, notifying object and user-supplied extra information [implementation: as keyword parameters to the registered callback]
-	* Perform as O(1) in the case of no registered observers.
-	* Permit out-of-process or cross-network extension.
-	
+    * Provide synchronous notification of events to all registered observers.
+    * Provide typed or labeled notification types
+    * Allow observers to register callbacks for individual or all notification types
+    * Allow observers to register callbacks for events from individual or all notifying objects
+    * Notification to the observer consists of the notification type, notifying object and user-supplied extra information [implementation: as keyword parameters to the registered callback]
+    * Perform as O(1) in the case of no registered observers.
+    * Permit out-of-process or cross-network extension.
+    
 What's not included
 ==============================================================
 As written, the :mod:`IPython.kernel.core.notificaiton` module does not:
- 	* Provide out-of-process or network notifications [these should be handled by a separate, Twisted aware module in :mod:`IPython.kernel`].
-	* Provide zope.interface-style interfaces for the notification system [these should also be provided by the :mod:`IPython.kernel` module]
-	
+    * Provide out-of-process or network notifications [these should be handled by a separate, Twisted aware module in :mod:`IPython.kernel`].
+    * Provide zope.interface-style interfaces for the notification system [these should also be provided by the :mod:`IPython.kernel` module]
+    
 Use Cases
 =========
 The following use cases describe the main intended uses of the notificaiton module and illustrate the main success scenario for each use case:
 
-	1. Dwight Schroot is writing a frontend for the IPython project. His frontend is stuck in the stone age and must communicate synchronously with an IPython.kernel.core.Interpreter instance. Because code is executed in blocks by the Interpreter, Dwight's UI freezes every time he executes a long block of code. To keep track of the progress of his long running block, Dwight adds the following code to his frontend's set-up code::
-		from IPython.kernel.core.notification import NotificationCenter
-		center = NotificationCenter.sharedNotificationCenter
-		center.registerObserver(self, type=IPython.kernel.core.Interpreter.STDOUT_NOTIFICATION_TYPE, notifying_object=self.interpreter, callback=self.stdout_notification)
-	
-	and elsewhere in his front end::
-		def stdout_notification(self, type, notifying_object, out_string=None):
-			self.writeStdOut(out_string)
-	
-	If everything works, the Interpreter will (according to its published API) fire a notification via the :data:`IPython.kernel.core.notification.sharedCenter` of type :const:`STD_OUT_NOTIFICATION_TYPE` before writing anything to stdout [it's up to the Intereter implementation to figure out when to do this]. The notificaiton center will then call the registered callbacks for that event type (in this case, Dwight's frontend's stdout_notification method). Again, according to its API, the Interpreter provides an additional keyword argument when firing the notificaiton of out_string, a copy of the string it will write to stdout.
-
-	Like magic, Dwight's frontend is able to provide output, even during long-running calculations. Now if Jim could just convince Dwight to use Twisted...
-
-	2. Boss Hog is writing a frontend for the IPython project. Because Boss Hog is stuck in the stone age, his frontend will be written in a new Fortran-like dialect of python and will run only from the command line. Because he doesn't need any fancy notification system and is used to worrying about every cycle on his rat-wheel powered mini, Boss Hog is adamant that the new notification system not produce any performance penalty. As they say in Hazard county, there's no such thing as a free lunch. If he wanted zero overhead, he should have kept using IPython 0.8. Instead, those tricky Duke boys slide in a suped-up bridge-out jumpin' awkwardly confederate-lovin' notification module that imparts only a constant (and small) performance penalty when the Interpreter (or any other object) fires an event for which there are no registered observers. Of course, the same notificaiton-enabled Interpreter can then be used in frontends that require notifications, thus saving the IPython project from a nasty civil war.
-	
-	3. Barry is wrting a frontend for the IPython project. Because Barry's front end is the *new hotness*, it uses an asynchronous event model to communicate with a Twisted :mod:`~IPython.kernel.engineservice` that communicates with the IPython :class:`~IPython.kernel.core.interpreter.Interpreter`. Using the :mod:`IPython.kernel.notification` module, an asynchronous wrapper on the :mod:`IPython.kernel.core.notification` module, Barry's frontend can register for notifications from the interpreter that are delivered asynchronously. Even if Barry's frontend is running on a separate process or even host from the Interpreter, the notifications are delivered, as if by dark and twisted magic. Just like Dwight's frontend, Barry's frontend can now recieve notifications of e.g. writing to stdout/stderr, opening/closing an external file, an exception in the executing code, etc.
  No newline at end of file
+    1. Dwight Schroot is writing a frontend for the IPython project. His frontend is stuck in the stone age and must communicate synchronously with an IPython.kernel.core.Interpreter instance. Because code is executed in blocks by the Interpreter, Dwight's UI freezes every time he executes a long block of code. To keep track of the progress of his long running block, Dwight adds the following code to his frontend's set-up code::
+
+        from IPython.kernel.core.notification import NotificationCenter
+        center = NotificationCenter.sharedNotificationCenter
+        center.registerObserver(self, type=IPython.kernel.core.Interpreter.STDOUT_NOTIFICATION_TYPE, notifying_object=self.interpreter, callback=self.stdout_notification)
+    
+    and elsewhere in his front end::
+
+        def stdout_notification(self, type, notifying_object, out_string=None):
+            self.writeStdOut(out_string)
+    
+    If everything works, the Interpreter will (according to its published API) fire a notification via the :data:`IPython.kernel.core.notification.sharedCenter` of type :const:`STD_OUT_NOTIFICATION_TYPE` before writing anything to stdout [it's up to the Intereter implementation to figure out when to do this]. The notificaiton center will then call the registered callbacks for that event type (in this case, Dwight's frontend's stdout_notification method). Again, according to its API, the Interpreter provides an additional keyword argument when firing the notificaiton of out_string, a copy of the string it will write to stdout.
+
+    Like magic, Dwight's frontend is able to provide output, even during long-running calculations. Now if Jim could just convince Dwight to use Twisted...
+
+    2. Boss Hog is writing a frontend for the IPython project. Because Boss Hog is stuck in the stone age, his frontend will be written in a new Fortran-like dialect of python and will run only from the command line. Because he doesn't need any fancy notification system and is used to worrying about every cycle on his rat-wheel powered mini, Boss Hog is adamant that the new notification system not produce any performance penalty. As they say in Hazard county, there's no such thing as a free lunch. If he wanted zero overhead, he should have kept using IPython 0.8. Instead, those tricky Duke boys slide in a suped-up bridge-out jumpin' awkwardly confederate-lovin' notification module that imparts only a constant (and small) performance penalty when the Interpreter (or any other object) fires an event for which there are no registered observers. Of course, the same notificaiton-enabled Interpreter can then be used in frontends that require notifications, thus saving the IPython project from a nasty civil war.
+    
+    3. Barry is wrting a frontend for the IPython project. Because Barry's front end is the *new hotness*, it uses an asynchronous event model to communicate with a Twisted :mod:`~IPython.kernel.engineservice` that communicates with the IPython :class:`~IPython.kernel.core.interpreter.Interpreter`. Using the :mod:`IPython.kernel.notification` module, an asynchronous wrapper on the :mod:`IPython.kernel.core.notification` module, Barry's frontend can register for notifications from the interpreter that are delivered asynchronously. Even if Barry's frontend is running on a separate process or even host from the Interpreter, the notifications are delivered, as if by dark and twisted magic. Just like Dwight's frontend, Barry's frontend can now recieve notifications of e.g. writing to stdout/stderr, opening/closing an external file, an exception in the executing code, etc.
  No newline at end of file

docs/source/development/roadmap.txt

@@ -32,16 +32,21 b' IPython is implemented using a distributed set of processes that communicate usi'
 
 We need to build a system that makes it trivial for users to start and manage IPython processes.  This system should have the following properties:
 
-	* It should possible to do everything through an extremely simple API that users
-	  can call from their own Python script.  No shell commands should be needed.
-	* This simple API should be configured using standard .ini files.
-	* The system should make it possible to start processes using a number of different
-	  approaches:  SSH, PBS/Torque, Xgrid, Windows Server, mpirun, etc.
-	* The controller and engine processes should each have a daemon for monitoring, 
-	  signaling and clean up.
-	* The system should be secure.
-	* The system should work under all the major operating systems, including
-	  Windows.
+* It should possible to do everything through an extremely simple API that users
+  can call from their own Python script.  No shell commands should be needed.
+  
+* This simple API should be configured using standard .ini files.
+
+* The system should make it possible to start processes using a number of different
+  approaches:  SSH, PBS/Torque, Xgrid, Windows Server, mpirun, etc.
+  
+* The controller and engine processes should each have a daemon for monitoring, 
+  signaling and clean up.
+  
+* The system should be secure.
+
+* The system should work under all the major operating systems, including
+  Windows.
 
 Initial work has begun on the daemon infrastructure, and some of the needed logic is contained in the ipcluster script.
 
@@ -57,12 +62,15 b' Security'
 
 Currently, IPython has no built in security or security model.  Because we would like IPython to be usable on public computer systems and over wide area networks, we need to come up with a robust solution for security.  Here are some of the specific things that need to be included:
 
-	* User authentication between all processes (engines, controller and clients).
-	* Optional TSL/SSL based encryption of all communication channels.
-	* A good way of picking network ports so multiple users on the same system can
-	  run their own controller and engines without interfering with those of others.
-	* A clear model for security that enables users to evaluate the security risks
-	  associated with using IPython in various manners.
+* User authentication between all processes (engines, controller and clients).
+
+* Optional TSL/SSL based encryption of all communication channels.
+
+* A good way of picking network ports so multiple users on the same system can
+  run their own controller and engines without interfering with those of others.
+  
+* A clear model for security that enables users to evaluate the security risks
+  associated with using IPython in various manners.
 
 For the implementation of this, we plan on using Twisted's support for SSL and authentication.  One things that we really should look at is the `Foolscap`_ network protocol, which provides many of these things out of the box.
 
@@ -70,6 +78,9 b" For the implementation of this, we plan on using Twisted's support for SSL and a"
 
 The security work needs to be done in conjunction with other network protocol stuff.
 
+As of the 0.9 release of IPython, we are using Foolscap and we have implemented
+a full security model.
+
 Latent performance issues
 -------------------------
 
@@ -82,7 +93,7 b' Currently, we have a number of performance issues that are waiting to bite users'
 	* Currently, the client to controller connections are done through XML-RPC using 
 	  HTTP 1.0.  This is very inefficient as XML-RPC is a very verbose protocol and 
 	  each request must be handled with a new connection.  We need to move these network
-	  connections over to PB or Foolscap.
+	  connections over to PB or Foolscap.  Done!
 	* We currently don't have a good way of handling large objects in the controller.
 	  The biggest problem is that because we don't have any way of streaming objects,
 	  we get lots of temporary copies in the low-level buffers.  We need to implement

docs/source/faq.txt

@@ -16,10 +16,13 b' Will IPython speed my Python code up?'
 Yes and no. When converting a serial code to run in parallel, there often many
 difficulty questions that need to be answered, such as:
 
-	* How should data be decomposed onto the set of processors?
-	* What are the data movement patterns?
-	* Can the algorithm be structured to minimize data movement?
-	* Is dynamic load balancing important?
+* How should data be decomposed onto the set of processors?
+
+* What are the data movement patterns?
+
+* Can the algorithm be structured to minimize data movement?
+
+* Is dynamic load balancing important?
 
 We can't answer such questions for you. This is the hard (but fun) work of parallel
 computing. But, once you understand these things IPython will make it easier for you to
@@ -28,9 +31,7 b' resulting parallel code interactively.'
 
 With that said, if your problem is trivial to parallelize, IPython has a number of
 different interfaces that will enable you to parallelize things is almost no time at
-all.  A good place to start is the ``map`` method of our `multiengine interface`_.
-
-.. _multiengine interface: ./parallel_multiengine
+all.  A good place to start is the ``map`` method of our :class:`MultiEngineClient`.
 
 What is the best way to use MPI from Python?
 --------------------------------------------
@@ -40,26 +41,33 b' What about all the other parallel computing packages in Python?'
 
 Some of the unique characteristic of IPython are:
 
-	* IPython is the only architecture that abstracts out the notion of a 
-	  parallel computation in such a way that new models of parallel computing
-	  can be explored quickly and easily.  If you don't like the models we
-	  provide, you can simply create your own using the capabilities we provide.
-	* IPython is asynchronous from the ground up (we use `Twisted`_).
-	* IPython's architecture is designed to avoid subtle problems 
-	  that emerge because of Python's global interpreter lock (GIL).
-	* While IPython'1 architecture is designed to support a wide range
-	  of novel parallel computing models, it is fully interoperable with
-	  traditional MPI applications.
-	* IPython has been used and tested extensively on modern supercomputers.
-	* IPython's networking layers are completely modular.  Thus, is 
-	  straightforward to replace our existing network protocols with 
-	  high performance alternatives (ones based upon Myranet/Infiniband).
-	* IPython is designed from the ground up to support collaborative
-	  parallel computing.  This enables multiple users to actively develop
-	  and run the *same* parallel computation.
-	* Interactivity is a central goal for us.  While IPython does not have
-	  to be used interactivly, is can be.
-	  
+* IPython is the only architecture that abstracts out the notion of a 
+  parallel computation in such a way that new models of parallel computing
+  can be explored quickly and easily.  If you don't like the models we
+  provide, you can simply create your own using the capabilities we provide.
+  
+* IPython is asynchronous from the ground up (we use `Twisted`_).
+
+* IPython's architecture is designed to avoid subtle problems 
+  that emerge because of Python's global interpreter lock (GIL).
+  
+* While IPython's architecture is designed to support a wide range
+  of novel parallel computing models, it is fully interoperable with
+  traditional MPI applications.
+  
+* IPython has been used and tested extensively on modern supercomputers.
+
+* IPython's networking layers are completely modular.  Thus, is 
+  straightforward to replace our existing network protocols with 
+  high performance alternatives (ones based upon Myranet/Infiniband).
+  
+* IPython is designed from the ground up to support collaborative
+  parallel computing.  This enables multiple users to actively develop
+  and run the *same* parallel computation.
+  
+* Interactivity is a central goal for us.  While IPython does not have
+  to be used interactivly, it can be.
+      
 .. _Twisted: http://www.twistedmatrix.com
 
 Why The IPython controller a bottleneck in my parallel calculation?
@@ -71,13 +79,17 b' too much data is being pushed and pulled to and from the engines. If your algori'
 is structured in this way, you really should think about alternative ways of
 handling the data movement. Here are some ideas:
 
-	1. Have the engines write data to files on the locals disks of the engines.
-	2. Have the engines write data to files on a file system that is shared by
-	   the engines.
-	3. Have the engines write data to a database that is shared by the engines.
-	4. Simply keep data in the persistent memory of the engines and move the
-	   computation to the data (rather than the data to the computation).
-	5. See if you can pass data directly between engines using MPI.
+1. Have the engines write data to files on the locals disks of the engines.
+
+2. Have the engines write data to files on a file system that is shared by
+   the engines.
+
+3. Have the engines write data to a database that is shared by the engines.
+
+4. Simply keep data in the persistent memory of the engines and move the
+   computation to the data (rather than the data to the computation).
+
+5. See if you can pass data directly between engines using MPI.
 
 Isn't Python slow to be used for high-performance parallel computing?
 ---------------------------------------------------------------------

docs/source/history.txt

@@ -7,50 +7,32 b' History'
 Origins
 =======
 
-The current IPython system grew out of the following three projects:
-
-    * [ipython] by Fernando Pérez. I was working on adding
-      Mathematica-type prompts and a flexible configuration system
-      (something better than $PYTHONSTARTUP) to the standard Python
-      interactive interpreter.
-    * [IPP] by Janko Hauser. Very well organized, great usability. Had
-      an old help system. IPP was used as the 'container' code into
-      which I added the functionality from ipython and LazyPython.
-    * [LazyPython] by Nathan Gray. Simple but very powerful. The quick
-      syntax (auto parens, auto quotes) and verbose/colored tracebacks
-      were all taken from here.
-
-When I found out about IPP and LazyPython I tried to join all three
-into a unified system. I thought this could provide a very nice
-working environment, both for regular programming and scientific
-computing: shell-like features, IDL/Matlab numerics, Mathematica-type
-prompt history and great object introspection and help facilities. I
-think it worked reasonably well, though it was a lot more work than I
-had initially planned.
-
-
-Current status
-==============
-
-The above listed features work, and quite well for the most part. But
-until a major internal restructuring is done (see below), only bug
-fixing will be done, no other features will be added (unless very minor
-and well localized in the cleaner parts of the code).
-
-IPython consists of some 18000 lines of pure python code, of which
-roughly two thirds is reasonably clean. The rest is, messy code which
-needs a massive restructuring before any further major work is done.
-Even the messy code is fairly well documented though, and most of the
-problems in the (non-existent) class design are well pointed to by a
-PyChecker run. So the rewriting work isn't that bad, it will just be
-time-consuming.
-
-
-Future
-------
-
-See the separate new_design document for details. Ultimately, I would
-like to see IPython become part of the standard Python distribution as a
-'big brother with batteries' to the standard Python interactive
-interpreter. But that will never happen with the current state of the
-code, so all contributions are welcome.
  No newline at end of file
+IPython was starting in 2001 by Fernando Perez.  IPython as we know it
+today grew out of the following three projects:
+
+* ipython by Fernando Pérez. I was working on adding
+  Mathematica-type prompts and a flexible configuration system
+  (something better than $PYTHONSTARTUP) to the standard Python
+  interactive interpreter.
+* IPP by Janko Hauser. Very well organized, great usability. Had
+  an old help system. IPP was used as the 'container' code into
+  which I added the functionality from ipython and LazyPython.
+* LazyPython by Nathan Gray. Simple but very powerful. The quick
+  syntax (auto parens, auto quotes) and verbose/colored tracebacks
+  were all taken from here.
+
+Here is how Fernando describes it:
+
+    When I found out about IPP and LazyPython I tried to join all three
+    into a unified system. I thought this could provide a very nice
+    working environment, both for regular programming and scientific
+    computing: shell-like features, IDL/Matlab numerics, Mathematica-type
+    prompt history and great object introspection and help facilities. I
+    think it worked reasonably well, though it was a lot more work than I
+    had initially planned.
+
+Today and how we got here
+=========================
+
+This needs to be filled in.
+

docs/source/index.txt

@@ -2,11 +2,15 b''
 IPython Documentation
 =====================
 
-Contents
-========
+.. htmlonly::
+
+   :Release: |release|
+   :Date: |today|
+
+   Contents:
 
 .. toctree::
-   :maxdepth: 1
+   :maxdepth: 2
 
    overview.txt
    install/index.txt
@@ -20,9 +24,9 b' Contents'
    license_and_copyright.txt
    credits.txt
 
-Indices and tables
-==================
 
-* :ref:`genindex`
-* :ref:`modindex`
-* :ref:`search`
  No newline at end of file
+.. htmlonly::
+
+  * :ref:`genindex`
+  * :ref:`modindex`
+  * :ref:`search`

docs/source/install/index.txt

@@ -7,5 +7,4 b' Installation'
 .. toctree::
    :maxdepth: 2
 
-   basic.txt
-   advanced.txt
+   install.txt

docs/source/interactive/reference.txt

@@ -8,7 +8,7 b' IPython reference'
 
 .. contents::
 
-.. _Command line options:
+.. _command_line_options:
 
 Command-line usage
 ==================
@@ -288,12 +288,13 b' All options with a [no] prepended can be specified in negated form'
 	recursive inclusions.
 
     -prompt_in1, pi1 <string>
-	Specify the string used for input prompts. Note that if you
-	are using numbered prompts, the number is represented with a
-	'\#' in the string. Don't forget to quote strings with spaces
-	embedded in them. Default: 'In [\#]:'.  Sec. Prompts_
-	discusses in detail all the available escapes to customize
-	your prompts.
+
+        Specify the string used for input prompts. Note that if you are using
+	numbered prompts, the number is represented with a '\#' in the
+	string. Don't forget to quote strings with spaces embedded in
+	them. Default: 'In [\#]:'.  The :ref:`prompts section <prompts>`
+	discusses in detail all the available escapes to customize your
+	prompts.
 
     -prompt_in2, pi2 <string>
 	Similar to the previous option, but used for the continuation
@@ -2077,13 +2078,14 b' customizations.'
 Access to the standard Python help
 ----------------------------------
 
-As of Python 2.1, a help system is available with access to object
-docstrings and the Python manuals. Simply type 'help' (no quotes) to
-access it. You can also type help(object) to obtain information about a
-given object, and help('keyword') for information on a keyword. As noted
-in sec. `accessing help`_, you need to properly configure
-your environment variable PYTHONDOCS for this feature to work correctly.
+As of Python 2.1, a help system is available with access to object docstrings
+and the Python manuals. Simply type 'help' (no quotes) to access it. You can
+also type help(object) to obtain information about a given object, and
+help('keyword') for information on a keyword. As noted :ref:`here
+<accessing_help>`, you need to properly configure your environment variable
+PYTHONDOCS for this feature to work correctly.
 
+.. _dynamic_object_info:
 
 Dynamic object information
 --------------------------
@@ -2126,7 +2128,7 b' are not really defined as separate identifiers. Try for example typing'
 {}.get? or after doing import os, type os.path.abspath??.
 
 
-.. _Readline:
+.. _readline:
 
 Readline-based features
 -----------------------
@@ -2240,10 +2242,9 b' explanation in your ipythonrc file.'
 Session logging and restoring
 -----------------------------
 
-You can log all input from a session either by starting IPython with
-the command line switches -log or -logfile (see sec. `command line
-options`_) or by activating the logging at any moment with the magic
-function %logstart.
+You can log all input from a session either by starting IPython with the
+command line switches -log or -logfile (see :ref:`here <command_line_options>`)
+or by activating the logging at any moment with the magic function %logstart.
 
 Log files can later be reloaded with the -logplay option and IPython
 will attempt to 'replay' the log by executing all the lines in it, thus
@@ -2279,6 +2280,8 b' resume logging to a file which had previously been started with'
 %logstart. They will fail (with an explanation) if you try to use them
 before logging has been started.
 
+.. _system_shell_access:
+
 System shell access
 -------------------
 
@@ -2389,7 +2392,7 b" These features are basically a terminal version of Ka-Ping Yee's cgitb"
 module, now part of the standard Python library.
 
 
-.. _Input caching:
+.. _input_caching:
 
 Input caching system
 --------------------
@@ -2429,7 +2432,7 b' sec. 6.2 <#sec:magic> for more details on the macro system.'
 A history function %hist allows you to see any part of your input
 history by printing a range of the _i variables.
 
-.. _Output caching:
+.. _output_caching:
 
 Output caching system
 ---------------------
@@ -3034,7 +3037,7 b' which is being shared by the interactive IPython loop and your GUI'
 thread, you should really handle it with thread locking and
 syncrhonization properties. The Python documentation discusses these.
 
-.. _Interactive demos:
+.. _interactive_demos:
 
 Interactive demos with IPython
 ==============================
@@ -3143,21 +3146,17 b' toolkits, including Tk, GTK and WXPython. It also provides a number of'
 commands useful for scientific computing, all with a syntax compatible
 with that of the popular Matlab program.
 
-IPython accepts the special option -pylab (Sec. `Command line
-options`_). This configures it to support matplotlib, honoring the
-settings in the .matplotlibrc file. IPython will detect the user's
-choice of matplotlib GUI backend, and automatically select the proper
-threading model to prevent blocking. It also sets matplotlib in
-interactive mode and modifies %run slightly, so that any
-matplotlib-based script can be executed using %run and the final
-show() command does not block the interactive shell.
-
-The -pylab option must be given first in order for IPython to
-configure its threading mode. However, you can still issue other
-options afterwards. This allows you to have a matplotlib-based
-environment customized with additional modules using the standard
-IPython profile mechanism (Sec. Profiles_): ''ipython -pylab -p
-myprofile'' will load the profile defined in ipythonrc-myprofile after
-configuring matplotlib.
-
-
+IPython accepts the special option -pylab (see :ref:`here
+<command_line_options>`). This configures it to support matplotlib, honoring
+the settings in the .matplotlibrc file. IPython will detect the user's choice
+of matplotlib GUI backend, and automatically select the proper threading model
+to prevent blocking. It also sets matplotlib in interactive mode and modifies
+%run slightly, so that any matplotlib-based script can be executed using %run
+and the final show() command does not block the interactive shell.
+
+The -pylab option must be given first in order for IPython to configure its
+threading mode. However, you can still issue other options afterwards. This
+allows you to have a matplotlib-based environment customized with additional
+modules using the standard IPython profile mechanism (see :ref:`here
+<profiles>`): ``ipython -pylab -p myprofile`` will load the profile defined in
+ipythonrc-myprofile after configuring matplotlib.

docs/source/interactive/tutorial.txt

@@ -24,11 +24,11 b' Tab completion'
 --------------
 
 TAB-completion, especially for attributes, is a convenient way to explore the
-structure of any object you're dealing with. Simply type object_name.<TAB>
-and a list of the object's attributes will be printed (see readline_ for
-more). Tab completion also works on file and directory names, which combined
-with IPython's alias system allows you to do from within IPython many of the
-things you normally would need the system shell for.
+structure of any object you're dealing with. Simply type object_name.<TAB> and
+a list of the object's attributes will be printed (see :ref:`the readline
+section <readline>` for more). Tab completion also works on file and directory
+names, which combined with IPython's alias system allows you to do from within
+IPython many of the things you normally would need the system shell for.
 
 Explore your objects
 --------------------
@@ -39,18 +39,18 b' constructor details for classes. The magic commands %pdoc, %pdef, %psource'
 and %pfile will respectively print the docstring, function definition line,
 full source code and the complete file for any object (when they can be
 found). If automagic is on (it is by default), you don't need to type the '%'
-explicitly. See sec. `dynamic object information`_ for more.
+explicitly. See :ref:`this section <dynamic_object_info>` for more.
 
 The `%run` magic command
 ------------------------
 
-The %run magic command allows you to run any python script and load all of
-its data directly into the interactive namespace. Since the file is re-read
-from disk each time, changes you make to it are reflected immediately (in
-contrast to the behavior of import). I rarely use import for code I am
-testing, relying on %run instead. See magic_ section for more on this and
-other magic commands, or type the name of any magic command and ? to get
-details on it. See also sec. dreload_ for a recursive reload command. %run
+The %run magic command allows you to run any python script and load all of its
+data directly into the interactive namespace. Since the file is re-read from
+disk each time, changes you make to it are reflected immediately (in contrast
+to the behavior of import). I rarely use import for code I am testing, relying
+on %run instead. See :ref:`this section <magic>` for more on this and other
+magic commands, or type the name of any magic command and ? to get details on
+it. See also :ref:`this section <dreload>` for a recursive reload command. %run
 also has special flags for timing the execution of your scripts (-t) and for
 executing them under the control of either Python's pdb debugger (-d) or
 profiler (-p). With all of these, %run can be used as the main tool for
@@ -60,21 +60,21 b' choice.'
 Debug a Python script
 ---------------------
 
-Use the Python debugger, pdb. The %pdb command allows you to toggle on and
-off the automatic invocation of an IPython-enhanced pdb debugger (with
-coloring, tab completion and more) at any uncaught exception. The advantage
-of this is that pdb starts inside the function where the exception occurred,
-with all data still available. You can print variables, see code, execute
-statements and even walk up and down the call stack to track down the true
-source of the problem (which often is many layers in the stack above where
-the exception gets triggered). Running programs with %run and pdb active can
-be an efficient to develop and debug code, in many cases eliminating the need
-for print statements or external debugging tools. I often simply put a 1/0 in
-a place where I want to take a look so that pdb gets called, quickly view
-whatever variables I need to or test various pieces of code and then remove
-the 1/0. Note also that '%run -d' activates pdb and automatically sets
-initial breakpoints for you to step through your code, watch variables, etc.
-See Sec. `Output caching`_ for details.
+Use the Python debugger, pdb. The %pdb command allows you to toggle on and off
+the automatic invocation of an IPython-enhanced pdb debugger (with coloring,
+tab completion and more) at any uncaught exception. The advantage of this is
+that pdb starts inside the function where the exception occurred, with all data
+still available. You can print variables, see code, execute statements and even
+walk up and down the call stack to track down the true source of the problem
+(which often is many layers in the stack above where the exception gets
+triggered). Running programs with %run and pdb active can be an efficient to
+develop and debug code, in many cases eliminating the need for print statements
+or external debugging tools. I often simply put a 1/0 in a place where I want
+to take a look so that pdb gets called, quickly view whatever variables I need
+to or test various pieces of code and then remove the 1/0. Note also that '%run
+-d' activates pdb and automatically sets initial breakpoints for you to step
+through your code, watch variables, etc.  The :ref:`output caching section
+<output_caching>` has more details.
 
 Use the output cache
 --------------------
@@ -84,7 +84,8 b' and variables named _1, _2, etc. alias them. For example, the result of input'
 line 4 is available either as Out[4] or as _4. Additionally, three variables
 named _, __ and ___ are always kept updated with the for the last three
 results. This allows you to recall any previous result and further use it for
-new calculations. See Sec. `Output caching`_ for more.
+new calculations. See :ref:`the output caching section <output_caching>` for
+more.
 
 Suppress output
 ---------------
@@ -102,7 +103,7 b' A similar system exists for caching input. All input is stored in a global'
 list called In , so you can re-execute lines 22 through 28 plus line 34 by
 typing 'exec In[22:29]+In[34]' (using Python slicing notation). If you need
 to execute the same set of lines often, you can assign them to a macro with
-the %macro function. See sec. `Input caching`_ for more.
+the %macro function. See :ref:`here <input_caching>` for more.
 
 Use your input history
 ----------------------
@@ -134,17 +135,18 b' into Python variables.'
 Use Python variables when calling the shell
 -------------------------------------------
 
-Expand python variables when calling the shell (either via '!' and '!!' or
-via aliases) by prepending a $ in front of them. You can also expand complete
-python expressions. See `System shell access`_ for more.
+Expand python variables when calling the shell (either via '!' and '!!' or via
+aliases) by prepending a $ in front of them. You can also expand complete
+python expressions. See :ref:`our shell section <system_shell_access>` for
+more details.
 
 Use profiles
 ------------
 
 Use profiles to maintain different configurations (modules to load, function
 definitions, option settings) for particular tasks. You can then have
-customized versions of IPython for specific purposes. See sec. profiles_ for
-more.
+customized versions of IPython for specific purposes. :ref:`This section
+<profiles>` has more details.
 
 
 Embed IPython in your programs
@@ -152,7 +154,7 b' Embed IPython in your programs'
 
 A few lines of code are enough to load a complete IPython inside your own
 programs, giving you the ability to work with your data interactively after
-automatic processing has been completed. See sec. embedding_ for more.
+automatic processing has been completed. See :ref:`here <embedding>` for more.
 
 Use the Python profiler
 -----------------------
@@ -166,8 +168,8 b' Use IPython to present interactive demos'
 ----------------------------------------
 
 Use the IPython.demo.Demo class to load any Python script as an interactive
-demo. With a minimal amount of simple markup, you can control the execution
-of the script, stopping as needed. See sec. `interactive demos`_ for more.
+demo. With a minimal amount of simple markup, you can control the execution of
+the script, stopping as needed. See :ref:`here <interactive_demos>` for more.
 
 Run doctests
 ------------

docs/source/license_and_copyright.txt

@@ -1,56 +1,82 b''
 .. _license:
 
-=============================
-License and Copyright 
-=============================
+=====================
+License and Copyright
+=====================
 
-This files needs to be updated to reflect what the new COPYING.txt files says about our license and copyright!
+License
+=======
 
-IPython is released under the terms of the BSD license, whose general
-form can be found at: http://www.opensource.org/licenses/bsd-license.php. The full text of the
-IPython license is reproduced below::
+IPython is licensed under the terms of the new or revised BSD license, as follows::
 
-    IPython is released under a BSD-type license.
+    Copyright (c) 2008, IPython Development Team
 
-    Copyright (c) 2001, 2002, 2003, 2004 Fernando Perez
-    <[e-mail]>.
+    All rights reserved.
 
-    Copyright (c) 2001 Janko Hauser <[e-mail]> and  
-    Nathaniel Gray <[e-mail]>.
+    Redistribution and use in source and binary forms, with or without modification,
+    are permitted provided that the following conditions are met:
 
-    All rights reserved.
+    Redistributions of source code must retain the above copyright notice, this list of
+    conditions and the following disclaimer.
+
+    Redistributions in binary form must reproduce the above copyright notice, this list
+    of conditions and the following disclaimer in the documentation and/or other
+    materials provided with the distribution.
+
+    Neither the name of the IPython Development Team nor the names of its contributors
+    may be used to endorse or promote products derived from this software without
+    specific prior written permission.
+
+    THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+    EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+    WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+    IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
+    INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+    NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+    PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+    WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+    ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+    POSSIBILITY OF SUCH DAMAGE.
+
+About the IPython Development Team
+==================================
+
+Fernando Perez began IPython in 2001 based on code from Janko Hauser <[e-mail]> 
+and Nathaniel Gray <[e-mail]>.  Fernando is still the project lead.
+
+The IPython Development Team is the set of all contributors to the IPython project.
+This includes all of the IPython subprojects. Here is a list of the currently active contributors:
+
+	* Matthieu Brucher
+	* Ondrej Certik
+	* Laurent Dufrechou
+	* Robert Kern
+	* Brian E. Granger
+	* Fernando Perez (project leader)
+	* Benjamin Ragan-Kelley
+	* Ville M. Vainio
+	* Gael Varoququx
+	* Stefan van der Walt
+	* Tech-X Corporation
+	* Barry Wark
+
+If your name is missing, please add it.  
+
+Our Copyright Policy
+====================
+
+IPython uses a shared copyright model. Each contributor maintains copyright over
+their contributions to IPython. But, it is important to note that these
+contributions are typically only changes to the repositories. Thus, the IPython
+source code, in its entirety is not the copyright of any single person or
+institution. Instead, it is the collective copyright of the entire IPython
+Development Team. If individual contributors want to maintain a record of what
+changes/contributions they have specific copyright on, they should indicate their
+copyright in the commit message of the change, when they commit the change to
+one of the IPython repositories.
 
-    Redistribution and use in source and binary forms, with or without
-    modification, are permitted provided that the following conditions
-    are met:
-
-    a. Redistributions of source code must retain the above copyright
-    notice, this list of conditions and the following disclaimer.
-
-    b. Redistributions in binary form must reproduce the above copyright
-    notice, this list of conditions and the following disclaimer in the
-    documentation and/or other materials provided with the distribution.
-
-    c. Neither the name of the copyright holders nor the names of any
-    contributors to this software may be used to endorse or promote
-    products derived from this software without specific prior written
-    permission.
-
-    THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-    "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-    LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
-    FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
-    REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
-    INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
-    BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-    LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
-    CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
-    LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
-    ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-    POSSIBILITY OF SUCH DAMAGE. 
-
-Individual authors are the holders of the copyright for their code and
-are listed in each file.
+Miscellaneous
+=============
 
 Some files (DPyGetOpt.py, for example) may be licensed under different
 conditions. Ultimately each file indicates clearly the conditions under

docs/source/overview.txt

@@ -14,136 +14,164 b' However, the interpreter supplied with the standard Python distribution'
 is somewhat limited for extended interactive use.
 
 The goal of IPython is to create a comprehensive environment for
-interactive and exploratory computing.  To support, this goal, IPython
+interactive and exploratory computing.  To support this goal, IPython
 has two main components:
 
-	* An enhanced interactive Python shell.
-	* An architecture for interactive parallel computing.
+* An enhanced interactive Python shell.
+* An architecture for interactive parallel computing.
 
 All of IPython is open source (released under the revised BSD license).
 
 Enhanced interactive Python shell
 =================================
 
-IPython's interactive shell (`ipython`), has the following goals:
-
-   1. Provide an interactive shell superior to Python's default. IPython
-      has many features for object introspection, system shell access,
-      and its own special command system for adding functionality when
-      working interactively. It tries to be a very efficient environment
-      both for Python code development and for exploration of problems
-      using Python objects (in situations like data analysis).
-   2. Serve as an embeddable, ready to use interpreter for your own
-      programs. IPython can be started with a single call from inside
-      another program, providing access to the current namespace. This
-      can be very useful both for debugging purposes and for situations
-      where a blend of batch-processing and interactive exploration are
-      needed.
-   3. Offer a flexible framework which can be used as the base
-      environment for other systems with Python as the underlying
-      language. Specifically scientific environments like Mathematica,
-      IDL and Matlab inspired its design, but similar ideas can be
-      useful in many fields.
-   4. Allow interactive testing of threaded graphical toolkits. IPython
-      has support for interactive, non-blocking control of GTK, Qt and
-      WX applications via special threading flags. The normal Python
-      shell can only do this for Tkinter applications.
+IPython's interactive shell (:command:`ipython`), has the following goals,
+amongst others:
+
+1. Provide an interactive shell superior to Python's default. IPython
+   has many features for object introspection, system shell access,
+   and its own special command system for adding functionality when
+   working interactively. It tries to be a very efficient environment
+   both for Python code development and for exploration of problems
+   using Python objects (in situations like data analysis).
+  
+2. Serve as an embeddable, ready to use interpreter for your own
+   programs. IPython can be started with a single call from inside
+   another program, providing access to the current namespace. This
+   can be very useful both for debugging purposes and for situations
+   where a blend of batch-processing and interactive exploration are
+   needed.  New in the 0.9 version of IPython is a reusable wxPython
+   based IPython widget.
+  
+3. Offer a flexible framework which can be used as the base
+   environment for other systems with Python as the underlying
+   language. Specifically scientific environments like Mathematica,
+   IDL and Matlab inspired its design, but similar ideas can be
+   useful in many fields.
+  
+4. Allow interactive testing of threaded graphical toolkits. IPython
+   has support for interactive, non-blocking control of GTK, Qt and
+   WX applications via special threading flags. The normal Python
+   shell can only do this for Tkinter applications.
 
 Main features of the interactive shell
 --------------------------------------
 
-	* Dynamic object introspection. One can access docstrings, function
-	  definition prototypes, source code, source files and other details
-	  of any object accessible to the interpreter with a single
-	  keystroke (:samp:`?`, and using :samp:`??` provides additional detail).
-	* Searching through modules and namespaces with :samp:`*` wildcards, both
-	  when using the :samp:`?` system and via the :samp:`%psearch` command.
-	* Completion in the local namespace, by typing :kbd:`TAB` at the prompt.
-	  This works for keywords, modules, methods, variables and files in the
-	  current directory. This is supported via the readline library, and
-	  full access to configuring readline's behavior is provided. 
-	  Custom completers can be implemented easily for different purposes
-	  (system commands, magic arguments etc.)
-	* Numbered input/output prompts with command history (persistent
-	  across sessions and tied to each profile), full searching in this
-	  history and caching of all input and output.
-	* User-extensible 'magic' commands. A set of commands prefixed with
-	  :samp:`%` is available for controlling IPython itself and provides
-	  directory control, namespace information and many aliases to
-	  common system shell commands.
-	* Alias facility for defining your own system aliases.
-	* Complete system shell access. Lines starting with :samp:`!` are passed
-	  directly to the system shell, and using :samp:`!!` or :samp:`var = !cmd` 
-	  captures shell output into python variables for further use.
-	* Background execution of Python commands in a separate thread.
-	  IPython has an internal job manager called jobs, and a
-	  conveninence backgrounding magic function called :samp:`%bg`.
-	* The ability to expand python variables when calling the system
-	  shell. In a shell command, any python variable prefixed with :samp:`$` is
-	  expanded. A double :samp:`$$` allows passing a literal :samp:`$` to the shell (for
-	  access to shell and environment variables like :envvar:`PATH`).
-	* Filesystem navigation, via a magic :samp:`%cd` command, along with a
-	  persistent bookmark system (using :samp:`%bookmark`) for fast access to
-	  frequently visited directories.
-	* A lightweight persistence framework via the :samp:`%store` command, which
-	  allows you to save arbitrary Python variables. These get restored
-	  automatically when your session restarts.
-	* Automatic indentation (optional) of code as you type (through the
-	  readline library).
-	* Macro system for quickly re-executing multiple lines of previous
-	  input with a single name. Macros can be stored persistently via
-	  :samp:`%store` and edited via :samp:`%edit`.
-	* Session logging (you can then later use these logs as code in your
-	  programs). Logs can optionally timestamp all input, and also store
-	  session output (marked as comments, so the log remains valid
-	  Python source code).
-	* Session restoring: logs can be replayed to restore a previous
-	  session to the state where you left it.
-	* Verbose and colored exception traceback printouts. Easier to parse
-	  visually, and in verbose mode they produce a lot of useful
-	  debugging information (basically a terminal version of the cgitb
-	  module).
-	* Auto-parentheses: callable objects can be executed without
-	  parentheses: :samp:`sin 3` is automatically converted to :samp:`sin(3)`.
-	* Auto-quoting: using :samp:`,`, or :samp:`;` as the first character forces
-	  auto-quoting of the rest of the line: :samp:`,my_function a b` becomes
-	  automatically :samp:`my_function("a","b")`, while :samp:`;my_function a b`
-	  becomes :samp:`my_function("a b")`.
-	* Extensible input syntax. You can define filters that pre-process
-	  user input to simplify input in special situations. This allows
-	  for example pasting multi-line code fragments which start with
-	  :samp:`>>>` or :samp:`...` such as those from other python sessions or the
-	  standard Python documentation.
-	* Flexible configuration system. It uses a configuration file which
-	  allows permanent setting of all command-line options, module
-	  loading, code and file execution. The system allows recursive file
-	  inclusion, so you can have a base file with defaults and layers
-	  which load other customizations for particular projects.
-	* Embeddable. You can call IPython as a python shell inside your own
-	  python programs. This can be used both for debugging code or for
-	  providing interactive abilities to your programs with knowledge
-	  about the local namespaces (very useful in debugging and data
-	  analysis situations).
-	* Easy debugger access. You can set IPython to call up an enhanced
-	  version of the Python debugger (pdb) every time there is an
-	  uncaught exception. This drops you inside the code which triggered
-	  the exception with all the data live and it is possible to
-	  navigate the stack to rapidly isolate the source of a bug. The
-	  :samp:`%run` magic command (with the :samp:`-d` option) can run any script under
-	  pdb's control, automatically setting initial breakpoints for you.
-	  This version of pdb has IPython-specific improvements, including
-	  tab-completion and traceback coloring support. For even easier
-	  debugger access, try :samp:`%debug` after seeing an exception. winpdb is 
-	  also supported, see ipy_winpdb extension.
-	* Profiler support. You can run single statements (similar to
-	  :samp:`profile.run()`) or complete programs under the profiler's control.
-	  While this is possible with standard cProfile or profile modules,
-	  IPython wraps this functionality with magic commands (see :samp:`%prun`
-	  and :samp:`%run -p`) convenient for rapid interactive work.
-	* Doctest support. The special :samp:`%doctest_mode` command toggles a mode
-	  that allows you to paste existing doctests (with leading :samp:`>>>`
-	  prompts and whitespace) and uses doctest-compatible prompts and
-	  output, so you can use IPython sessions as doctest code.
+* Dynamic object introspection. One can access docstrings, function
+  definition prototypes, source code, source files and other details
+  of any object accessible to the interpreter with a single
+  keystroke (:samp:`?`, and using :samp:`??` provides additional detail).
+  
+* Searching through modules and namespaces with :samp:`*` wildcards, both
+  when using the :samp:`?` system and via the :samp:`%psearch` command.
+
+* Completion in the local namespace, by typing :kbd:`TAB` at the prompt.
+  This works for keywords, modules, methods, variables and files in the
+  current directory. This is supported via the readline library, and
+  full access to configuring readline's behavior is provided. 
+  Custom completers can be implemented easily for different purposes
+  (system commands, magic arguments etc.)
+
+* Numbered input/output prompts with command history (persistent
+  across sessions and tied to each profile), full searching in this
+  history and caching of all input and output.
+
+* User-extensible 'magic' commands. A set of commands prefixed with
+  :samp:`%` is available for controlling IPython itself and provides
+  directory control, namespace information and many aliases to
+  common system shell commands.
+
+* Alias facility for defining your own system aliases.
+
+* Complete system shell access. Lines starting with :samp:`!` are passed
+  directly to the system shell, and using :samp:`!!` or :samp:`var = !cmd` 
+  captures shell output into python variables for further use.
+
+* Background execution of Python commands in a separate thread.
+  IPython has an internal job manager called jobs, and a
+  convenience backgrounding magic function called :samp:`%bg`.
+
+* The ability to expand python variables when calling the system
+  shell. In a shell command, any python variable prefixed with :samp:`$` is
+  expanded. A double :samp:`$$` allows passing a literal :samp:`$` to the shell (for
+  access to shell and environment variables like :envvar:`PATH`).
+
+* Filesystem navigation, via a magic :samp:`%cd` command, along with a
+  persistent bookmark system (using :samp:`%bookmark`) for fast access to
+  frequently visited directories.
+
+* A lightweight persistence framework via the :samp:`%store` command, which
+  allows you to save arbitrary Python variables. These get restored
+  automatically when your session restarts.
+
+* Automatic indentation (optional) of code as you type (through the
+  readline library).
+
+* Macro system for quickly re-executing multiple lines of previous
+  input with a single name. Macros can be stored persistently via
+  :samp:`%store` and edited via :samp:`%edit`.
+
+* Session logging (you can then later use these logs as code in your
+  programs). Logs can optionally timestamp all input, and also store
+  session output (marked as comments, so the log remains valid
+  Python source code).
+
+* Session restoring: logs can be replayed to restore a previous
+  session to the state where you left it.
+
+* Verbose and colored exception traceback printouts. Easier to parse
+  visually, and in verbose mode they produce a lot of useful
+  debugging information (basically a terminal version of the cgitb
+  module).
+
+* Auto-parentheses: callable objects can be executed without
+  parentheses: :samp:`sin 3` is automatically converted to :samp:`sin(3)`.
+
+* Auto-quoting: using :samp:`,`, or :samp:`;` as the first character forces
+  auto-quoting of the rest of the line: :samp:`,my_function a b` becomes
+  automatically :samp:`my_function("a","b")`, while :samp:`;my_function a b`
+  becomes :samp:`my_function("a b")`.
+
+* Extensible input syntax. You can define filters that pre-process
+  user input to simplify input in special situations. This allows
+  for example pasting multi-line code fragments which start with
+  :samp:`>>>` or :samp:`...` such as those from other python sessions or the
+  standard Python documentation.
+
+* Flexible configuration system. It uses a configuration file which
+  allows permanent setting of all command-line options, module
+  loading, code and file execution. The system allows recursive file
+  inclusion, so you can have a base file with defaults and layers
+  which load other customizations for particular projects.
+
+* Embeddable. You can call IPython as a python shell inside your own
+  python programs. This can be used both for debugging code or for
+  providing interactive abilities to your programs with knowledge
+  about the local namespaces (very useful in debugging and data
+  analysis situations).
+
+* Easy debugger access. You can set IPython to call up an enhanced
+  version of the Python debugger (pdb) every time there is an
+  uncaught exception. This drops you inside the code which triggered
+  the exception with all the data live and it is possible to
+  navigate the stack to rapidly isolate the source of a bug. The
+  :samp:`%run` magic command (with the :samp:`-d` option) can run any script under
+  pdb's control, automatically setting initial breakpoints for you.
+  This version of pdb has IPython-specific improvements, including
+  tab-completion and traceback coloring support. For even easier
+  debugger access, try :samp:`%debug` after seeing an exception. winpdb is 
+  also supported, see ipy_winpdb extension.
+
+* Profiler support. You can run single statements (similar to
+  :samp:`profile.run()`) or complete programs under the profiler's control.
+  While this is possible with standard cProfile or profile modules,
+  IPython wraps this functionality with magic commands (see :samp:`%prun`
+  and :samp:`%run -p`) convenient for rapid interactive work.
+
+* Doctest support. The special :samp:`%doctest_mode` command toggles a mode
+  that allows you to paste existing doctests (with leading :samp:`>>>`
+  prompts and whitespace) and uses doctest-compatible prompts and
+  output, so you can use IPython sessions as doctest code.
 
 Interactive parallel computing
 ==============================
@@ -153,6 +181,37 b' architecture within IPython that allows such hardware to be used quickly and eas'
 from Python. Moreover, this architecture is designed to support interactive and
 collaborative parallel computing.
 
+The main features of this system are:
+
+* Quickly parallelize Python code from an interactive Python/IPython session.
+
+* A flexible and dynamic process model that be deployed on anything from 
+  multicore workstations to supercomputers.
+
+* An architecture that supports many different styles of parallelism, from
+  message passing to task farming.  And all of these styles can be handled
+  interactively.
+
+* Both blocking and fully asynchronous interfaces.
+
+* High level APIs that enable many things to be parallelized in a few lines
+  of code.
+
+* Write parallel code that will run unchanged on everything from multicore
+  workstations to supercomputers.
+
+* Full integration with Message Passing libraries (MPI).
+
+* Capabilities based security model with full encryption of network connections.
+
+* Share live parallel jobs with other users securely.  We call this collaborative
+  parallel computing.
+
+* Dynamically load balanced task farming system.
+
+* Robust error handling.  Python exceptions raised in parallel execution are
+  gathered and presented to the top-level code.
+
 For more information, see our :ref:`overview <parallel_index>` of using IPython for
 parallel computing.
 

docs/source/parallel/index.txt

@@ -1,12 +1,9 b''
 .. _parallel_index:
 
 ====================================
-Using IPython for Parallel computing
+Using IPython for parallel computing
 ====================================
 
-User Documentation
-==================
-
 .. toctree::
    :maxdepth: 2
 

docs/source/parallel/parallel_intro.txt

@@ -1,57 +1,68 b''
 .. _ip1par:
 
-======================================
-Using IPython for parallel computing
-======================================
+============================
+Overview and getting started
+============================
 
 .. contents::
 
 Introduction
 ============
 
-This file gives an overview of IPython. IPython has a sophisticated and
+This file gives an overview of IPython's sophisticated and
 powerful architecture for parallel and distributed computing. This
 architecture abstracts out parallelism in a very general way, which
 enables IPython to support many different styles of parallelism
 including:
 
-	* Single program, multiple data (SPMD) parallelism.
-	* Multiple program, multiple data (MPMD) parallelism.
-	* Message passing using ``MPI``.
-	* Task farming.
-	* Data parallel.
-	* Combinations of these approaches.
-	* Custom user defined approaches.
+* Single program, multiple data (SPMD) parallelism.
+* Multiple program, multiple data (MPMD) parallelism.
+* Message passing using ``MPI``.
+* Task farming.
+* Data parallel.
+* Combinations of these approaches.
+* Custom user defined approaches.
 
 Most importantly, IPython enables all types of parallel applications to
 be developed, executed, debugged and monitored *interactively*. Hence,
 the ``I`` in IPython.  The following are some example usage cases for IPython:
 
-	* Quickly parallelize algorithms that are embarrassingly parallel
-	  using a number of simple approaches.  Many simple things can be
-	  parallelized interactively in one or two lines of code.
-	* Steer traditional MPI applications on a supercomputer from an
-	  IPython session on your laptop.
-	* Analyze and visualize large datasets (that could be remote and/or
-	  distributed) interactively using IPython and tools like
-	  matplotlib/TVTK.
-	* Develop, test and debug new parallel algorithms 
-	  (that may use MPI) interactively.
-	* Tie together multiple MPI jobs running on different systems into
-	  one giant distributed and parallel system.
-	* Start a parallel job on your cluster and then have a remote 	
-	  collaborator connect to it and pull back data into their 
-	  local IPython session for plotting and analysis.
-	* Run a set of tasks on a set of CPUs using dynamic load balancing.
+* Quickly parallelize algorithms that are embarrassingly parallel
+  using a number of simple approaches.  Many simple things can be
+  parallelized interactively in one or two lines of code.
+
+* Steer traditional MPI applications on a supercomputer from an
+  IPython session on your laptop.
+
+* Analyze and visualize large datasets (that could be remote and/or
+  distributed) interactively using IPython and tools like
+  matplotlib/TVTK.
+
+* Develop, test and debug new parallel algorithms 
+  (that may use MPI) interactively.
+
+* Tie together multiple MPI jobs running on different systems into
+  one giant distributed and parallel system.
+
+* Start a parallel job on your cluster and then have a remote 	
+  collaborator connect to it and pull back data into their 
+  local IPython session for plotting and analysis.
+
+* Run a set of tasks on a set of CPUs using dynamic load balancing.
 
 Architecture overview
 =====================
 
 The IPython architecture consists of three components:
 
-	* The IPython engine.
-	* The IPython controller.
-	* Various controller Clients.
+* The IPython engine.
+* The IPython controller.
+* Various controller clients.
+
+These components live in the :mod:`IPython.kernel` package and are
+installed with IPython.  They do, however, have additional dependencies
+that must be installed.  For more information, see our
+:ref:`installation documentation <install_index>`.
 
 IPython engine
 ---------------
@@ -75,16 +86,21 b' IPython engines can connect. For each connected engine, the controller'
 manages a queue. All actions that can be performed on the engine go
 through this queue. While the engines themselves block when user code is
 run, the controller hides that from the user to provide a fully
-asynchronous interface to a set of engines. Because the controller
-listens on a network port for engines to connect to it, it must be
-started before any engines are started.  
+asynchronous interface to a set of engines. 
+
+.. note::
+
+    Because the controller listens on a network port for engines to 
+    connect to it, it must be started *before* any engines are started.  
 
 The controller also provides a single point of contact for users who wish
 to utilize the engines connected to the controller. There are different
 ways of working with a controller. In IPython these ways correspond to different interfaces that the controller is adapted to.  Currently we have two default interfaces to the controller: 
 
-	* The MultiEngine interface.
-	* The Task interface.
+* The MultiEngine interface, which provides the simplest possible way of working
+  with engines interactively.
+* The Task interface, which provides presents the engines as a load balanced
+  task farming system.
 
 Advanced users can easily add new custom interfaces to enable other
 styles of parallelism. 
@@ -100,18 +116,37 b' Controller clients'
 
 For each controller interface, there is a corresponding client. These
 clients allow users to interact with a set of engines through the
-interface.
+interface.  Here are the two default clients:
+
+* The :class:`MultiEngineClient` class.
+* The :class:`TaskClient` class.
 
 Security
 --------
 
-By default (as long as `pyOpenSSL` is installed) all network connections between the controller and engines and the controller and clients are secure.  What does this mean?  First of all, all of the connections will be encrypted using SSL.  Second, the connections are authenticated.  We handle authentication in a `capabilities`__ based security model.  In this model,  a "capability (known in some systems as a key) is a communicable, unforgeable token of authority".  Put simply, a capability is like a key to your house.  If you have the key to your house, you can get in, if not you can't.  
+By default (as long as `pyOpenSSL` is installed) all network connections between the controller and engines and the controller and clients are secure.  What does this mean?  First of all, all of the connections will be encrypted using SSL.  Second, the connections are authenticated.  We handle authentication in a `capabilities`__ based security model.  In this model,  a "capability (known in some systems as a key) is a communicable, unforgeable token of authority".  Put simply, a capability is like a key to your house.  If you have the key to your house, you can get in.  If not, you can't.  
 
 .. __: http://en.wikipedia.org/wiki/Capability-based_security
 
-In our architecture, the controller is the only process that listens on network ports, and is thus responsible to creating these keys.  In IPython, these keys are known as Foolscap URLs, or FURLs, because of the underlying network protocol we are using.  As a user, you don't need to know anything about the details of these FURLs, other than that when the controller starts, it saves a set of FURLs to files named something.furl.  The default location of these files is your ~./ipython directory.
+In our architecture, the controller is the only process that listens on network ports, and is thus responsible to creating these keys.  In IPython, these keys are known as Foolscap URLs, or FURLs, because of the underlying network protocol we are using.  As a user, you don't need to know anything about the details of these FURLs, other than that when the controller starts, it saves a set of FURLs to files named :file:`something.furl`.  The default location of these files is the :file:`~./ipython/security` directory.
 
-To connect and authenticate to the controller an engine or client simply needs to present an appropriate furl (that was originally created by the controller) to the controller.  Thus, the .furl files need to be copied to a location where the clients and engines can find them.  Typically, this is the ~./ipython directory on the host where the client/engine is running (which could be a different host than the controller).  Once the .furl files are copied over, everything should work fine.
+To connect and authenticate to the controller an engine or client simply needs to present an appropriate furl (that was originally created by the controller) to the controller.  Thus, the .furl files need to be copied to a location where the clients and engines can find them.  Typically, this is the :file:`~./ipython/security` directory on the host where the client/engine is running (which could be a different host than the controller).  Once the .furl files are copied over, everything should work fine.
+
+Currently, there are three .furl files that the controller creates:
+
+ipcontroller-engine.furl
+    This ``.furl`` file is the key that gives an engine the ability to connect
+    to a controller.
+
+ipcontroller-tc.furl
+    This ``.furl`` file is the key that a :class:`TaskClient` must use to
+    connect to the task interface of a controller.
+    
+ipcontroller-mec.furl
+    This ``.furl`` file is the key that a :class:`MultiEngineClient` must use to
+    connect to the multiengine interface of a controller.
+
+More details of how these ``.furl`` files are used are given below.
 
 Getting Started
 ===============
@@ -127,28 +162,40 b' Starting the controller and engine on your local machine'
 
 This is the simplest configuration that can be used and is useful for
 testing the system and on machines that have multiple cores and/or
-multple CPUs. The easiest way of doing this is using the ``ipcluster``
+multple CPUs. The easiest way of getting started is to use the :command:`ipcluster`
 command::
 
 	$ ipcluster -n 4
-	
+
 This will start an IPython controller and then 4 engines that connect to
 the controller. Lastly, the script will print out the Python commands
 that you can use to connect to the controller. It is that easy.
 
-Underneath the hood, the ``ipcluster`` script uses two other top-level
+.. warning::
+
+    The :command:`ipcluster` does not currently work on Windows.  We are
+    working on it though.
+
+Underneath the hood, the controller creates ``.furl`` files in the
+:file:`~./ipython/security` directory.  Because the engines are on the
+same host, they automatically find the needed :file:`ipcontroller-engine.furl`
+there and use it to connect to the controller.
+
+The :command:`ipcluster` script uses two other top-level
 scripts that you can also use yourself. These scripts are
-``ipcontroller``, which starts the controller and ``ipengine`` which
+:command:`ipcontroller`, which starts the controller and :command:`ipengine` which
 starts one engine. To use these scripts to start things on your local
 machine, do the following.
 
 First start the controller::
 
-	$ ipcontroller &
+	$ ipcontroller
 	
 Next, start however many instances of the engine you want using (repeatedly) the command::
 
-	$ ipengine &
+	$ ipengine
+
+The engines should start and automatically connect to the controller using the ``.furl`` files in :file:`~./ipython/security`. You are now ready to use the controller and engines from IPython.
 
 .. warning:: 
 	
@@ -156,47 +203,71 b' Next, start however many instances of the engine you want using (repeatedly) the'
  	start the controller before the engines, since the engines connect 
 	to the controller as they get started.
 
-On some platforms you may need to give these commands in the form
-``(ipcontroller &)`` and ``(ipengine &)`` for them to work properly. The
-engines should start and automatically connect to the controller on the
-default ports, which are chosen for this type of setup. You are now ready
-to use the controller and engines from IPython.
+.. note::
 
-Starting the controller and engines on different machines
----------------------------------------------------------
+    On some platforms (OS X), to put the controller and engine into the background
+    you may need to give these commands in the form ``(ipcontroller &)`` 
+    and ``(ipengine &)`` (with the parentheses) for them to work properly. 
 
-This section needs to be updated to reflect the new Foolscap capabilities based
-model.
 
-Using ``ipcluster`` with ``ssh``
---------------------------------
+Starting the controller and engines on different hosts
+------------------------------------------------------
 
-The ``ipcluster`` command can also start a controller and engines using
-``ssh``.  We need more documentation on this, but for now here is any
-example startup script::
+When the controller and engines are running on different hosts, things are
+slightly more complicated, but the underlying ideas are the same:
 
-	controller = dict(host='myhost',
-	                  engine_port=None, # default is 10105
-	                  control_port=None,
-	                  )
+1. Start the controller on a host using :command:`ipcontroler`.
+2. Copy :file:`ipcontroller-engine.furl` from :file:`~./ipython/security` on the controller's host to the host where the engines will run.
+3. Use :command:`ipengine` on the engine's hosts to start the engines.
 
-	# keys are hostnames, values are the number of engine on that host
-	engines = dict(node1=2,
-	               node2=2,
-	               node3=2,
-	               node3=2,
-	               )
+The only thing you have to be careful of is to tell :command:`ipengine` where the :file:`ipcontroller-engine.furl` file is located.  There are two ways you can do this:
+
+* Put :file:`ipcontroller-engine.furl` in the :file:`~./ipython/security` directory
+  on the engine's host, where it will be found automatically.
+* Call :command:`ipengine` with the ``--furl-file=full_path_to_the_file`` flag.
+
+The ``--furl-file`` flag works like this::
+
+    $ ipengine --furl-file=/path/to/my/ipcontroller-engine.furl
+
+.. note::
+    
+    If the controller's and engine's hosts all have a shared file system  
+    (:file:`~./ipython/security` is the same on all of them), then things
+    will just work!
+
+Make .furl files persistent
+---------------------------
+
+At fist glance it may seem that that managing the ``.furl`` files is a bit annoying.  Going back to the house and key analogy, copying the ``.furl`` around each time you start the controller is like having to make a new key everytime you want to unlock the door and enter your house.  As with your house, you want to be able to create the key (or ``.furl`` file) once, and then simply use it at any point in the future.
+
+This is possible.  The only thing you have to do is decide what ports the controller will listen on for the engines and clients.  This is done as follows::
+
+    $ ipcontroller --client-port=10101 --engine-port=10102
+
+Then, just copy the furl files over the first time and you are set.  You can start and stop the controller and engines any many times as you want in the future, just make sure to tell the controller to use the *same* ports.
+
+.. note::
+
+    You may ask the question: what ports does the controller listen on if you
+    don't tell is to use specific ones?  The default is to use high random port
+    numbers.  We do this for two reasons: i) to increase security through obcurity
+    and ii) to multiple controllers on a given host to start and automatically
+    use different ports.
 
 Starting engines using ``mpirun``
 ---------------------------------
 
 The IPython engines can be started using ``mpirun``/``mpiexec``, even if
-the engines don't call MPI_Init() or use the MPI API in any way. This is
+the engines don't call ``MPI_Init()`` or use the MPI API in any way. This is
 supported on modern MPI implementations like `Open MPI`_.. This provides
 an really nice way of starting a bunch of engine. On a system with MPI
 installed you can do::
 
-	mpirun -n 4 ipengine --controller-port=10000 --controller-ip=host0
+	mpirun -n 4 ipengine
+
+to start 4 engine on a cluster.  This works even if you don't have any
+Python-MPI bindings installed.
 
 .. _Open MPI: http://www.open-mpi.org/
 
@@ -214,12 +285,12 b' Next Steps'
 ==========
 
 Once you have started the IPython controller and one or more engines, you
-are ready to use the engines to do somnething useful. To make sure
+are ready to use the engines to do something useful. To make sure
 everything is working correctly, try the following commands::
 
 	In [1]: from IPython.kernel import client
 	
-	In [2]: mec = client.MultiEngineClient()    # This looks for .furl files in ~./ipython
+	In [2]: mec = client.MultiEngineClient()
 	
 	In [4]: mec.get_ids()
 	Out[4]: [0, 1, 2, 3]
@@ -239,4 +310,18 b' everything is working correctly, try the following commands::'
 	[3] In [1]: print "Hello World"
 	[3] Out[1]: Hello World
 
-If this works, you are ready to learn more about the :ref:`MultiEngine <parallelmultiengine>` and :ref:`Task <paralleltask>` interfaces to the controller.
+Remember, a client also needs to present a ``.furl`` file to the controller.  How does this happen?  When a multiengine client is created with no arguments, the client tries to find the corresponding ``.furl`` file in the local :file:`~./ipython/security` directory.  If it finds it, you are set.  If you have put the ``.furl`` file in a different location or it has a different name, create the client like this::
+
+    mec = client.MultiEngineClient('/path/to/my/ipcontroller-mec.furl')
+
+Same thing hold true of creating a task client::
+
+    tc = client.TaskClient('/path/to/my/ipcontroller-tc.furl')
+
+You are now ready to learn more about the :ref:`MultiEngine <parallelmultiengine>` and :ref:`Task <paralleltask>` interfaces to the controller.
+
+.. note:: 
+
+    Don't forget that the engine, multiengine client and task client all have
+    *different* furl files.  You must move *each* of these around to an appropriate
+    location so that the engines and clients can use them to connect to the controller.

docs/source/parallel/parallel_multiengine.txt

@@ -1,57 +1,115 b''
 .. _parallelmultiengine:
 
-=================================
-IPython's MultiEngine interface
-=================================
+===============================
+IPython's multiengine interface
+===============================
 
 .. contents::
 
-The MultiEngine interface represents one possible way of working with a
-set of IPython engines. The basic idea behind the MultiEngine interface is
-that the capabilities of each engine are explicitly exposed to the user.
-Thus, in the MultiEngine interface, each engine is given an id that is
-used to identify the engine and give it work to do. This interface is very
-intuitive and is designed with interactive usage in mind, and is thus the
-best place for new users of IPython to begin.
+The multiengine interface represents one possible way of working with a set of
+IPython engines. The basic idea behind the multiengine interface is that the
+capabilities of each engine are directly and explicitly exposed to the user.
+Thus, in the multiengine interface, each engine is given an id that is used to
+identify the engine and give it work to do. This interface is very intuitive
+and is designed with interactive usage in mind, and is thus the best place for
+new users of IPython to begin.
 
 Starting the IPython controller and engines
 ===========================================
 
 To follow along with this tutorial, you will need to start the IPython
-controller and four IPython engines. The simplest way of doing this is to
-use the ``ipcluster`` command::
+controller and four IPython engines. The simplest way of doing this is to use
+the :command:`ipcluster` command::
 
 	$ ipcluster -n 4
 	
-For more detailed information about starting the controller and engines, see our :ref:`introduction <ip1par>` to using IPython for parallel computing.
+For more detailed information about starting the controller and engines, see
+our :ref:`introduction <ip1par>` to using IPython for parallel computing.
 
 Creating a ``MultiEngineClient`` instance
 =========================================
 
-The first step is to import the IPython ``client`` module and then create a ``MultiEngineClient`` instance::
+The first step is to import the IPython :mod:`IPython.kernel.client` module
+and then create a :class:`MultiEngineClient` instance::
 
 	In [1]: from IPython.kernel import client
 	
 	In [2]: mec = client.MultiEngineClient()
 
-To make sure there are engines connected to the controller, use can get a list of engine ids::
+This form assumes that the :file:`ipcontroller-mec.furl` is in the
+:file:`~./ipython/security` directory on the client's host. If not, the
+location of the ``.furl`` file must be given as an argument to the
+constructor::
+
+    In[2]: mec = client.MultiEngineClient('/path/to/my/ipcontroller-mec.furl')
+
+To make sure there are engines connected to the controller, use can get a list
+of engine ids::
 
 	In [3]: mec.get_ids()
 	Out[3]: [0, 1, 2, 3]
 
 Here we see that there are four engines ready to do work for us.
 
+Quick and easy parallelism
+==========================
+
+In many cases, you simply want to apply a Python function to a sequence of objects, but *in parallel*.  The multiengine interface provides two simple ways of accomplishing this:  a parallel version of :func:`map` and ``@parallel`` function decorator.
+
+Parallel map
+------------
+
+Python's builtin :func:`map` functions allows a function to be applied to a
+sequence element-by-element. This type of code is typically trivial to
+parallelize. In fact, the multiengine interface in IPython already has a
+parallel version of :meth:`map` that works just like its serial counterpart::
+
+	In [63]: serial_result = map(lambda x:x**10, range(32))
+
+	In [64]: parallel_result = mec.map(lambda x:x**10, range(32))
+
+	In [65]: serial_result==parallel_result
+	Out[65]: True
+
+.. note::
+
+    The multiengine interface version of :meth:`map` does not do any load
+    balancing.  For a load balanced version, see the task interface.
+
+.. seealso::
+
+    The :meth:`map` method has a number of options that can be controlled by
+    the :meth:`mapper` method.  See its docstring for more information.
+
+Parallel function decorator
+---------------------------
+
+Parallel functions are just like normal function, but they can be called on sequences and *in parallel*.  The multiengine interface provides a decorator that turns any Python function into a parallel function::
+
+    In [10]: @mec.parallel()
+       ....: def f(x):
+       ....:     return 10.0*x**4
+       ....: 
+
+    In [11]: f(range(32))    # this is done in parallel
+    Out[11]: 
+    [0.0,10.0,160.0,...]
+
+See the docstring for the :meth:`parallel` decorator for options.
+
 Running Python commands
 =======================
 
-The most basic type of operation that can be performed on the engines is to execute Python code. Executing Python code can be done in blocking or non-blocking mode (blocking is default) using the ``execute`` method.
+The most basic type of operation that can be performed on the engines is to
+execute Python code. Executing Python code can be done in blocking or
+non-blocking mode (blocking is default) using the :meth:`execute` method.
 
 Blocking execution
 ------------------
 
-In blocking mode, the ``MultiEngineClient`` object (called ``mec`` in
+In blocking mode, the :class:`MultiEngineClient` object (called ``mec`` in
 these examples) submits the command to the controller, which places the
-command in the engines' queues for execution. The ``execute`` call then
+command in the engines' queues for execution. The :meth:`execute` call then
 blocks until the engines are done executing the command::
 
 	# The default is to run on all engines
@@ -71,7 +129,8 b' blocks until the engines are done executing the command::'
 	[2] In [2]: b=10
 	[3] In [2]: b=10
 
-Python commands can be executed on specific engines by calling execute using the ``targets`` keyword argument::
+Python commands can be executed on specific engines by calling execute using
+the ``targets`` keyword argument::
 
 	In [6]: mec.execute('c=a+b',targets=[0,2])
 	Out[6]: 
@@ -102,7 +161,9 b' Python commands can be executed on specific engines by calling execute using the'
 	[3] In [4]: print c
 	[3] Out[4]: -5
 
-This example also shows one of the most important things about the IPython engines: they have a persistent user namespaces.  The ``execute`` method returns a Python ``dict`` that contains useful information::
+This example also shows one of the most important things about the IPython
+engines: they have a persistent user namespaces. The :meth:`execute` method
+returns a Python ``dict`` that contains useful information::
 
 	In [9]: result_dict = mec.execute('d=10; print d')
 
@@ -118,10 +179,12 b' This example also shows one of the most important things about the IPython engin'
 Non-blocking execution
 ----------------------
 
-In non-blocking mode, ``execute`` submits the command to be executed and then returns a
-``PendingResult`` object immediately. The ``PendingResult`` object gives you a way of getting a
-result at a later time through its ``get_result`` method or ``r`` attribute. This allows you to
-quickly submit long running commands without blocking your local Python/IPython session::
+In non-blocking mode, :meth:`execute` submits the command to be executed and
+then returns a :class:`PendingResult` object immediately. The
+:class:`PendingResult` object gives you a way of getting a result at a later
+time through its :meth:`get_result` method or :attr:`r` attribute. This allows
+you to quickly submit long running commands without blocking your local
+Python/IPython session::
 
 	# In blocking mode
 	In [6]: mec.execute('import time')
@@ -159,7 +222,10 b' quickly submit long running commands without blocking your local Python/IPython '
 	[2] In [3]: time.sleep(10)
 	[3] In [3]: time.sleep(10)
 
-Often, it is desirable to wait until a set of ``PendingResult`` objects are done.  For this, there is a the method ``barrier``.  This method takes a tuple of ``PendingResult`` objects and blocks until all of the associated results are ready::
+Often, it is desirable to wait until a set of :class:`PendingResult` objects
+are done. For this, there is a the method :meth:`barrier`. This method takes a
+tuple of :class:`PendingResult` objects and blocks until all of the associated
+results are ready::
 
 	In [72]: mec.block=False
 
@@ -182,14 +248,16 b' Often, it is desirable to wait until a set of ``PendingResult`` objects are done'
 The ``block`` and ``targets`` keyword arguments and attributes
 --------------------------------------------------------------
 
-Most commands in the multiengine interface (like ``execute``) accept ``block`` and ``targets``
-as keyword arguments. As we have seen above, these keyword arguments control the blocking mode
-and which engines the command is applied to. The ``MultiEngineClient`` class also has ``block``
-and ``targets`` attributes that control the default behavior when the keyword arguments are not
-provided. Thus the following logic is used for ``block`` and ``targets``:
+Most methods in the multiengine interface (like :meth:`execute`) accept
+``block`` and ``targets`` as keyword arguments. As we have seen above, these
+keyword arguments control the blocking mode and which engines the command is
+applied to. The :class:`MultiEngineClient` class also has :attr:`block` and
+:attr:`targets` attributes that control the default behavior when the keyword
+arguments are not provided. Thus the following logic is used for :attr:`block`
+and :attr:`targets`:
 
-	* If no keyword argument is provided, the instance attributes are used.
-	* Keyword argument, if provided override the instance attributes.
+* If no keyword argument is provided, the instance attributes are used.
+* Keyword argument, if provided override the instance attributes.
 
 The following examples demonstrate how to use the instance attributes::
 
@@ -225,14 +293,19 b' The following examples demonstrate how to use the instance attributes::'
 	[3] In [6]: b=10; print b
 	[3] Out[6]: 10
 
-The ``block`` and ``targets`` instance attributes also determine the behavior of the parallel
-magic commands...
+The :attr:`block` and :attr:`targets` instance attributes also determine the
+behavior of the parallel magic commands.
 
 
 Parallel magic commands
 -----------------------
 
-We provide a few IPython magic commands (``%px``, ``%autopx`` and ``%result``) that make it more pleasant to execute Python commands on the engines interactively. These are simply shortcuts to ``execute`` and ``get_result``. The ``%px`` magic executes a single Python command on the engines specified by the `magicTargets``targets` attribute of the ``MultiEngineClient`` instance (by default this is 'all')::
+We provide a few IPython magic commands (``%px``, ``%autopx`` and ``%result``)
+that make it more pleasant to execute Python commands on the engines
+interactively. These are simply shortcuts to :meth:`execute` and
+:meth:`get_result`. The ``%px`` magic executes a single Python command on the
+engines specified by the :attr:`targets` attribute of the
+:class:`MultiEngineClient` instance (by default this is ``'all'``)::
 
 	# Make this MultiEngineClient active for parallel magic commands
 	In [23]: mec.activate()
@@ -277,7 +350,9 b' We provide a few IPython magic commands (``%px``, ``%autopx`` and ``%result``) t'
 	[3] In [9]: print numpy.linalg.eigvals(a)
 	[3] Out[9]: [ 0.83664764 -0.25602658]
 
-The ``%result`` magic gets and prints the stdin/stdout/stderr of the last command executed on each engine.  It is simply a shortcut to the ``get_result`` method::
+The ``%result`` magic gets and prints the stdin/stdout/stderr of the last
+command executed on each engine. It is simply a shortcut to the
+:meth:`get_result` method::
 
 	In [29]: %result
 	Out[29]:
@@ -294,7 +369,8 b' The ``%result`` magic gets and prints the stdin/stdout/stderr of the last comman'
 	[3] In [9]: print numpy.linalg.eigvals(a)
 	[3] Out[9]: [ 0.83664764 -0.25602658]
 
-The ``%autopx`` magic switches to a mode where everything you type is executed on the engines given by the ``targets`` attribute::
+The ``%autopx`` magic switches to a mode where everything you type is executed
+on the engines given by the :attr:`targets` attribute::
 
 	In [30]: mec.block=False
 
@@ -335,51 +411,19 b' The ``%autopx`` magic switches to a mode where everything you type is executed o'
 	[3] In [12]: print "Average max eigenvalue is: ", sum(max_evals)/len(max_evals)
 	[3] Out[12]: Average max eigenvalue is:  10.1158837784
 
-Using the ``with`` statement of Python 2.5
-------------------------------------------
 
-Python 2.5 introduced the ``with`` statement.  The ``MultiEngineClient`` can be used with the ``with`` statement to execute a block of code on the engines indicated by the ``targets`` attribute::
+Moving Python objects around
+============================
 
-	In [3]: with mec:
-	   ...:     client.remote()    # Required so the following code is not run locally
-	   ...:     a = 10
-	   ...:     b = 30
-	   ...:     c = a+b
-	   ...:     
-	   ...:     
-
-	In [4]: mec.get_result()
-	Out[4]: 
-	<Results List>
-	[0] In [1]: a = 10
-	b = 30
-	c = a+b
-
-	[1] In [1]: a = 10
-	b = 30
-	c = a+b
-
-	[2] In [1]: a = 10
-	b = 30
-	c = a+b
-
-	[3] In [1]: a = 10
-	b = 30
-	c = a+b
-
-This is basically another way of calling execute, but one with allows you to avoid writing code in strings.  When used in this way, the attributes ``targets`` and ``block`` are used to control how the code is executed.  For now, if you run code in non-blocking mode you won't have access to the ``PendingResult``.
-
-Moving Python object around
-===========================
-
-In addition to executing code on engines, you can transfer Python objects to and from your
-IPython session and the engines. In IPython, these operations are called ``push`` (sending an
-object to the engines) and ``pull`` (getting an object from the engines).
+In addition to executing code on engines, you can transfer Python objects to
+and from your IPython session and the engines. In IPython, these operations
+are called :meth:`push` (sending an object to the engines) and :meth:`pull`
+(getting an object from the engines).
 
 Basic push and pull
 -------------------
 
-Here are some examples of how you use ``push`` and ``pull``::
+Here are some examples of how you use :meth:`push` and :meth:`pull`::
 
 	In [38]: mec.push(dict(a=1.03234,b=3453))
 	Out[38]: [None, None, None, None]
@@ -415,7 +459,8 b' Here are some examples of how you use ``push`` and ``pull``::'
 	[3] In [13]: print c
 	[3] Out[13]: speed
 
-In non-blocking mode ``push`` and ``pull`` also return ``PendingResult`` objects::
+In non-blocking mode :meth:`push` and :meth:`pull` also return
+:class:`PendingResult` objects::
 
 	In [47]: mec.block=False
 
@@ -428,7 +473,11 b' In non-blocking mode ``push`` and ``pull`` also return ``PendingResult`` objects'
 Push and pull for functions
 ---------------------------
 
-Functions can also be pushed and pulled using ``push_function`` and ``pull_function``::
+Functions can also be pushed and pulled using :meth:`push_function` and
+:meth:`pull_function`::
+
+
+    In [52]: mec.block=True
 
 	In [53]: def f(x):
 	   ....:     return 2.0*x**4
@@ -466,7 +515,10 b' Functions can also be pushed and pulled using ``push_function`` and ``pull_funct'
 Dictionary interface
 --------------------
 
-As a shorthand to ``push`` and ``pull``, the ``MultiEngineClient`` class implements some of the Python dictionary interface. This make the remote namespaces of the engines appear as a local dictionary. Underneath, this uses ``push`` and ``pull``::
+As a shorthand to :meth:`push` and :meth:`pull`, the
+:class:`MultiEngineClient` class implements some of the Python dictionary
+interface. This make the remote namespaces of the engines appear as a local
+dictionary. Underneath, this uses :meth:`push` and :meth:`pull`::
 
 	In [50]: mec.block=True
 
@@ -478,11 +530,13 b' As a shorthand to ``push`` and ``pull``, the ``MultiEngineClient`` class impleme'
 Scatter and gather
 ------------------
 
-Sometimes it is useful to partition a sequence and push the partitions to different engines. In
-MPI language, this is know as scatter/gather and we follow that terminology. However, it is
-important to remember that in IPython ``scatter`` is from the interactive IPython session to
-the engines and ``gather`` is from the engines back to the interactive IPython session. For
-scatter/gather operations between engines, MPI should be used::
+Sometimes it is useful to partition a sequence and push the partitions to
+different engines. In MPI language, this is know as scatter/gather and we
+follow that terminology. However, it is important to remember that in
+IPython's :class:`MultiEngineClient` class, :meth:`scatter` is from the
+interactive IPython session to the engines and :meth:`gather` is from the
+engines back to the interactive IPython session. For scatter/gather operations
+between engines, MPI should be used::
 
 	In [58]: mec.scatter('a',range(16))
 	Out[58]: [None, None, None, None]
@@ -510,24 +564,12 b' scatter/gather operations between engines, MPI should be used::'
 Other things to look at
 =======================
 
-Parallel map
-------------
-
-Python's builtin ``map`` functions allows a function to be applied to a sequence element-by-element.  This type of code is typically trivial to parallelize.  In fact, the MultiEngine interface in IPython already has a parallel version of ``map`` that works just like its serial counterpart::
-
-	In [63]: serial_result = map(lambda x:x**10, range(32))
-
-	In [64]: parallel_result = mec.map(lambda x:x**10, range(32))
-
-	In [65]: serial_result==parallel_result
-	Out[65]: True
-
-As you would expect, the parallel version of ``map`` is also influenced by the ``block`` and ``targets`` keyword arguments and attributes.
-
 How to do parallel list comprehensions
 --------------------------------------
 
-In many cases list comprehensions are nicer than using the map function.  While we don't have fully parallel list comprehensions, it is simple to get the basic effect using ``scatter`` and ``gather``::
+In many cases list comprehensions are nicer than using the map function. While
+we don't have fully parallel list comprehensions, it is simple to get the
+basic effect using :meth:`scatter` and :meth:`gather`::
 
 	In [66]: mec.scatter('x',range(64))
 	Out[66]: [None, None, None, None]
@@ -547,10 +589,16 b' In many cases list comprehensions are nicer than using the map function. While '
 	In [69]: print y
 	[0, 1, 1024, 59049, 1048576, 9765625, 60466176, 282475249, 1073741824,...]
 
-Parallel Exceptions
+Parallel exceptions
 -------------------
 
-In the MultiEngine interface, parallel commands can raise Python exceptions, just like serial commands.  But, it is a little subtle, because a single parallel command can actually raise multiple exceptions (one for each engine the command was run on).  To express this idea, the MultiEngine interface has a ``CompositeError`` exception class that will be raised in most cases.  The ``CompositeError`` class is a special type of exception that wraps one or more other types of exceptions.  Here is how it works::
+In the multiengine interface, parallel commands can raise Python exceptions,
+just like serial commands. But, it is a little subtle, because a single
+parallel command can actually raise multiple exceptions (one for each engine
+the command was run on). To express this idea, the MultiEngine interface has a
+:exc:`CompositeError` exception class that will be raised in most cases. The
+:exc:`CompositeError` class is a special type of exception that wraps one or
+more other types of exceptions. Here is how it works::
 
 	In [76]: mec.block=True
 
@@ -580,7 +628,7 b' In the MultiEngine interface, parallel commands can raise Python exceptions, jus'
 	[2:execute]: ZeroDivisionError: integer division or modulo by zero
 	[3:execute]: ZeroDivisionError: integer division or modulo by zero
 
-Notice how the error message printed when ``CompositeError`` is raised has information about the individual exceptions that were raised on each engine.  If you want, you can even raise one of these original exceptions::
+Notice how the error message printed when :exc:`CompositeError` is raised has information about the individual exceptions that were raised on each engine.  If you want, you can even raise one of these original exceptions::
 
 	In [80]: try:
 	   ....:     mec.execute('1/0')
@@ -602,7 +650,9 b' Notice how the error message printed when ``CompositeError`` is raised has infor'
 
 	ZeroDivisionError: integer division or modulo by zero
 
-If you are working in IPython, you can simple type ``%debug`` after one of these ``CompositeError`` is raised, and inspect the exception instance::
+If you are working in IPython, you can simple type ``%debug`` after one of
+these :exc:`CompositeError` exceptions is raised, and inspect the exception
+instance::
 
 	In [81]: mec.execute('1/0')
 	---------------------------------------------------------------------------
@@ -679,6 +729,11 b' If you are working in IPython, you can simple type ``%debug`` after one of these'
 
 	ZeroDivisionError: integer division or modulo by zero
 
+.. note::
+
+    The above example appears to be broken right now because of a change in 
+    how we are using Twisted.
+
 All of this same error handling magic even works in non-blocking mode::
 
 	In [83]: mec.block=False

docs/source/parallel/parallel_task.txt

@@ -1,240 +1,93 b''
 .. _paralleltask:
 
-=================================
-The IPython Task interface
-=================================
+==========================
+The IPython task interface
+==========================
 
 .. contents::
 
-The ``Task`` interface to the controller presents the engines as a fault tolerant, dynamic load-balanced system or workers. Unlike the ``MultiEngine`` interface, in the ``Task`` interface, the user have no direct access to individual engines. In some ways, this interface is simpler, but in other ways it is more powerful. Best of all the user can use both of these interfaces at the same time to take advantage or both of their strengths.  When the user can break up the user's work into segments that do not depend on previous execution, the ``Task`` interface is ideal.  But it also has more power and flexibility, allowing the user to guide the distribution of jobs, without having to assign Tasks to engines explicitly.
+The task interface to the controller presents the engines as a fault tolerant, dynamic load-balanced system or workers. Unlike the multiengine interface, in the task interface, the user have no direct access to individual engines. In some ways, this interface is simpler, but in other ways it is more powerful.
+
+Best of all the user can use both of these interfaces running at the same time to take advantage or both of their strengths.  When the user can break up the user's work into segments that do not depend on previous execution, the task interface is ideal.  But it also has more power and flexibility, allowing the user to guide the distribution of jobs, without having to assign tasks to engines explicitly.
 
 Starting the IPython controller and engines
 ===========================================
 
-To follow along with this tutorial, the user will need to start the IPython
-controller and four IPython engines. The simplest way of doing this is to
-use the ``ipcluster`` command::
+To follow along with this tutorial, you will need to start the IPython
+controller and four IPython engines. The simplest way of doing this is to use
+the :command:`ipcluster` command::
 
 	$ ipcluster -n 4
 	
-For more detailed information about starting the controller and engines, see our :ref:`introduction <ip1par>` to using IPython for parallel computing.
+For more detailed information about starting the controller and engines, see
+our :ref:`introduction <ip1par>` to using IPython for parallel computing.
+
+Creating a ``TaskClient`` instance
+=========================================
+
+The first step is to import the IPython :mod:`IPython.kernel.client` module
+and then create a :class:`TaskClient` instance::
+
+	In [1]: from IPython.kernel import client
+	
+	In [2]: tc = client.TaskClient()
+
+This form assumes that the :file:`ipcontroller-tc.furl` is in the
+:file:`~./ipython/security` directory on the client's host. If not, the
+location of the ``.furl`` file must be given as an argument to the
+constructor::
+
+    In[2]: mec = client.TaskClient('/path/to/my/ipcontroller-tc.furl')
+
+Quick and easy parallelism
+==========================
+
+In many cases, you simply want to apply a Python function to a sequence of objects, but *in parallel*.  Like the multiengine interface, the task interface provides two simple ways of accomplishing this:  a parallel version of :func:`map` and ``@parallel`` function decorator.  However, the verions in the task interface have one important difference:  they are dynamically load balanced.  Thus, if the execution time per item varies significantly, you should use the versions in the task interface.
+
+Parallel map
+------------
+
+The parallel :meth:`map` in the task interface is similar to that in the multiengine interface::
+
+	In [63]: serial_result = map(lambda x:x**10, range(32))
+
+	In [64]: parallel_result = tc.map(lambda x:x**10, range(32))
+
+	In [65]: serial_result==parallel_result
+	Out[65]: True
+
+Parallel function decorator
+---------------------------
+
+Parallel functions are just like normal function, but they can be called on sequences and *in parallel*.  The multiengine interface provides a decorator that turns any Python function into a parallel function::
 
-The magic here is that this single controller and set of engines is running both the MultiEngine and ``Task`` interfaces simultaneously.
+    In [10]: @tc.parallel()
+       ....: def f(x):
+       ....:     return 10.0*x**4
+       ....: 
 
-QuickStart Task Farming
-=======================
+    In [11]: f(range(32))    # this is done in parallel
+    Out[11]: 
+    [0.0,10.0,160.0,...]
 
-First, a quick example of how to start running the most basic Tasks.
-The first step is to import the IPython ``client`` module and then create a ``TaskClient`` instance::
-    
-    In [1]: from IPython.kernel import client
-    
-    In [2]: tc = client.TaskClient()
+More details
+============
 
-Then the user wrap the commands the user want to run in Tasks::
+The :class:`TaskClient` has many more powerful features that allow quite a bit of flexibility in how tasks are defined and run.  The next places to look are in the following classes:
 
-    In [3]: tasklist = []
-    In [4]: for n in range(1000):
-        ...     tasklist.append(client.Task("a = %i"%n, pull="a"))
+* :class:`IPython.kernel.client.TaskClient`
+* :class:`IPython.kernel.client.StringTask`
+* :class:`IPython.kernel.client.MapTask`
 
-The first argument of the ``Task`` constructor is a string, the command to be executed.  The most important optional keyword argument is ``pull``, which can be a string or list of strings, and it specifies the variable names to be saved as results of the ``Task``.
+The following is an overview of how to use these classes together:
 
-Next, the user need to submit the Tasks to the ``TaskController`` with the ``TaskClient``::
+1. Create a :class:`TaskClient`.
+2. Create one or more instances of :class:`StringTask` or :class:`MapTask`
+   to define your tasks.
+3. Submit your tasks to using the :meth:`run` method of your
+   :class:`TaskClient` instance.
+4. Use :meth:`TaskClient.get_task_result` to get the results of the
+   tasks.
 
-    In [5]: taskids = [ tc.run(t) for t in tasklist ]
+We are in the process of developing more detailed information about the task interface.  For now, the docstrings of the :class:`TaskClient`, :class:`StringTask` and :class:`MapTask` classes should be consulted.
 
-This will give the user a list of the TaskIDs used by the controller to keep track of the Tasks and their results.  Now at some point the user are going to want to get those results back.  The ``barrier`` method allows the user to wait for the Tasks to finish running::
-
-    In [6]: tc.barrier(taskids)
-
-This command will block until all the Tasks in ``taskids`` have finished.  Now, the user probably want to look at the user's results::
-
-    In [7]: task_results = [ tc.get_task_result(taskid) for taskid in taskids ]
-
-Now the user have a list of ``TaskResult`` objects, which have the actual result as a dictionary, but also keep track of some useful metadata about the ``Task``::
-
-    In [8]: tr = ``Task``_results[73]
-    
-    In [9]: tr
-    Out[9]: ``TaskResult``[ID:73]:{'a':73}
-    
-    In [10]: tr.engineid
-    Out[10]: 1
-    
-    In [11]: tr.submitted, tr.completed, tr.duration
-    Out[11]: ("2008/03/08 03:41:42", "2008/03/08 03:41:44", 2.12345)
-
-The actual results are stored in a dictionary, ``tr.results``, and a namespace object ``tr.ns`` which accesses the result keys by attribute::
-
-    In [12]: tr.results['a']
-    Out[12]: 73
-    
-    In [13]: tr.ns.a
-    Out[13]: 73
-
-That should cover the basics of running simple Tasks.  There are several more powerful things the user can do with Tasks covered later.  The most useful probably being using a ``MutiEngineClient`` interface to initialize all the engines with the import dependencies necessary to run the user's Tasks.
-
-There are many options for running and managing Tasks. The best way to learn further about the ``Task`` interface is to study the examples in ``docs/examples``. If the user do so and learn a lots about this interface, we encourage the user to expand this documentation about the ``Task`` system.
-
-Overview of the Task System
-===========================
-
-The user's view of the ``Task`` system has three basic objects:  The ``TaskClient``, the ``Task``, and the ``TaskResult``.  The names of these three objects well indicate their role.
-
-The ``TaskClient`` is the user's ``Task`` farming connection to the IPython cluster.  Unlike the ``MultiEngineClient``, the ``TaskControler`` handles all the scheduling and distribution of work, so the ``TaskClient`` has no notion of engines, it just submits Tasks and requests their results.  The Tasks are described as ``Task`` objects, and their results are wrapped in ``TaskResult`` objects.  Thus, there are very few necessary methods for the user to manage.
-
-Inside the task system is a Scheduler object, which assigns tasks to workers.  The default scheduler is a simple FIFO queue.  Subclassing the Scheduler should be easy, just implementing your own priority system.
-
-The TaskClient
-==============
-
-The ``TaskClient`` is the object the user use to connect to the ``Controller`` that is managing the user's Tasks.  It is the analog of the ``MultiEngineClient`` for the standard IPython multiplexing interface.  As with all client interfaces, the first step is to import the IPython Client Module::
-
-    In [1]: from IPython.kernel import client
-
-Just as with the ``MultiEngineClient``, the user create the ``TaskClient`` with a tuple, containing the ip-address and port of the ``Controller``.  the ``client`` module conveniently has the default address of the ``Task`` interface of the controller.  Creating a default ``TaskClient`` object would be done with this::
-
-    In [2]: tc = client.TaskClient(client.default_task_address)
-
-or, if the user want to specify a non default location of the ``Controller``, the user can specify explicitly::
-
-    In [3]: tc = client.TaskClient(("192.168.1.1", 10113))
-
-As discussed earlier, the ``TaskClient`` only has a few basic methods.
-
- * ``tc.run(task)``
-    ``run`` is the method by which the user submits Tasks.  It takes exactly one argument, a ``Task`` object.  All the advanced control of ``Task`` behavior is handled by properties of the ``Task`` object, rather than the submission command, so they will be discussed later in the `Task`_ section.  ``run`` returns an integer, the ``Task``ID by which the ``Task`` and its results can be tracked and retrieved::
-    
-        In [4]: ``Task``ID = tc.run(``Task``)
-    
- * ``tc.get_task_result(taskid, block=``False``)``
-    ``get_task_result`` is the method by which results are retrieved.  It takes a single integer argument, the ``Task``ID`` of the result the user wish to retrieve.  ``get_task_result`` also takes a keyword argument ``block``.  ``block`` specifies whether the user actually want to wait for the result.  If ``block`` is false, as it is by default, ``get_task_result`` will return immediately.  If the ``Task`` has completed, it will return the ``TaskResult`` object for that ``Task``.  But if the ``Task`` has not completed, it will return ``None``.  If the user specify ``block=``True``, then ``get_task_result`` will wait for the ``Task`` to complete, and always return the ``TaskResult`` for the requested ``Task``.
- * ``tc.barrier(taskid(s))``
-    ``barrier`` is a synchronization method.  It takes exactly one argument, a ``Task``ID or list of taskIDs.  ``barrier`` will block until all the specified Tasks have completed.  In practice, a barrier is often called between the ``Task`` submission section of the code and the result gathering section::
-        
-        In [5]: taskIDs = [ tc.run(``Task``) for ``Task`` in myTasks ]
-        
-        In [6]: tc.get_task_result(taskIDs[-1]) is None
-        Out[6]: ``True``
-        
-        In [7]: tc.barrier(``Task``ID)
-        
-        In [8]: results = [ tc.get_task_result(tid) for tid in taskIDs ]
-        
- * ``tc.queue_status(verbose=``False``)``
-    ``queue_status`` is a method for querying the state of the ``TaskControler``.  ``queue_status`` returns a dict of the form::
-        
-        {'scheduled': Tasks that have been submitted but yet run
-         'pending'  : Tasks that are currently running
-         'succeeded': Tasks that have completed successfully
-         'failed'   : Tasks that have finished with a failure
-        }
-    
-    if @verbose is not specified (or is ``False``), then the values of the dict are integers - the number of Tasks in each state.  if @verbose is ``True``, then each element in the dict is a list of the taskIDs in that state::
-    
-        In [8]: tc.queue_status()
-        Out[8]: {'scheduled': 4,
-                'pending'  : 2,
-                'succeeded': 5,
-                'failed'   : 1
-                }
-        
-        In [9]: tc.queue_status(verbose=True)
-        Out[9]: {'scheduled': [8,9,10,11],
-                'pending'  : [6,7],
-                'succeeded': [0,1,2,4,5],
-                'failed'   : [3]
-                }
-        
- * ``tc.abort(taskid)``
-    ``abort`` allows the user to abort Tasks that have already been submitted.  ``abort`` will always return immediately.  If the ``Task`` has completed, ``abort`` will raise an ``IndexError ``Task`` Already Completed``.  An obvious case for ``abort`` would be where the user submits a long-running ``Task`` with a number of retries (see ``Task``_ section for how to specify retries) in an interactive session, but realizes there has been a typo.  The user can then abort the ``Task``, preventing certain failures from cluttering up the queue.  It can also be used for parallel search-type problems, where only one ``Task`` will give the solution, so once the user find the solution, the user would want to abort all remaining Tasks to prevent wasted work.
- * ``tc.spin()``
-    ``spin`` simply triggers the scheduler in the ``TaskControler``.  Under most normal circumstances, this will do nothing.  The primary known usage case involves the ``Task`` dependency (see `Dependencies`_).  The dependency is a function of an Engine's ``properties``, but changing the ``properties`` via the ``MutliEngineClient`` does not trigger a reschedule event.  The main example case for this requires the following event sequence:
-     * ``engine`` is available, ``Task`` is submitted, but ``engine`` does not have ``Task``'s dependencies.
-     * ``engine`` gets necessary dependencies while no new Tasks are submitted or completed.
-     * now ``engine`` can run ``Task``, but a ``Task`` event is required for the ``TaskControler`` to try scheduling ``Task`` again.
-     
-    ``spin`` is just an empty ping method to ensure that the Controller has scheduled all available Tasks, and should not be needed under most normal circumstances.
-
-That covers the ``TaskClient``, a simple interface to the cluster.  With this, the user can submit jobs (and abort if necessary), request their results, synchronize on arbitrary subsets of jobs.
-
-.. _task: The Task Object
-
-The Task Object
-===============
-
-The ``Task`` is the basic object for describing a job.  It can be used in a very simple manner, where the user just specifies a command string to be executed as the ``Task``.  The usage of this first argument is exactly the same as the ``execute`` method of the ``MultiEngine`` (in fact, ``execute`` is called to run the code)::
-
-    In [1]: t = client.Task("a = str(id)")
-    
-This ``Task`` would run, and store the string representation of the ``id`` element in ``a`` in each worker's namespace, but it is fairly useless because the user does not know anything about the state of the ``worker`` on which it ran at the time of retrieving results.  It is important that each ``Task`` not expect the state of the ``worker`` to persist after the ``Task`` is completed.  
-There are many different situations for using ``Task`` Farming, and the ``Task`` object has many attributes for use in customizing the ``Task`` behavior.  All of a ``Task``'s attributes may be specified in the constructor, through keyword arguments, or after ``Task`` construction through attribute assignment.
-
-Data Attributes
-***************
-It is likely that the user may want to move data around before or after executing the ``Task``.  We provide methods of sending data to initialize the worker's namespace, and specifying what data to bring back as the ``Task``'s results.
-
- * pull = []
-    The obvious case is as above, where ``t`` would execute and store the result of ``myfunc`` in ``a``, it is likely that the user would want to bring ``a`` back to their namespace.  This is done through the ``pull`` attribute.  ``pull`` can be a string or list of strings, and it specifies the names of variables to be retrieved.  The ``TaskResult`` object retrieved by ``get_task_result`` will have a dictionary of keys and values, and the ``Task``'s ``pull`` attribute determines what goes into it::
-        
-        In [2]: t = client.Task("a = str(id)", pull = "a")
-        
-        In [3]: t = client.Task("a = str(id)", pull = ["a", "id"])
-        
- * push = {}
-    A user might also want to initialize some data into the namespace before the code part of the ``Task`` is run.  Enter ``push``.  ``push`` is a dictionary of key/value pairs to be loaded from the user's namespace into the worker's immediately before execution::
-
-        In [4]: t = client.Task("a = f(submitted)", push=dict(submitted=time.time()), pull="a")
-
-push and pull result directly in calling an ``engine``'s ``push`` and ``pull`` methods before and after ``Task`` execution respectively, and thus their api is the same.
-
-Namespace Cleaning
-******************
-When a user is running a large number of Tasks, it is likely that the namespace of the worker's could become cluttered.  Some Tasks might be sensitive to clutter, while others might be known to cause namespace pollution.  For these reasons, Tasks have two boolean attributes for cleaning up the namespace.
-
- * ``clear_after``
-    if clear_after is specified ``True``, the worker on which the ``Task`` was run will be reset (via ``engine.reset``) upon completion of the ``Task``.  This can be useful for both Tasks that produce clutter or Tasks whose intermediate data one might wish to be kept private::
-    
-        In [5]: t = client.Task("a = range(1e10)", pull = "a",clear_after=True)
-        
-    
- * ``clear_before``
-    as one might guess, clear_before is identical to ``clear_after``, but it takes place before the ``Task`` is run.  This ensures that the ``Task`` runs on a fresh worker::
-
-        In [6]: t = client.Task("a = globals()", pull = "a",clear_before=True)
-
-Of course, a user can both at the same time, ensuring that all workers are clear except when they are currently running a job.  Both of these default to ``False``.
-
-Fault Tolerance
-***************
-It is possible that Tasks might fail, and there are a variety of reasons this could happen.  One might be that the worker it was running on disconnected, and there was nothing wrong with the ``Task`` itself.  With the fault tolerance attributes of the ``Task``, the user can specify how many times to resubmit the ``Task``, and what to do if it never succeeds.
-
- * ``retries``
-    ``retries`` is an integer, specifying the number of times a ``Task`` is to be retried.  It defaults to zero.  It is often a good idea for this number to be 1 or 2, to protect the ``Task`` from disconnecting engines, but not a large number.  If a ``Task`` is failing 100 times, there is probably something wrong with the ``Task``.  The canonical bad example:
-    
-        In [7]: t = client.Task("os.kill(os.getpid(), 9)", retries=99)
-        
-    This would actually take down 100 workers.
-    
- * ``recovery_task``
-    ``recovery_task`` is another ``Task`` object, to be run in the event of the original ``Task`` still failing after running out of retries.  Since ``recovery_task`` is another ``Task`` object, it can have its own ``recovery_task``.  The chain of Tasks is limitless, except loops are not allowed (that would be bad!).
-
-Dependencies
-************
-Dependencies are the most powerful part of the ``Task`` farming system, because it allows the user to do some classification of the workers, and guide the ``Task`` distribution without meddling with the controller directly.  It makes use of two objects - the ``Task``'s ``depend`` attribute, and the engine's ``properties``.  See the `MultiEngine`_ reference for how to use engine properties.  The engine properties api exists for extending IPython, allowing conditional execution and new controllers that make decisions based on properties of its engines.  Currently the ``Task`` dependency is the only internal use of the properties api.
-
-.. _MultiEngine: ./parallel_multiengine
-
-The ``depend`` attribute of a ``Task`` must be a function of exactly one argument, the worker's properties dictionary, and it should return ``True`` if the ``Task`` should be allowed to run on the worker and ``False`` if not.  The usage in the controller is fault tolerant, so exceptions raised by ``Task.depend`` will be ignored and functionally equivalent to always returning ``False``.  Tasks`` with invalid ``depend`` functions will never be assigned to a worker::
-
-    In [8]: def dep(properties):
-        ...     return properties["RAM"] > 2**32 # have at least 4GB
-    In [9]: t = client.Task("a = bigfunc()", depend=dep)
-    
-It is important to note that assignment of values to the properties dict is done entirely by the user, either locally (in the engine) using the EngineAPI, or remotely, through the ``MultiEngineClient``'s get/set_properties methods.
-
-
-
-
-
-    

scripts/ipython_win_post_install.py

@@ -2,6 +2,7 b''
 """Windows-specific part of the installation"""
 
 import os, sys, shutil
+pjoin = os.path.join
 
 def mkshortcut(target,description,link_file,*args,**kw):
     """make a shortcut if it doesn't exist, and register its creation"""
@@ -11,69 +12,85 b' def mkshortcut(target,description,link_file,*args,**kw):'
 
 def install():
     """Routine to be run by the win32 installer with the -install switch."""
-
+    
     from IPython.Release import version
-
+    
     # Get some system constants
     prefix = sys.prefix
-    python = prefix + r'\python.exe'
-    # Lookup path to common startmenu ...
-    ip_dir = get_special_folder_path('CSIDL_COMMON_PROGRAMS') + r'\IPython'
+    python = pjoin(prefix, 'python.exe')
     
-    # Some usability warnings at installation time.  I don't want them at the
-    # top-level, so they don't appear if the user is uninstalling.
-    try:
-        import ctypes
-    except ImportError:
-        print ('To take full advantage of IPython, you need ctypes from:\n'
-               'http://sourceforge.net/projects/ctypes')
-
-    try:
-        import win32con
-    except ImportError:
-        print ('To take full advantage of IPython, you need pywin32 from:\n'
-               'http://starship.python.net/crew/mhammond/win32/Downloads.html')
-
-    try:
-        import readline
-    except ImportError:
-        print ('To take full advantage of IPython, you need readline from:\n'
-               'https://launchpad.net/pyreadline')
-
-    ipybase = '"' + prefix + r'\scripts\ipython"'
+    # Lookup path to common startmenu ...
+    ip_start_menu = pjoin(get_special_folder_path('CSIDL_COMMON_PROGRAMS'), 'IPython')
     # Create IPython entry ...
-    if not os.path.isdir(ip_dir):
-        os.mkdir(ip_dir)
-        directory_created(ip_dir)
-
-    # Create program shortcuts ...
-    f = ip_dir + r'\IPython.lnk'
-    a = ipybase
-    mkshortcut(python,'IPython',f,a)
-
-    f = ip_dir + r'\pysh.lnk'
-    a = ipybase+' -p sh'
-    mkshortcut(python,'IPython (command prompt mode)',f,a)
-
-    f = ip_dir + r'\pylab.lnk'
-    a = ipybase+' -pylab'
-    mkshortcut(python,'IPython (PyLab mode)',f,a)
-
-    f = ip_dir + r'\scipy.lnk'
-    a = ipybase+' -pylab -p scipy'
-    mkshortcut(python,'IPython (scipy profile)',f,a)
-
+    if not os.path.isdir(ip_start_menu):
+        os.mkdir(ip_start_menu)
+        directory_created(ip_start_menu)
+    
+    # Create .py and .bat files to make things available from
+    # the Windows command line.  Thanks to the Twisted project
+    # for this logic!
+    programs = [
+        'ipython',
+        'iptest',
+        'ipcontroller',
+        'ipengine',
+        'ipcluster',
+        'ipythonx',
+        'ipython-wx',
+        'irunner'
+    ]
+    scripts = pjoin(prefix,'scripts')
+    for program in programs:
+        raw = pjoin(scripts, program)
+        bat = raw + '.bat'
+        py = raw + '.py'
+        # Create .py versions of the scripts
+        shutil.copy(raw, py)
+        # Create .bat files for each of the scripts
+        bat_file = file(bat,'w')
+        bat_file.write("@%s %s %%*" % (python, py))
+        bat_file.close()
+            
+    # Now move onto setting the Start Menu up
+    ipybase = pjoin(scripts, 'ipython')
+
+    link = pjoin(ip_start_menu, 'IPython.lnk')
+    cmd = '"%s"' % ipybase
+    mkshortcut(python,'IPython',link,cmd)
+    
+    link = pjoin(ip_start_menu, 'pysh.lnk')
+    cmd = '"%s" -p sh' % ipybase
+    mkshortcut(python,'IPython (command prompt mode)',link,cmd)
+    
+    link = pjoin(ip_start_menu, 'pylab.lnk')
+    cmd = '"%s" -pylab' % ipybase
+    mkshortcut(python,'IPython (PyLab mode)',link,cmd)
+    
+    link = pjoin(ip_start_menu, 'scipy.lnk')
+    cmd = '"%s" -pylab -p scipy' % ipybase
+    mkshortcut(python,'IPython (scipy profile)',link,cmd)
+    
+    link = pjoin(ip_start_menu, 'IPython test suite.lnk')
+    cmd = '"%s" -vv' % pjoin(scripts, 'iptest')
+    mkshortcut(python,'Run the IPython test suite',link,cmd)
+    
+    link = pjoin(ip_start_menu, 'ipcontroller.lnk')
+    cmd = '"%s" -xy' % pjoin(scripts, 'ipcontroller')
+    mkshortcut(python,'IPython controller',link,cmd)
+    
+    link = pjoin(ip_start_menu, 'ipengine.lnk')
+    cmd = '"%s"' % pjoin(scripts, 'ipengine')
+    mkshortcut(python,'IPython engine',link,cmd)
+    
     # Create documentation shortcuts ...
     t = prefix + r'\share\doc\ipython\manual\ipython.pdf'
-    f = ip_dir + r'\Manual in PDF.lnk'
+    f = ip_start_menu + r'\Manual in PDF.lnk'
     mkshortcut(t,r'IPython Manual - PDF-Format',f)
-
+    
     t = prefix + r'\share\doc\ipython\manual\html\index.html'
-    f = ip_dir + r'\Manual in HTML.lnk'
+    f = ip_start_menu + r'\Manual in HTML.lnk'
     mkshortcut(t,'IPython Manual - HTML-Format',f)
-
-    # make ipython.py
-    shutil.copy(prefix + r'\scripts\ipython', prefix + r'\scripts\ipython.py')
+    
     
 def remove():
     """Routine to be run by the win32 installer with the -remove switch."""

setupbase.py

@@ -115,6 +115,7 b' def find_packages():'
     add_package(packages, 'kernel', config=True, tests=True, scripts=True)
     add_package(packages, 'kernel.core', config=True, tests=True)
     add_package(packages, 'testing', tests=True)
+    add_package(packages, 'tests')
     add_package(packages, 'testing.plugin', tests=False)
     add_package(packages, 'tools', tests=True)
     add_package(packages, 'UserConfig')

setupegg.py

@@ -1,14 +1,8 b''
 #!/usr/bin/env python
 """Wrapper to run setup.py using setuptools."""
 
-import os
 import sys
 
-# Add my local path to sys.path
-home = os.environ['HOME']
-sys.path.insert(0,'%s/usr/local/lib/python%s/site-packages' %
-                (home,sys.version[:3]))
-
 # now, import setuptools and call the actual setup
 import setuptools
 # print sys.argv

tools/release

@@ -10,17 +10,15 b' echo'
 echo "Releasing IPython version $version"
 echo "=================================="
 
-echo "Marking ChangeLog with release information and making NEWS file..."
-
-# Clean up build/dist directories
-rm -rf $ipdir/build/*
-rm -rf $ipdir/dist/*
-
 # Perform local backup
 cd $ipdir/tools
 ./make_tarball.py
 mv ipython-*.tgz $ipbackupdir
 
+# Clean up build/dist directories
+rm -rf $ipdir/build/*
+rm -rf $ipdir/dist/*
+
 # Build source and binary distros
 cd $ipdir
 ./setup.py sdist --formats=gztar
@@ -28,8 +26,8 b' cd $ipdir'
 # Build version-specific RPMs, where we must use the --python option to ensure
 # that the resulting RPM is really built with the requested python version (so
 # things go to lib/python2.X/...)
-python2.4 ./setup.py bdist_rpm --binary-only --release=py24 --python=/usr/bin/python2.4
-python2.5 ./setup.py bdist_rpm --binary-only --release=py25 --python=/usr/bin/python2.5
+#python2.4 ./setup.py bdist_rpm --binary-only --release=py24 --python=/usr/bin/python2.4
+#python2.5 ./setup.py bdist_rpm --binary-only --release=py25 --python=/usr/bin/python2.5
 
 # Build eggs
 python2.4 ./setup_bdist_egg.py

tools/testrel

@@ -15,8 +15,8 b' cd $ipdir'
 ./setup.py sdist --formats=gztar
 
 # Build rpms
-#python2.4 ./setup.py bdist_rpm --binary-only --release=py24 --python=/usr/bin/python2.4
-#python2.5 ./setup.py bdist_rpm --binary-only --release=py25 --python=/usr/bin/python2.5
+python2.4 ./setup.py bdist_rpm --binary-only --release=py24 --python=/usr/bin/python2.4
+python2.5 ./setup.py bdist_rpm --binary-only --release=py25 --python=/usr/bin/python2.5
 
 # Build eggs
 python2.4 ./setup_bdist_egg.py