Work in the documentation.

IPython/lib/inputhookwx.py

@@ -51,19 +51,22 b' def inputhook_wx1():'
     This approach seems to work, but its performance is not great as it 
     relies on having PyOS_InputHook called regularly.
     """
-    app = wx.GetApp()
-    if app is not None:
-        assert wx.Thread_IsMain()
-
-        # Make a temporary event loop and process system events until
-        # there are no more waiting, then allow idle events (which
-        # will also deal with pending or posted wx events.)
-        evtloop = wx.EventLoop()
-        ea = wx.EventLoopActivator(evtloop)
-        while evtloop.Pending():
-            evtloop.Dispatch()
-        app.ProcessIdle()
-        del ea
+    try:
+        app = wx.GetApp()
+        if app is not None:
+            assert wx.Thread_IsMain()
+
+            # Make a temporary event loop and process system events until
+            # there are no more waiting, then allow idle events (which
+            # will also deal with pending or posted wx events.)
+            evtloop = wx.EventLoop()
+            ea = wx.EventLoopActivator(evtloop)
+            while evtloop.Pending():
+                evtloop.Dispatch()
+            app.ProcessIdle()
+            del ea
+    except KeyboardInterrupt:
+        pass
     return 0
 
 class EventLoopTimer(wx.Timer):
@@ -102,13 +105,16 b' def inputhook_wx2():'
     but eventually performance would suffer from calling select/kbhit too 
     often.
     """
-    app = wx.GetApp()
-    if app is not None:
-        assert wx.Thread_IsMain()
-        elr = EventLoopRunner()
-        # As this time is made shorter, keyboard response improves, but idle
-        # CPU load goes up.  10 ms seems like a good compromise.
-        elr.Run(time=10)  # CHANGE time here to control polling interval
+    try:
+        app = wx.GetApp()
+        if app is not None:
+            assert wx.Thread_IsMain()
+            elr = EventLoopRunner()
+            # As this time is made shorter, keyboard response improves, but idle
+            # CPU load goes up.  10 ms seems like a good compromise.
+            elr.Run(time=10)  # CHANGE time here to control polling interval
+    except KeyboardInterrupt:
+        pass
     return 0
 
 def inputhook_wx3():
@@ -119,49 +125,54 b' def inputhook_wx3():'
     time.sleep is inserted.  This is needed, otherwise, CPU usage is at 100%.
     This sleep time should be tuned though for best performance.
     """
-    app = wx.GetApp()
-    if app is not None:
-        assert wx.Thread_IsMain()
-
-        # The import of wx on Linux sets the handler for signal.SIGINT
-        # to 0.  This is a bug in wx or gtk.  We fix by just setting it
-        # back to the Python default.
-        if not callable(signal.getsignal(signal.SIGINT)):
-            signal.signal(signal.SIGINT, signal.default_int_handler)
-
-        evtloop = wx.EventLoop()
-        ea = wx.EventLoopActivator(evtloop)
-        t = clock()
-        while not stdin_ready():
-            while evtloop.Pending():
-                t = clock()
-                evtloop.Dispatch()
-            app.ProcessIdle()
-            # We need to sleep at this point to keep the idle CPU load
-            # low.  However, if sleep to long, GUI response is poor.  As 
-            # a compromise, we watch how often GUI events are being processed
-            # and switch between a short and long sleep time.  Here are some
-            # stats useful in helping to tune this.
-            # time    CPU load
-            # 0.001   13%
-            # 0.005   3%
-            # 0.01    1.5%
-            # 0.05    0.5%
-            used_time = clock() - t
-            if used_time > 5*60.0:
-                # print 'Sleep for 5 s'  # dbg
-                time.sleep(5.0)
-            elif used_time > 10.0:
-                # print 'Sleep for 1 s'  # dbg
-                time.sleep(1.0)
-            elif used_time > 0.1:
-                # Few GUI events coming in, so we can sleep longer
-                # print 'Sleep for 0.05 s'  # dbg
-                time.sleep(0.05)
-            else:
-                # Many GUI events coming in, so sleep only very little
-                time.sleep(0.001)
-        del ea
+    # We need to protect against a user pressing Control-C when IPython is
+    # idle and this is running. We trap KeyboardInterrupt and pass.
+    try:
+        app = wx.GetApp()
+        if app is not None:
+            assert wx.Thread_IsMain()
+
+            # The import of wx on Linux sets the handler for signal.SIGINT
+            # to 0.  This is a bug in wx or gtk.  We fix by just setting it
+            # back to the Python default.
+            if not callable(signal.getsignal(signal.SIGINT)):
+                signal.signal(signal.SIGINT, signal.default_int_handler)
+
+            evtloop = wx.EventLoop()
+            ea = wx.EventLoopActivator(evtloop)
+            t = clock()
+            while not stdin_ready():
+                while evtloop.Pending():
+                    t = clock()
+                    evtloop.Dispatch()
+                app.ProcessIdle()
+                # We need to sleep at this point to keep the idle CPU load
+                # low.  However, if sleep to long, GUI response is poor.  As 
+                # a compromise, we watch how often GUI events are being processed
+                # and switch between a short and long sleep time.  Here are some
+                # stats useful in helping to tune this.
+                # time    CPU load
+                # 0.001   13%
+                # 0.005   3%
+                # 0.01    1.5%
+                # 0.05    0.5%
+                used_time = clock() - t
+                if used_time > 5*60.0:
+                    # print 'Sleep for 5 s'  # dbg
+                    time.sleep(5.0)
+                elif used_time > 10.0:
+                    # print 'Sleep for 1 s'  # dbg
+                    time.sleep(1.0)
+                elif used_time > 0.1:
+                    # Few GUI events coming in, so we can sleep longer
+                    # print 'Sleep for 0.05 s'  # dbg
+                    time.sleep(0.05)
+                else:
+                    # Many GUI events coming in, so sleep only very little
+                    time.sleep(0.001)
+            del ea
+    except KeyboardInterrupt:
+        pass
     return 0
 
 # This is our default implementation

docs/examples/kernel/mcdriver.py

@@ -1,25 +1,30 b''
 #!/usr/bin/env python
-# encoding: utf-8
 """Run a Monte-Carlo options pricer in parallel."""
 
 from IPython.kernel import client
 import numpy as np
 from mcpricer import price_options
 
-
-tc = client.TaskClient(profile='default')
+# The MultiEngineClient is used to setup the calculation and works with all
+# engine.
 mec = client.MultiEngineClient(profile='default')
 
+# The TaskClient is an interface to the engines that provides dynamic load 
+# balancing at the expense of not knowing which engine will execute the code.
+tc = client.TaskClient(profile='default')
 
-# Initialize the common code on the engines
+# Initialize the common code on the engines. This Python module has the
+# price_options function that prices the options.
 mec.run('mcpricer.py')
 
-# Define the function that will do the calculation
+# Define the function that will make up our tasks. We basically want to
+# call the price_options function with all but two arguments (K, sigma)
+# fixed.
 def my_prices(K, sigma):
     S = 100.0
     r = 0.05
     days = 260
-    paths = 10000
+    paths = 100000
     return price_options(S, K, sigma, r, days, paths)
 
 # Create arrays of strike prices and volatilities
@@ -28,7 +33,9 b' nsigma = 5'
 K_vals = np.linspace(90.0, 100.0, nK)
 sigma_vals = np.linspace(0.0, 0.2, nsigma)
 
-# Submit tasks
+# Submit tasks to the TaskClient for each (K, sigma) pair as a MapTask.
+# The MapTask simply applies a function (my_prices) to the arguments:
+# my_prices(K, sigma) and returns the result.
 taskids = []
 for K in K_vals:
     for sigma in sigma_vals:
@@ -37,24 +44,24 b' for K in K_vals:'
 
 print "Submitted tasks: ", taskids
 
-# Block until tasks are completed
+# Block until all tasks are completed.
 tc.barrier(taskids)
 
-# Get the results
+# Get the results using TaskClient.get_task_result.
 results = [tc.get_task_result(tid) for tid in taskids]
 
-# Assemble the result
+# Assemble the result into a structured NumPy array.
 prices = np.empty(nK*nsigma,
-    dtype=[('vcall',float),('vput',float),('acall',float),('aput',float)]
+    dtype=[('ecall',float),('eput',float),('acall',float),('aput',float)]
 )
 for i, price_tuple in enumerate(results):
     prices[i] = price_tuple
 prices.shape = (nK, nsigma)
+K_vals, sigma_vals = np.meshgrid(K_vals, sigma_vals)
 
-
-def plot_options(K_vals, sigma_vals, prices):
+def plot_options(sigma_vals, K_vals, prices):
     """
-    Make a contour plot of the option prices.
+    Make a contour plot of the option price in (sigma, K) space.
     """
     from matplotlib import pyplot as plt
     plt.contourf(sigma_vals, K_vals, prices)

docs/examples/kernel/mcpricer.py

@@ -4,7 +4,26 b' from math import *'
 
 def price_options(S=100.0, K=100.0, sigma=0.25, r=0.05, days=260, paths=10000):
     """
-    Price vanilla and asian options using a Monte Carlo method.
+    Price European and Asian options using a Monte Carlo method.
+
+    Parameters
+    ----------
+    S : float
+        The initial price of the stock.
+    K : float
+        The strike price of the option.
+    sigma : float
+        The volatility of the stock.
+    r : float
+        The risk free interest rate.
+    days : int
+        The number of days until the option expires.
+    paths : int
+        The number of Monte Carlo paths used to price the option.
+
+    Returns
+    -------
+    A tuple of (E. call, E. put, A. call, A. put) option prices.
     """
     h = 1.0/days
     const1 = exp((r-0.5*sigma**2)*h)
@@ -18,16 +37,9 b' def price_options(S=100.0, K=100.0, sigma=0.25, r=0.05, days=260, paths=10000):'
     stock_price_avg = stock_price_sum/days
     zeros = np.zeros(paths, dtype='float64')
     r_factor = exp(-r*h*days)
-    vanilla_put = r_factor*np.mean(np.maximum(zeros, K-stock_price))
+    euro_put = r_factor*np.mean(np.maximum(zeros, K-stock_price))
     asian_put = r_factor*np.mean(np.maximum(zeros, K-stock_price_avg))
-    vanilla_call = r_factor*np.mean(np.maximum(zeros, stock_price-K))
+    euro_call = r_factor*np.mean(np.maximum(zeros, stock_price-K))
     asian_call = r_factor*np.mean(np.maximum(zeros, stock_price_avg-K))
-    return (vanilla_call, vanilla_put, asian_call, asian_put)
-
+    return (euro_call, euro_put, asian_call, asian_put)
 
-if __name__ == '__main__':
-    (vc, vp, ac, ap) = price_options()
-    print "Vanilla Put Price = ", vp
-    print "Asian Put Price = ", ap
-    print "Vanilla Call Price = ", vc
-    print "Asian Call Price = ", ac

docs/examples/kernel/parallelpi.py

@@ -25,15 +25,6 b" filestring = 'pi200m-ascii-%(i)02dof20.txt'"
 files = [filestring % {'i':i} for i in range(1,16)]
 
 
-# A function for reducing the frequencies calculated
-# by different engines.
-def reduce_freqs(freqlist):
-    allfreqs = np.zeros_like(freqlist[0])
-    for f in freqlist:
-        allfreqs += f
-    return allfreqs
-
-
 # Connect to the IPython cluster
 mec = client.MultiEngineClient(profile='mycluster')
 mec.run('pidigits.py')
@@ -42,9 +33,7 b" mec.run('pidigits.py')"
 # Run 10m digits on 1 engine
 mapper = mec.mapper(targets=0)
 t1 = clock()
-
 freqs10m = mapper.map(compute_two_digit_freqs, files[:1])[0]
-
 t2 = clock()
 digits_per_second1 = 10.0e6/(t2-t1)
 print "Digits per second (1 core, 10m digits):   ", digits_per_second1
@@ -52,10 +41,8 b' print "Digits per second (1 core, 10m digits): ", digits_per_second1'
 
 # Run 150m digits on 15 engines (8 cores)
 t1 = clock()
-
 freqs_all = mec.map(compute_two_digit_freqs, files[:len(mec)])
 freqs150m = reduce_freqs(freqs_all)
-
 t2 = clock()
 digits_per_second8 = 150.0e6/(t2-t1)
 print "Digits per second (8 cores, 150m digits): ", digits_per_second8

docs/examples/kernel/pidigits.py

@@ -24,16 +24,34 b' from matplotlib import pyplot as plt'
 # Top-level functions
 
 def compute_one_digit_freqs(filename):
+    """
+    Read digits of pi from a file and compute the 1 digit frequencies.
+    """
     d = txt_file_to_digits(filename)
     freqs = one_digit_freqs(d)
     return freqs
 
 def compute_two_digit_freqs(filename):
+    """
+    Read digits of pi from a file and compute the 2 digit frequencies.
+    """
     d = txt_file_to_digits(filename)
     freqs = two_digit_freqs(d)
     return freqs
 
+def reduce_freqs(freqlist):
+    """
+    Add up a list of freq counts to get the total counts.
+    """
+    allfreqs = np.zeros_like(freqlist[0])
+    for f in freqlist:
+        allfreqs += f
+    return allfreqs
+
 def compute_n_digit_freqs(filename, n):
+    """
+    Read digits of pi from a file and compute the n digit frequencies.
+    """
     d = txt_file_to_digits(filename)
     freqs = n_digit_freqs(d, n)
     return freqs

docs/source/parallel/parallel_demos.txt

@@ -2,125 +2,269 b''
 Parallel examples
 =================
 
-In this section we describe a few more involved examples of using an IPython
-cluster to perform a parallel computation.
+In this section we describe two more involved examples of using an IPython
+cluster to perform a parallel computation. In these examples, we will be using
+IPython's "pylab" mode, which enables interactive plotting using the
+Matplotlib package. IPython can be started in this mode by typing::
+
+    ipython -p pylab
+
+at the system command line. If this prints an error message, you will 
+need to install the default profiles from within IPython by doing,
+
+.. sourcecode:: ipython
+
+    In [1]: %install_profiles
+
+and then restarting IPython.
 
 150 million digits of pi
 ========================
 
 In this example we would like to study the distribution of digits in the
-number pi. More specifically, we are going to study how often each 2
-digits sequence occurs in the first 150 million digits of pi. If the digits
-0-9 occur with equal probability, we expect that each two digits sequence
-(00, 01, ..., 99) will occur 1% of the time.
-
-This examples uses precomputed digits of pi from the website of Professor
-Yasumasa Kanada at the University of Tokoyo (http://www.super-computing.org).
-These digits come in a set of ``.txt`` files
-(ftp://pi.super-computing.org/.2/pi200m/) that each have 10 million digits of
-pi. In the parallel computation, we will use the :meth:`MultiEngineClient.map`
-method to have each engine compute the desired statistics on a subset of these
-files. Before I started the parallel computation, I copied the data files
-to the compute nodes so the engine have fast access to them.
-
-Here are the Python functions for counting the frequencies of each two digit
-sequence in serial::
-
-    def compute_two_digit_freqs(filename):
-        """
-        Compute the two digit frequencies from a single file.
-        """
-        d = txt_file_to_digits(filename)
-        freqs = two_digit_freqs(d)
-        return freqs
-    
-    def txt_file_to_digits(filename, the_type=str):
-        """
-        Yield the digits of pi read from a .txt file.
-        """
-        with open(filename, 'r') as f:
-            for line in f.readlines():
-                for c in line:
-                    if c != '\n' and c!= ' ':
-                        yield the_type(c)
-
-    def two_digit_freqs(digits, normalize=False):
-        """
-        Consume digits of pi and compute 2 digits freq. counts.
-        """
-        freqs = np.zeros(100, dtype='i4')
-        last = digits.next()
-        this = digits.next()
-        for d in digits:
-            index = int(last + this)
-            freqs[index] += 1
-            last = this
-            this = d
-        if normalize:
-            freqs = freqs/freqs.sum()
-        return freqs
-
-These functions are defined in the file :file:`pidigits.py`. To perform the
-calculation in parallel, we use an additional file: :file:`parallelpi.py`::
-
-    from IPython.kernel import client
-    from matplotlib import pyplot as plt
-    import numpy as np
-    from pidigits import *
-    from timeit import default_timer as clock
-
-    # Files with digits of pi (10m digits each)
-    filestring = 'pi200m-ascii-%(i)02dof20.txt'
-    files = [filestring % {'i':i} for i in range(1,16)]
-
-    # A function for reducing the frequencies calculated
-    # by different engines.
-    def reduce_freqs(freqlist):
-        allfreqs = np.zeros_like(freqlist[0])
-        for f in freqlist:
-            allfreqs += f
-        return allfreqs
-
-    # Connect to the IPython cluster
-    mec = client.MultiEngineClient(profile='mycluster')
-    mec.run('pidigits.py')
-
-    # Run 10m digits on 1 engine
-    mapper = mec.mapper(targets=0)
-    t1 = clock()
-    freqs10m = mapper.map(compute_two_digit_freqs, files[:1])[0]
-    t2 = clock()
-    digits_per_second1 = 10.0e6/(t2-t1)
-    print "Digits per second (1 core, 10m digits):   ", digits_per_second1
-
-    # Run 150m digits on 15 engines (8 cores)
-    t1 = clock()
-    freqs_all = mec.map(compute_two_digit_freqs, files[:len(mec)])
-    freqs150m = reduce_freqs(freqs_all)
-    t2 = clock()
-    digits_per_second8 = 150.0e6/(t2-t1)
-    print "Digits per second (8 cores, 150m digits): ", digits_per_second8
-
-    print "Speedup: ", digits_per_second8/digits_per_second1
-
-    plot_two_digit_freqs(freqs150m)
-    plt.title("2 digit sequences in 150m digits of pi")
-
-To run this code on an IPython cluster:
-
-1. Start an IPython cluster with 15 engines: ``ipcluster start -p mycluster -n 15``
-2. Open IPython's interactive shell using the pylab profile
-   ``ipython -p pylab`` and type ``run parallelpi.py``.
-
-At this point, the parallel calculation will begin. On a small an 8 core
-cluster, we observe a speedup of 7.7x. The resulting plot of the two digit
-sequences is shown in the following screenshot.
-
-.. image:: parallel_pi.*
-
-
-Parallel option pricing
-=======================
-
-The example will be added at a later point.
+number pi (in base 10). While it is not known if pi is a normal number (a
+number is normal in base 10 if 0-9 occur with equal likelihood) numerical
+investigations suggest that it is. We will begin with a serial calculation on
+10,000 digits of pi and then perform a parallel calculation involving 150
+million digits.
+
+In both the serial and parallel calculation we will be using functions defined
+in the :file:`pidigits.py` file, which is available in the
+:file:`docs/examples/kernel` directory of the IPython source distribution.
+These functions provide basic facilities for working with the digits of pi and
+can be loaded into IPython by putting :file:`pidigits.py` in your current
+working directory and then doing:
+
+.. sourcecode:: ipython
+
+    In [1]: run pidigits.py
+
+Serial calculation
+------------------
+
+For the serial calculation, we will use SymPy (http://www.sympy.org) to
+calculate 10,000 digits of pi and then look at the frequencies of the digits
+0-9. Out of 10,000 digits, we expect each digit to occur 1,000 times. While
+SymPy is capable of calculating many more digits of pi, our purpose here is to
+set the stage for the much larger parallel calculation.
+
+In this example, we use two functions from :file:`pidigits.py`: 
+:func:`one_digit_freqs` (which calculates how many times each digit occurs)
+and :func:`plot_one_digit_freqs` (which uses Matplotlib to plot the result).
+Here is an interactive IPython session that uses these functions with
+SymPy:
+
+.. sourcecode:: ipython
+
+    In [7]: import sympy
+
+    In [8]: pi = sympy.pi.evalf(40)
+
+    In [9]: pi
+    Out[9]: 3.141592653589793238462643383279502884197
+
+    In [10]: pi = sympy.pi.evalf(10000)
+
+    In [11]: digits = (d for d in str(pi)[2:])  # create a sequence of digits
+
+    In [12]: run pidigits.py  # load one_digit_freqs/plot_one_digit_freqs
+
+    In [13]: freqs = one_digit_freqs(digits)
+
+    In [14]: plot_one_digit_freqs(freqs)
+    Out[14]: [<matplotlib.lines.Line2D object at 0x18a55290>]
+
+The resulting plot of the single digit counts shows that each digit occurs
+approximately 1,000 times, but that with only 10,000 digits the
+statistical fluctuations are still rather large:
+
+.. image:: single_digits.*
+
+It is clear that to reduce the relative fluctuations in the counts, we need
+to look at many more digits of pi. That brings us to the parallel calculation.
+
+Parallel calculation
+--------------------
+
+Calculating many digits of pi is a challenging computational problem in itself. 
+Because we want to focus on the distribution of digits in this example, we
+will use pre-computed digit of pi from the website of Professor Yasumasa
+Kanada at the University of Tokoyo (http://www.super-computing.org). These
+digits come in a set of text files (ftp://pi.super-computing.org/.2/pi200m/)
+that each have 10 million digits of pi.
+
+For the parallel calculation, we have copied these files to the local hard
+drives of the compute nodes. A total of 15 of these files will be used, for a
+total of 150 million digits of pi. To make things a little more interesting we
+will calculate the frequencies of all 2 digits sequences (00-99) and then plot
+the result using a 2D matrix in Matplotlib.
+
+The overall idea of the calculation is simple: each IPython engine will
+compute the two digit counts for the digits in a single file. Then in a final
+step the counts from each engine will be added up. To perform this
+calculation, we will need two top-level functions from :file:`pidigits.py`:
+
+.. literalinclude:: ../../examples/kernel/pidigits.py
+   :language: python
+   :lines: 34-49
+
+We will also use the :func:`plot_two_digit_freqs` function to plot the
+results. The code to run this calculation in parallel is contained in
+:file:`docs/examples/kernel/parallelpi.py`. This code can be run in parallel
+using IPython by following these steps:
+
+1. Copy the text files with the digits of pi 
+   (ftp://pi.super-computing.org/.2/pi200m/) to the working directory of the
+   engines on the compute nodes.
+2. Use :command:`ipcluster` to start 15 engines. We used an 8 core cluster
+   with hyperthreading enabled which makes the 8 cores looks like 16 (1
+   controller + 15 engines) in the OS. However, the maximum speedup we can
+   observe is still only 8x.
+3. With the file :file:`parallelpi.py` in your current working directory, open
+   up IPython in pylab mode and type ``run parallelpi.py``.
+
+When run on our 8 core cluster, we observe a speedup of 7.7x. This is slightly
+less than linear scaling (8x) because the controller is also running on one of
+the cores.
+
+To emphasize the interactive nature of IPython, we now show how the
+calculation can also be run by simply typing the commands from
+:file:`parallelpi.py` interactively into IPython:
+
+.. sourcecode:: ipython
+
+    In [1]: from IPython.kernel import client
+    2009-11-19 11:32:38-0800 [-] Log opened.
+
+    # The MultiEngineClient allows us to use the engines interactively
+    In [2]: mec = client.MultiEngineClient(profile='mycluster')
+    2009-11-19 11:32:44-0800 [-] Connecting [0]
+    2009-11-19 11:32:44-0800 [Negotiation,client] Connected: ./ipcontroller-mec.furl
+
+    In [3]: mec.get_ids()
+    Out[3]: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14]
+
+    In [4]: run pidigits.py
+
+    In [5]: filestring = 'pi200m-ascii-%(i)02dof20.txt'
+
+    In [6]: files = [filestring % {'i':i} for i in range(1,16)]
+
+    In [7]: files
+    Out[7]: 
+    ['pi200m-ascii-01of20.txt',
+     'pi200m-ascii-02of20.txt',
+     'pi200m-ascii-03of20.txt',
+     'pi200m-ascii-04of20.txt',
+     'pi200m-ascii-05of20.txt',
+     'pi200m-ascii-06of20.txt',
+     'pi200m-ascii-07of20.txt',
+     'pi200m-ascii-08of20.txt',
+     'pi200m-ascii-09of20.txt',
+     'pi200m-ascii-10of20.txt',
+     'pi200m-ascii-11of20.txt',
+     'pi200m-ascii-12of20.txt',
+     'pi200m-ascii-13of20.txt',
+     'pi200m-ascii-14of20.txt',
+     'pi200m-ascii-15of20.txt']
+
+    # This is the parallel calculation using the MultiEngineClient.map method 
+    # which applies compute_two_digit_freqs to each file in files in parallel.
+    In [8]: freqs_all = mec.map(compute_two_digit_freqs, files)
+
+    # Add up the frequencies from each engine.
+    In [8]: freqs = reduce_freqs(freqs_all)
+
+    In [9]: plot_two_digit_freqs(freqs)
+    Out[9]: <matplotlib.image.AxesImage object at 0x18beb110>
+
+    In [10]: plt.title('2 digit counts of 150m digits of pi')
+    Out[10]: <matplotlib.text.Text object at 0x18d1f9b0>
+
+The resulting plot generated by Matplotlib is shown below. The colors indicate
+which two digit sequences are more (red) or less (blue) likely to occur in the
+first 150 million digits of pi. We clearly see that the sequence "41" is 
+most likely and that "06" and "07" are least likely. Further analysis would
+show that the relative size of the statistical fluctuations have decreased
+compared to the 10,000 digit calculation.
+
+.. image:: two_digit_counts.*
+
+To conclude this example, we summarize the key features of IPython's parallel
+architecture that this example demonstrates:
+
+* Serial code can be parallelized often with only a few extra lines of code.
+  In this case we have used :meth:`MultiEngineClient.map`; the
+  :class:`MultiEngineClient` class has a number of other methods that provide
+  more fine grained control of the IPython cluster.
+* The resulting parallel code can be run without ever leaving the IPython's
+  interactive shell.
+* Any data computed in parallel can be explored interactively through 
+  visualization or further numerical calculations.
+
+
+Parallel options pricing
+========================
+
+An option is a financial contract that gives the buyer of the contract the
+right to buy (a "call") or sell (a "put") a secondary asset (a stock for
+example) at a particular date in the future (the expiration date) for a
+pre-agreed upon price (the strike price). For this right, the buyer pays the
+seller a premium (the option price). There are a wide variety of flavors of
+options (American, European, Asian, etc.) that are useful for different
+purposes: hedging against risk, speculation, etc.
+
+Much of modern finance is driven by the need to price these contracts
+accurately based on what is known about the properties (such as volatility) of
+the underlying asset. One method of pricing options is to use a Monte Carlo
+simulation of the underlying assets. In this example we use this approach to
+price both European and Asian (path dependent) options for various strike
+prices and volatilities.
+
+The code for this example can be found in the :file:`docs/examples/kernel`
+directory of the IPython source.
+
+The function :func:`price_options`, calculates the option prices for a single
+option (:file:`mcpricer.py`):
+
+.. literalinclude:: ../../examples/kernel/mcpricer.py
+   :language: python
+
+To run this code in parallel, we will use IPython's :class:`TaskClient`, which
+distributes work to the engines using dynamic load balancing. This client
+can be used along side the :class:`MultiEngineClient` shown in the previous
+example.
+
+Here is the code that calls :func:`price_options` for a number of different
+volatilities and strike prices in parallel:
+
+.. literalinclude:: ../../examples/kernel/mcdriver.py
+   :language: python
+
+To run this code in parallel, start an IPython cluster using
+:command:`ipcluster`, open IPython in the pylab mode with the file
+:file:`mcdriver.py` in your current working directory and then type:
+
+.. sourcecode:: ipython
+
+    In [7]: run mcdriver.py
+    Submitted tasks:  [0, 1, 2, ...]
+
+Once all the tasks have finished, the results can be plotted using the
+:func:`plot_options` function. Here we make contour plots of the Asian
+call and Asian put as function of the volatility and strike price:
+
+.. sourcecode:: ipython
+
+    In [8]: plot_options(sigma_vals, K_vals, prices['acall'])
+
+    In [9]: plt.figure()
+    Out[9]: <matplotlib.figure.Figure object at 0x18c178d0>
+
+    In [10]: plot_options(sigma_vals, K_vals, prices['aput'])
+
+The plots generated by Matplotlib will look like this:
+
+.. image:: asian_call.*
 
+.. image:: asian_put.*

docs/source/parallel/parallel_winhpc.txt

@@ -5,18 +5,38 b' Getting started'
 Introduction
 ============
 
-IPython is an open source project focused on interactive and exploratory
-computing in the Python programming language. It consists of two
-main componenents:
+The Python programming language is increasingly popular language for numerical
+computing. This is due to a unique combination of factors. First, Python is a
+high-level and *interactive* language that is well matched for interactive
+numerical work. Second, it is easy (often times trivial) to integrate legacy
+C/C++/Fortran code into Python. Third, a large number of high-quality open
+source projects provide all the needed building blocks for numerical
+computing: numerical arrays (NumPy), algorithms (SciPy), 2D/3D Visualization
+(Matplotlib, Mayavi, Chaco), Symbolic Mathematics (Sage, Sympy) and others.
+
+The IPython project is a core part of this open-source toolchain and is
+focused on creating a comprehensive environment for interactive and
+exploratory computing in the Python programming language. It enables all of
+the above tools to be used interactively and consists of two main components:
 
 * An enhanced interactive Python shell with support for interactive plotting
   and visualization.
 * An architecture for interactive parallel computing.
 
 With these components, it is possible to perform all aspects of a parallel
-computation interactively. This document describes how to get started with
-IPython on Window HPC Server 2008. A more complete desription of IPython's
-parallel computing capabilities can be found in IPython's online documentation
+computation interactively. This type of workflow is particularly relevant in
+scientific and numerical computing where algorithms, code and data are
+continually evolving as the user/developer explores a problem. The broad
+treads in computing (commodity clusters, multicore, cloud computing, etc.)
+make these capabilities of IPython particularly relevant.
+
+While IPython is a cross platform tool, it has particularly strong support for
+Windows based compute clusters running Windows HPC Server 2008. This document
+describes how to get started with IPython on Windows HPC Server 2008. The
+content and emphasis here is practical: installing IPython, configuring
+IPython to use the Windows job scheduler and running example parallel programs
+interactively. A more complete description of IPython's parallel computing
+capabilities can be found in IPython's online documentation
 (http://ipython.scipy.org/moin/Documentation).
 
 Setting up your Windows cluster
@@ -38,7 +58,7 b' such a cluster:'
 
 More details about installing and configuring Windows HPC Server 2008 can be
 found on the Windows HPC Home Page (http://www.microsoft.com/hpc). Regardless
-of what steps you go through to set up your cluster, the remainder of this
+of what steps you follow to set up your cluster, the remainder of this
 document will assume that:
 
 * There are domain users that can log on to the AD domain and submit jobs
@@ -63,8 +83,8 b' IPython on Windows:'
 * pyOpenSSL (https://launchpad.net/pyopenssl)
 * IPython (http://ipython.scipy.org)
 
-In addition, the following dependencies are needed to run the demos
-described in this document.
+In addition, the following dependencies are needed to run the demos described
+in this document.
 
 * NumPy and SciPy (http://www.scipy.org)
 * wxPython (http://www.wxpython.org)
@@ -94,7 +114,7 b' need to follow:'
 Further details about installing IPython or its dependencies can be found in
 the online IPython documentation (http://ipython.scipy.org/moin/Documentation)
 Once you are finished with the installation, you can try IPython out by
-opening a Windows Command Prompt and typing :command:`ipython`. This will
+opening a Windows Command Prompt and typing ``ipython``. This will
 start IPython's interactive shell and you should see something like the
 following screenshot:
 
@@ -114,14 +134,15 b' IPython controller'
     command.
 
 IPython engine
-    IPython engines run your Python code in parallel on the compute nodes. 
+    IPython engines run a user's Python code in parallel on the compute nodes. 
     Engines are starting using the :command:`ipengine` command.
 
 Once these processes are started, a user can run Python code interactively and
-in parallel on the engines from within the IPython shell. This includes the
-ability to interact with, plot and visualize data from the engines.
+in parallel on the engines from within the IPython shell using an appropriate
+client. This includes the ability to interact with, plot and visualize data
+from the engines.
 
-IPython has a command line program called :command:`ipcluster` that handles
+IPython has a command line program called :command:`ipcluster` that automates
 all aspects of starting the controller and engines on the compute nodes.
 :command:`ipcluster` has full support for the Windows HPC job scheduler,
 meaning that :command:`ipcluster` can use this job scheduler to start the
@@ -142,38 +163,44 b' Command Prompt and type the following command::'
     ipcluster start -n 2
 
 You should see a number of messages printed to the screen, ending with 
-"IPython cluster: started". A screenshot of this follows.
-
+"IPython cluster: started". The result should look something like the following
+screenshot:
 
 .. image:: ipcluster_start.*
 
 At this point, the controller and two engines are running on your local host.
-This configuration is useful for testing and for situations where you
-have multiple cores on your local computer. 
+This configuration is useful for testing and for situations where you want to
+take advantage of multiple cores on your local computer. 
 
 Now that we have confirmed that :command:`ipcluster` is working properly, we
-describe how to configure and run an IPython cluster on an actual cluster
-running Windows HPC Server 2008. Here is an outline of the needed steps:
+describe how to configure and run an IPython cluster on an actual compute
+cluster running Windows HPC Server 2008. Here is an outline of the needed
+steps:
 
-1. Create a cluster profile:  ``ipcluster create -p mycluster``
+1. Create a cluster profile using: ``ipcluster create -p mycluster``
 
-2. Edit confguration files in :file:`.ipython\\cluster_mycluster`.
+2. Edit configuration files in the directory :file:`.ipython\\cluster_mycluster`
 
-3. Start the cluster:  ``ipcluser start -p mycluster -n 32``
+3. Start the cluster using: ``ipcluser start -p mycluster -n 32``
 
 Creating a cluster profile
 --------------------------
 
-In most cases, you will have to create and configure a cluster profile to use
-IPython on a cluster. A cluster profile is a specially named directory
-(typically located in the :file:`.ipython` subdirectory of your home
-directory) that contains the configuration files for a particular IPython
-cluster, as well as log files and security keys. The naming convention
-for cluster directories is: "cluster_<profile name>". Thus, the cluster
-directory for a profile named "foo" would be :file:`.ipython\\cluster_foo`.
+In most cases, you will have to create a cluster profile to use IPython on a
+cluster. A cluster profile is a name (like "mycluster") that is associated
+with a particular cluster configuration. The profile name is used by
+:command:`ipcluster` when working with the cluster.
+
+Associated with each cluster profile is a cluster directory. This cluster
+directory is a specially named directory (typically located in the
+:file:`.ipython` subdirectory of your home directory) that contains the
+configuration files for a particular cluster profile, as well as log files and
+security keys. The naming convention for cluster directories is:
+:file:`cluster_<profile name>`. Thus, the cluster directory for a profile named
+"foo" would be :file:`.ipython\\cluster_foo`.
 
-To create a new cluster profile (named "mycluster"), type the following
-command at the Windows Command Prompt::
+To create a new cluster profile (named "mycluster") and the associated cluster
+directory, type the following command at the Windows Command Prompt::
 
     ipcluster create -p mycluster
 
@@ -181,10 +208,8 b' The output of this command is shown in the screenshot below. Notice how'
 :command:`ipcluster` prints out the location of the newly created cluster
 directory.
 
-
 .. image:: ipcluster_create.*
 
-
 Configuring a cluster profile
 -----------------------------
 
@@ -221,18 +246,19 b' in most cases these will be sufficient to get you started.'
     If any of your configuration attributes involve specifying the location
     of shared directories or files, you must make sure that you use UNC paths
     like :file:`\\\\host\\share`. It is also important that you specify
-    these paths using raw Python strings: ``r'\\host\share'``.
+    these paths using raw Python strings: ``r'\\host\share'`` to make sure
+    that the backslashes are properly escaped.
 
 Starting the cluster profile
 ----------------------------
 
 Once a cluster profile has been configured, starting an IPython cluster using
-the profile is simple:
+the profile is simple::
 
     ipcluster start -p mycluster -n 32
 
-The ``-n 32`` option tells :command:`ipcluster` how many engines to start.
-Stopping the cluster is as simple as typing Control-C. 
+The ``-n`` option tells :command:`ipcluster` how many engines to start (in
+this case 32). Stopping the cluster is as simple as typing Control-C.
 
 Using the HPC Job Manager
 -------------------------
@@ -247,32 +273,56 b' Once these files have been created, they can be imported into the HPC Job'
 Manager application. Then, the controller and engines for that profile can be
 started using the HPC Job Manager directly, without using :command:`ipcluster`.
 However, anytime the cluster profile is re-configured, ``ipcluster start``
-has to be run again to regenerate the XML job description files. The
+must be run again to regenerate the XML job description files. The
 following screenshot shows what the HPC Job Manager interface looks like
 with a running IPython cluster.
 
-
 .. image:: hpc_job_manager.*
 
 Performing a simple interactive parallel computation
 ====================================================
 
 Once you have started your IPython cluster, you can start to use it. To do
-this, start up IPython's interactive shell by typing::
+this, open up a new Windows Command Prompt and start up IPython's interactive
+shell by typing::
 
     ipython
 
-at the Windows Command Prompt. Then you can create a :class:`MultiEngineClient`
-instance for your profile and use the resulting instance to
-have the cluster do a simple interactive parallel computation. In the 
-screenshot that follows, we take a simple Python function::
-
-    def f(x): return x**10
-
-and apply it to each element of an array of integers in
-parallel using the :meth:`MultiEngineClient.map` method::
-
-    mec.map(f, range(15))
+Then you can create a :class:`MultiEngineClient` instance for your profile and
+use the resulting instance to do a simple interactive parallel computation. In
+the code and screenshot that follows, we take a simple Python function and
+apply it to each element of an array of integers in parallel using the
+:meth:`MultiEngineClient.map` method:
+
+.. sourcecode:: ipython
+
+    In [1]: from IPython.kernel.client import *
+
+    In [2]: mec = MultiEngineClient(profile='mycluster')
+
+    In [3]: mec.get_ids()
+    Out[3]: [0, 1, 2, 3, 4, 5, 67, 8, 9, 10, 11, 12, 13, 14]
+
+    In [4]: def f(x):
+       ...:     return x**10
+
+    In [5]: mec.map(f, range(15))  # f is applied in parallel
+    Out[5]:
+    [0,
+     1,
+     1024,
+     59049,
+     1048576,
+     9765625,
+     60466176,
+     282475249,
+     1073741824,
+     3486784401L,
+     10000000000L,
+     25937424601L,
+     61917364224L,
+     137858491849L,
+     289254654976L]
 
 The :meth:`map` method has the same signature as Python's builtin :func:`map`
 function, but runs the calculation in parallel. More involved examples of using