Cleanup of docs.

* Updated all of the sphinx extensions from sampledoc.
* Using mpl versions of extensions they ship.
* Using nice theme of sampledoc.
* Reorganizing our docs.

docs/source/_static/default.css

@@ -0,0 +1,507 b''
+/**
+ * Alternate Sphinx design
+ * Originally created by Armin Ronacher for Werkzeug, adapted by Georg Brandl.
+ */
+
+body {
+    font-family: 'Lucida Grande', 'Lucida Sans Unicode', 'Geneva', 'Verdana', sans-serif;
+    font-size: 14px;
+    letter-spacing: -0.01em;
+    line-height: 150%;
+    text-align: center;
+    /*background-color: #AFC1C4; */
+    background-color: #BFD1D4;
+    color: black;
+    padding: 0;
+    border: 1px solid #aaa;
+
+    margin: 0px 80px 0px 80px;
+    min-width: 740px;
+}
+
+a {
+    color: #CA7900;
+    text-decoration: none;
+}
+
+a:hover {
+    color: #2491CF;
+}
+
+pre {
+    font-family: 'Consolas', 'Deja Vu Sans Mono', 'Bitstream Vera Sans Mono', monospace;
+    font-size: 0.95em;
+    letter-spacing: 0.015em;
+    padding: 0.5em;
+    border: 1px solid #ccc;
+    background-color: #f8f8f8;
+}
+
+td.linenos pre {
+    padding: 0.5em 0;
+    border: 0;
+    background-color: transparent;
+    color: #aaa;
+}
+
+table.highlighttable {
+    margin-left: 0.5em;
+}
+
+table.highlighttable td {
+    padding: 0 0.5em 0 0.5em;
+}
+
+cite, code, tt {
+    font-family: 'Consolas', 'Deja Vu Sans Mono', 'Bitstream Vera Sans Mono', monospace;
+    font-size: 0.95em;
+    letter-spacing: 0.01em;
+}
+
+hr {
+    border: 1px solid #abc;
+    margin: 2em;
+}
+
+tt {
+    background-color: #f2f2f2;
+    border-bottom: 1px solid #ddd;
+    color: #333;
+}
+
+tt.descname {
+    background-color: transparent;
+    font-weight: bold;
+    font-size: 1.2em;
+    border: 0;
+}
+
+tt.descclassname {
+    background-color: transparent;
+    border: 0;
+}
+
+tt.xref {
+    background-color: transparent;
+    font-weight: bold;
+    border: 0;
+}
+
+a tt {
+    background-color: transparent;
+    font-weight: bold;
+    border: 0;
+    color: #CA7900;
+}
+
+a tt:hover {
+    color: #2491CF;
+}
+
+dl {
+    margin-bottom: 15px;
+}
+
+dd p {
+    margin-top: 0px;
+}
+
+dd ul, dd table {
+    margin-bottom: 10px;
+}
+
+dd {
+    margin-top: 3px;
+    margin-bottom: 10px;
+    margin-left: 30px;
+}
+
+.refcount {
+    color: #060;
+}
+
+dt:target,
+.highlight {
+    background-color: #fbe54e;
+}
+
+dl.class, dl.function {
+    border-top: 2px solid #888;
+}
+
+dl.method, dl.attribute {
+    border-top: 1px solid #aaa;
+}
+
+dl.glossary dt {
+    font-weight: bold;
+    font-size: 1.1em;
+}
+
+pre {
+    line-height: 120%;
+}
+
+pre a {
+    color: inherit;
+    text-decoration: underline;
+}
+
+.first {
+    margin-top: 0 !important;
+}
+
+div.document {
+    background-color: white;
+    text-align: left;
+    background-image: url(contents.png);
+    background-repeat: repeat-x;
+}
+
+/*
+div.documentwrapper {
+    width: 100%;
+}
+*/
+
+div.clearer {
+    clear: both;
+}
+
+div.related h3 {
+    display: none;
+}
+
+div.related ul {
+    background-image: url(navigation.png);
+    height: 2em;
+    list-style: none;
+    border-top: 1px solid #ddd;
+    border-bottom: 1px solid #ddd;
+    margin: 0;
+    padding-left: 10px;
+}
+
+div.related ul li {
+    margin: 0;
+    padding: 0;
+    height: 2em;
+    float: left;
+}
+
+div.related ul li.right {
+    float: right;
+    margin-right: 5px;
+}
+
+div.related ul li a {
+    margin: 0;
+    padding: 0 5px 0 5px;
+    line-height: 1.75em;
+    color: #EE9816;
+}
+
+div.related ul li a:hover {
+    color: #3CA8E7;
+}
+
+div.body {
+    margin: 0;
+    padding: 0.5em 20px 20px 20px;
+}
+
+div.bodywrapper {
+    margin: 0 240px 0 0;
+    border-right: 1px solid #ccc;
+}
+
+div.body a {
+    text-decoration: underline;
+}
+
+div.sphinxsidebar {
+    margin: 0;
+    padding: 0.5em 15px 15px 0;
+    width: 210px;
+    float: right;
+    text-align: left;
+/*    margin-left: -100%; */
+}
+
+div.sphinxsidebar h4, div.sphinxsidebar h3 {
+    margin: 1em 0 0.5em 0;
+    font-size: 0.9em;
+    padding: 0.1em 0 0.1em 0.5em;
+    color: white;
+    border: 1px solid #86989B;
+    background-color: #AFC1C4;
+}
+
+div.sphinxsidebar ul {
+    padding-left: 1.5em;
+    margin-top: 7px;
+    list-style: none;
+    padding: 0;
+    line-height: 130%;
+}
+
+div.sphinxsidebar ul ul {
+    list-style: square;
+    margin-left: 20px;
+}
+
+p {
+    margin: 0.8em 0 0.5em 0;
+}
+
+p.rubric {
+    font-weight: bold;
+}
+
+h1 {
+    margin: 0;
+    padding: 0.7em 0 0.3em 0;
+    font-size: 1.5em;
+    color: #11557C;
+}
+
+h2 {
+    margin: 1.3em 0 0.2em 0;
+    font-size: 1.35em;
+    padding: 0;
+}
+
+h3 {
+    margin: 1em 0 -0.3em 0;
+    font-size: 1.2em;
+}
+
+h1 a, h2 a, h3 a, h4 a, h5 a, h6 a {
+    color: black!important;
+}
+
+h1 a.anchor, h2 a.anchor, h3 a.anchor, h4 a.anchor, h5 a.anchor, h6 a.anchor {
+    display: none;
+    margin: 0 0 0 0.3em;
+    padding: 0 0.2em 0 0.2em;
+    color: #aaa!important;
+}
+
+h1:hover a.anchor, h2:hover a.anchor, h3:hover a.anchor, h4:hover a.anchor,
+h5:hover a.anchor, h6:hover a.anchor {
+    display: inline;
+}
+
+h1 a.anchor:hover, h2 a.anchor:hover, h3 a.anchor:hover, h4 a.anchor:hover,
+h5 a.anchor:hover, h6 a.anchor:hover {
+    color: #777;
+    background-color: #eee;
+}
+
+table {
+    border-collapse: collapse;
+    margin: 0 -0.5em 0 -0.5em;
+}
+
+table td, table th {
+    padding: 0.2em 0.5em 0.2em 0.5em;
+}
+
+div.footer {
+    background-color: #E3EFF1;
+    color: #86989B;
+    padding: 3px 8px 3px 0;
+    clear: both;
+    font-size: 0.8em;
+    text-align: right;
+}
+
+div.footer a {
+    color: #86989B;
+    text-decoration: underline;
+}
+
+div.pagination {
+    margin-top: 2em;
+    padding-top: 0.5em;
+    border-top: 1px solid black;
+    text-align: center;
+}
+
+div.sphinxsidebar ul.toc {
+    margin: 1em 0 1em 0;
+    padding: 0 0 0 0.5em;
+    list-style: none;
+}
+
+div.sphinxsidebar ul.toc li {
+    margin: 0.5em 0 0.5em 0;
+    font-size: 0.9em;
+    line-height: 130%;
+}
+
+div.sphinxsidebar ul.toc li p {
+    margin: 0;
+    padding: 0;
+}
+
+div.sphinxsidebar ul.toc ul {
+    margin: 0.2em 0 0.2em 0;
+    padding: 0 0 0 1.8em;
+}
+
+div.sphinxsidebar ul.toc ul li {
+    padding: 0;
+}
+
+div.admonition, div.warning {
+    font-size: 0.9em;
+    margin: 1em 0 0 0;
+    border: 1px solid #86989B;
+    background-color: #f7f7f7;
+}
+
+div.admonition p, div.warning p {
+    margin: 0.5em 1em 0.5em 1em;
+    padding: 0;
+}
+
+div.admonition pre, div.warning pre {
+    margin: 0.4em 1em 0.4em 1em;
+}
+
+div.admonition p.admonition-title,
+div.warning p.admonition-title {
+    margin: 0;
+    padding: 0.1em 0 0.1em 0.5em;
+    color: white;
+    border-bottom: 1px solid #86989B;
+    font-weight: bold;
+    background-color: #AFC1C4;
+}
+
+div.warning {
+    border: 1px solid #940000;
+}
+
+div.warning p.admonition-title {
+    background-color: #CF0000;
+    border-bottom-color: #940000;
+}
+
+div.admonition ul, div.admonition ol,
+div.warning ul, div.warning ol {
+    margin: 0.1em 0.5em 0.5em 3em;
+    padding: 0;
+}
+
+div.versioninfo {
+    margin: 1em 0 0 0;
+    border: 1px solid #ccc;
+    background-color: #DDEAF0;
+    padding: 8px;
+    line-height: 1.3em;
+    font-size: 0.9em;
+}
+
+
+a.headerlink {
+    color: #c60f0f!important;
+    font-size: 1em;
+    margin-left: 6px;
+    padding: 0 4px 0 4px;
+    text-decoration: none!important;
+    visibility: hidden;
+}
+
+h1:hover > a.headerlink,
+h2:hover > a.headerlink,
+h3:hover > a.headerlink,
+h4:hover > a.headerlink,
+h5:hover > a.headerlink,
+h6:hover > a.headerlink,
+dt:hover > a.headerlink {
+    visibility: visible;
+}
+
+a.headerlink:hover {
+    background-color: #ccc;
+    color: white!important;
+}
+
+table.indextable td {
+    text-align: left;
+    vertical-align: top;
+}
+
+table.indextable dl, table.indextable dd {
+    margin-top: 0;
+    margin-bottom: 0;
+}
+
+table.indextable tr.pcap {
+    height: 10px;
+}
+
+table.indextable tr.cap {
+    margin-top: 10px;
+    background-color: #f2f2f2;
+}
+
+img.toggler {
+    margin-right: 3px;
+    margin-top: 3px;
+    cursor: pointer;
+}
+
+img.inheritance {
+    border: 0px
+}
+
+form.pfform {
+    margin: 10px 0 20px 0;
+}
+
+table.contentstable {
+    width: 90%;
+}
+
+table.contentstable p.biglink {
+    line-height: 150%;
+}
+
+a.biglink {
+    font-size: 1.3em;
+}
+
+span.linkdescr {
+    font-style: italic;
+    padding-top: 5px;
+    font-size: 90%;
+}
+
+ul.search {
+    margin: 10px 0 0 20px;
+    padding: 0;
+}
+
+ul.search li {
+    padding: 5px 0 5px 20px;
+    background-image: url(file.png);
+    background-repeat: no-repeat;
+    background-position: 0 7px;
+}
+
+ul.search li a {
+    font-weight: bold;
+}
+
+ul.search li div.context {
+    color: #888;
+    margin: 2px 0 0 30px;
+    text-align: left;
+}
+
+ul.keywordmatches li.goodmatch a {
+    font-weight: bold;
+}

docs/source/_templates/layout.html

@@ -0,0 +1,23 b''
+{% extends "!layout.html" %}
+
+
+{% block rootrellink %}
+        <li><a href="{{ pathto('index') }}">home</a>|&nbsp;</li>
+        <li><a href="{{ pathto('search') }}">search</a>|&nbsp;</li>
+       <li><a href="{{ pathto('whatsnew/index') }}">what's new </a> &raquo;</li>
+{% endblock %}
+
+
+{% block relbar1 %}
+
+<div style="background-color: white; text-align: left; padding: 10px 10px 15px 15px">
+<a href="{{ pathto('index') }}"><img src="{{
+pathto("_static/logo.png", 1) }}" border="0" alt="IPython Documentation"/></a>
+</div>
+{{ super() }}
+{% endblock %}
+
+{# put the sidebar before the body #}
+{% block sidebar1 %}{{ sidebar() }}{% endblock %}
+{% block sidebar2 %}{% endblock %}
+

docs/source/about/index.txt

@@ -0,0 +1,13 b''
+.. _about_index:
+
+=============
+About IPython
+=============
+
+.. toctree::
+   :maxdepth: 1
+
+   credits
+   history
+   license_and_copyright
+

docs/source/whatsnew/development.txt

@@ -0,0 +1,49 b''
+================================================
+Development version
+================================================
+
+Main `ipython` branch
+=====================
+
+New features
+------------
+
+* Added a new module :mod:`IPython.lib.inputhook` to manage the integration
+  with GUI event loops using `PyOS_InputHook`.  See the docstrings in this
+  module or the main IPython docs for details.
+* For users, GUI event loop integration is now handled through the new
+  :command:`%gui` magic command.  Type ``%gui?`` at an IPython prompt for
+  documentation.
+* For developers :mod:`IPython.lib.inputhook` provides a simple interface
+  for managing the event loops in their interactive GUI applications.
+  Examples can be found in our :file:`docs/examples/lib` directory.
+
+Bug fixes
+---------
+
+* Keyboard interrupts now work with GUI support enabled across all platforms
+  and all GUI toolkits reliably.
+
+Backwards incompatible changes
+------------------------------
+
+* Support for ``qt3`` has been dropped.  User's who need this should use
+  previous versions of IPython.
+* Removed :mod:`shellglobals` as it was obsolete. 
+* Removed all the threaded shells in :mod:`IPython.core.shell`. These are no
+  longer needed because of the new capabilities in
+  :mod:`IPython.lib.inputhook`.
+* The old threading command line flags (pylab/wthread/etc.) have been
+  deprecated.  Use :mod:`IPython.inputhook` or the new :command:`%gui` magic
+  command instead.
+* New top-level sub-packages have been created: :mod:`IPython.core`, 
+  :mod:`IPython.lib`, :mod:`IPython.utils`, :mod:`IPython.deathrow`,
+  :mod:`IPython.quarantine`.  All existing top-level modules have been
+  moved to appropriate sub-packages.  All internal import statements
+  have been updated and tests have been added.  The build system (setup.py
+  and friends) have been updated.
+* Compatability modules have been created for :mod:`IPython.Shell`, 
+  :mod:`IPython.ipapi` and :mod:`IPython.iplib` that display warnings
+  and then load the actual implementation from :mod:`IPython.core`.
+* :mod:`Extensions` has been moved to :mod:`extensions`.
+

docs/source/whatsnew/index.txt

@@ -0,0 +1,23 b''
+.. Developers should add in this file, during each release cycle, information
+.. about important changes they've made, in a summary format that's meant for
+.. end users.  For each release we normally have three sections: features,  bug
+.. fixes and api breakage.
+.. Please remember to credit the authors of the contributions by name,
+.. especially when they are new users or developers who do not regularly
+.. participate  in IPython's development.
+
+.. _whatsnew_index:
+
+=====================
+What's new in IPython
+=====================
+
+.. toctree::
+   :maxdepth: 1
+
+   development
+   version0.10
+   version0.9
+   version0.8
+
+

docs/source/whatsnew/version0.10.txt

@@ -0,0 +1,223 b''
+========================================
+0.10 series
+========================================
+
+Release 0.10
+============
+
+This release brings months of slow but steady development, and will be the last
+before a major restructuring and cleanup of IPython's internals that is already
+under way.  For this reason, we hope that 0.10 will be a stable and robust
+release so that while users adapt to some of the API changes that will come
+with the refactoring that will become IPython 0.11, they can safely use 0.10 in
+all existing projects with minimal changes (if any).
+
+IPython 0.10 is now a medium-sized project, with roughly (as reported by David
+Wheeler's :command:`sloccount` utility) 40750 lines of Python code, and a diff
+between 0.9.1 and this release that contains almost 28000 lines of code and
+documentation.  Our documentation, in PDF format, is a 495-page long PDF
+document (also available in HTML format, both generated from the same sources).
+
+Many users and developers contributed code, features, bug reports and ideas to
+this release.  Please do not hesitate in contacting us if we've failed to
+acknowledge your contribution here.  In particular, for this release we have
+contribution from the following people, a mix of new and regular names (in
+alphabetical order by first name):
+
+* Alexander Clausen: fix #341726.
+* Brian Granger: lots of work everywhere (features, bug fixes, etc).
+* Daniel Ashbrook: bug report on MemoryError during compilation, now fixed.
+* Darren Dale: improvements to documentation build system, feedback, design
+  ideas.
+* Fernando Perez: various places.
+* Gaël Varoquaux: core code, ipythonx GUI, design discussions, etc. Lots...
+* John Hunter: suggestions, bug fixes, feedback.
+* Jorgen Stenarson: work on many fronts, tests, fixes, win32 support, etc.
+* Laurent Dufréchou: many improvements to ipython-wx standalone app.
+* Lukasz Pankowski: prefilter, `%edit`, demo improvements.
+* Matt Foster: TextMate support in `%edit`.
+* Nathaniel Smith: fix #237073.
+* Pauli Virtanen: fixes and improvements to extensions, documentation.
+* Prabhu Ramachandran: improvements to `%timeit`.
+* Robert Kern: several extensions.
+* Sameer D'Costa: help on critical bug #269966.
+* Stephan Peijnik: feedback on Debian compliance and many man pages.
+* Steven Bethard: we are now shipping his :mod:`argparse` module.
+* Tom Fetherston: many improvements to :mod:`IPython.demo` module.
+* Ville Vainio: lots of work everywhere (features, bug fixes, etc).
+* Vishal Vasta: ssh support in ipcluster.
+* Walter Doerwald: work on the :mod:`IPython.ipipe` system.
+
+Below we give an overview of new features, bug fixes and backwards-incompatible
+changes.  For a detailed account of every change made, feel free to view the
+project log with :command:`bzr log`.
+
+New features
+------------
+
+* New `%paste` magic automatically extracts current contents of clipboard and
+  pastes it directly, while correctly handling code that is indented or
+  prepended with `>>>` or `...` python prompt markers.  A very useful new
+  feature contributed by Robert Kern.
+
+* IPython 'demos', created with the :mod:`IPython.demo` module, can now be
+  created from files on disk or strings in memory.  Other fixes and
+  improvements to the demo system, by Tom Fetherston.
+
+* Added :func:`find_cmd` function to :mod:`IPython.platutils` module, to find
+  commands in a cross-platform manner.
+
+* Many improvements and fixes to Gaël Varoquaux's :command:`ipythonx`, a
+  WX-based lightweight IPython instance that can be easily embedded in other WX
+  applications.  These improvements have made it possible to now have an
+  embedded IPython in Mayavi and other tools.
+
+* :class:`MultiengineClient` objects now have a :meth:`benchmark` method.
+
+* The manual now includes a full set of auto-generated API documents from the
+  code sources, using Sphinx and some of our own support code.  We are now
+  using the `Numpy Documentation Standard`_  for all docstrings, and we have
+  tried to update as many existing ones as possible to this format.
+
+* The new :mod:`IPython.Extensions.ipy_pretty` extension by Robert Kern
+  provides configurable pretty-printing.
+
+* Many improvements to the :command:`ipython-wx` standalone WX-based IPython
+  application by Laurent Dufréchou.  It can optionally run in a thread, and
+  this can be toggled at runtime (allowing the loading of Matplotlib in a
+  running session without ill effects).
+
+* IPython includes a copy of Steven Bethard's argparse_ in the
+  :mod:`IPython.external` package, so we can use it internally and it is also
+  available to any IPython user.  By installing it in this manner, we ensure
+  zero conflicts with any system-wide installation you may already have while
+  minimizing external dependencies for new users.  In IPython 0.10, We ship
+  argparse version 1.0.
+
+* An improved and much more robust test suite, that runs groups of tests in
+  separate subprocesses using either Nose or Twisted's :command:`trial` runner
+  to ensure proper management of Twisted-using code.  The test suite degrades
+  gracefully if optional dependencies are not available, so that the
+  :command:`iptest` command can be run with only Nose installed and nothing
+  else.  We also have more and cleaner test decorators to better select tests
+  depending on runtime conditions, do setup/teardown, etc.
+
+* The new ipcluster now has a fully working ssh mode that should work on
+  Linux, Unix and OS X.  Thanks to Vishal Vatsa for implementing this!
+
+* The wonderful TextMate editor can now be used with %edit on OS X.  Thanks
+  to Matt Foster for this patch.
+
+* The documentation regarding parallel uses of IPython, including MPI and PBS,
+  has been significantly updated and improved.
+
+* The developer guidelines in the documentation have been updated to explain
+  our workflow using :command:`bzr` and Launchpad.
+  
+* Fully refactored :command:`ipcluster` command line program for starting
+  IPython clusters.  This new version is a complete rewrite and 1) is fully
+  cross platform (we now use Twisted's process management), 2) has much
+  improved performance, 3) uses subcommands for different types of clusters, 4)
+  uses argparse for parsing command line options, 5) has better support for
+  starting clusters using :command:`mpirun`, 6) has experimental support for
+  starting engines using PBS.  It can also reuse FURL files, by appropriately
+  passing options to its subcommands.  However, this new version of ipcluster
+  should be considered a technology preview.  We plan on changing the API in
+  significant ways before it is final.
+
+* Full description of the security model added to the docs.
+
+* cd completer: show bookmarks if no other completions are available.
+
+* sh profile: easy way to give 'title' to prompt: assign to variable
+  '_prompt_title'. It looks like this::
+      
+        [~]|1> _prompt_title = 'sudo!'
+        sudo![~]|2>
+
+* %edit: If you do '%edit pasted_block', pasted_block variable gets updated
+  with new data (so repeated editing makes sense)
+
+.. _Numpy Documentation Standard: http://projects.scipy.org/numpy/wiki/CodingStyleGuidelines#docstring-standard
+
+.. _argparse: http://code.google.com/p/argparse/
+
+Bug fixes
+---------
+
+* Fix #368719, removed top-level debian/ directory to make the job of Debian
+  packagers easier.
+  
+* Fix #291143 by including man pages contributed by Stephan Peijnik from the
+  Debian project.
+
+* Fix #358202, effectively a race condition, by properly synchronizing file
+  creation at cluster startup time.
+
+* `%timeit` now handles correctly functions that take a long time to execute
+  even the first time, by not repeating them.
+
+* Fix #239054, releasing of references after exiting.
+
+* Fix #341726, thanks to Alexander Clausen.
+
+* Fix #269966.  This long-standing and very difficult bug (which is actually a
+  problem in Python itself) meant long-running sessions would inevitably grow
+  in memory size, often with catastrophic consequences if users had large
+  objects in their scripts.  Now, using `%run` repeatedly should not cause any
+  memory leaks.  Special thanks to John Hunter and Sameer D'Costa for their
+  help with this bug.
+
+* Fix #295371, bug in `%history`.
+
+* Improved support for py2exe.
+
+* Fix #270856: IPython hangs with PyGTK
+
+* Fix #270998: A magic with no docstring breaks the '%magic magic'
+
+* fix #271684: -c startup commands screw up raw vs. native history
+
+* Numerous bugs on Windows with the new ipcluster have been fixed.
+
+* The ipengine and ipcontroller scripts now handle missing furl files
+  more gracefully by giving better error messages.
+
+* %rehashx: Aliases no longer contain dots. python3.0 binary
+  will create alias python30. Fixes:
+  #259716 "commands with dots in them don't work"
+
+* %cpaste: %cpaste -r repeats the last pasted block.
+  The block is assigned to pasted_block even if code
+  raises exception.
+
+* Bug #274067 'The code in get_home_dir is broken for py2exe' was
+  fixed.
+
+* Many other small bug fixes not listed here by number (see the bzr log for
+  more info).
+  
+Backwards incompatible changes
+------------------------------
+
+* `ipykit` and related files were unmaintained and have been removed.
+
+* The :func:`IPython.genutils.doctest_reload` does not actually call
+  `reload(doctest)` anymore, as this was causing many problems with the test
+  suite.  It still resets `doctest.master` to None.
+
+* While we have not deliberately broken Python 2.4 compatibility, only minor
+  testing was done with Python 2.4, while 2.5 and 2.6 were fully tested.  But
+  if you encounter problems with 2.4, please do report them as bugs.
+
+* The :command:`ipcluster` now requires a mode argument; for example to start a
+  cluster on the local machine with 4 engines, you must now type::
+
+    $ ipcluster local -n 4
+
+* The controller now has a ``-r`` flag that needs to be used if you want to
+  reuse existing furl files.  Otherwise they are deleted (the default).
+
+* Remove ipy_leo.py. You can use :command:`easy_install ipython-extension` to
+  get it.  (done to decouple it from ipython release cycle)
+

docs/source/whatsnew/version0.8.txt

@@ -0,0 +1,34 b''
+========================================
+0.8 series
+========================================
+
+Release 0.8.4
+=============
+
+This was a quick release to fix an unfortunate bug that slipped into the 0.8.3
+release.  The ``--twisted`` option was disabled, as it turned out to be broken
+across several platforms.
+
+
+Release 0.8.3
+=============
+  
+* pydb is now disabled by default (due to %run -d problems). You can enable
+  it by passing -pydb command line argument to IPython. Note that setting
+  it in config file won't work.
+
+  
+Release 0.8.2
+=============
+
+* %pushd/%popd behave differently; now "pushd /foo" pushes CURRENT directory 
+  and jumps to /foo. The current behaviour is closer to the documented 
+  behaviour, and should not trip anyone.
+
+  
+Older releases
+==============
+
+Changes in earlier releases of IPython are described in the older file
+``ChangeLog``.  Please refer to this document for details.
+

docs/source/whatsnew/version0.9.txt

@@ -0,0 +1,283 b''
+========================================
+0.9 series
+========================================
+
+Release 0.9.1
+=============
+
+This release was quickly made to restore compatibility with Python 2.4, which
+version 0.9 accidentally broke.  No new features were introduced, other than
+some additional testing support for internal use.
+
+
+Release 0.9
+===========
+
+New features
+------------
+
+* All furl files and security certificates are now put in a read-only
+  directory named ~./ipython/security.
+
+* A single function :func:`get_ipython_dir`, in :mod:`IPython.genutils` that
+  determines the user's IPython directory in a robust manner.
+
+* Laurent's WX application has been given a top-level script called
+  ipython-wx, and it has received numerous fixes. We expect this code to be
+  architecturally better integrated with Gael's WX 'ipython widget' over the
+  next few releases.
+
+* The Editor synchronization work by Vivian De Smedt has been merged in.  This
+  code adds a number of new editor hooks to synchronize with editors under
+  Windows.
+
+* A new, still experimental but highly functional, WX shell by Gael Varoquaux.
+  This work was sponsored by Enthought, and while it's still very new, it is
+  based on a more cleanly organized arhictecture of the various IPython
+  components. We will continue to develop this over the next few releases as a
+  model for GUI components that use IPython.
+
+* Another GUI frontend, Cocoa based (Cocoa is the OSX native GUI framework),
+  authored by Barry Wark.  Currently the WX and the Cocoa ones have slightly
+  different internal organizations, but the whole team is working on finding
+  what the right abstraction points are for a unified codebase.
+
+* As part of the frontend work, Barry Wark also implemented an experimental
+  event notification system that various ipython components can use.  In the
+  next release the implications and use patterns of this system regarding the
+  various GUI options will be worked out.
+
+* IPython finally has a full test system, that can test docstrings with
+  IPython-specific functionality.  There are still a few pieces missing for it
+  to be widely accessible to all users (so they can run the test suite at any
+  time and report problems), but it now works for the developers.  We are
+  working hard on continuing to improve it, as this was probably IPython's
+  major Achilles heel (the lack of proper test coverage made it effectively
+  impossible to do large-scale refactoring).  The full test suite can now
+  be run using the :command:`iptest` command line program.
+
+* The notion of a task has been completely reworked.  An `ITask` interface has
+  been created.  This interface defines the methods that tasks need to
+  implement.  These methods are now responsible for things like submitting
+  tasks and processing results.  There are two basic task types:
+  :class:`IPython.kernel.task.StringTask` (this is the old `Task` object, but
+  renamed) and the new :class:`IPython.kernel.task.MapTask`, which is based on
+  a function.
+
+* A new interface, :class:`IPython.kernel.mapper.IMapper` has been defined to
+  standardize the idea of a `map` method.  This interface has a single `map`
+  method that has the same syntax as the built-in `map`.  We have also defined
+  a `mapper` factory interface that creates objects that implement
+  :class:`IPython.kernel.mapper.IMapper` for different controllers.  Both the
+  multiengine and task controller now have mapping capabilties.
+
+* The parallel function capabilities have been reworks.  The major changes are
+  that i) there is now an `@parallel` magic that creates parallel functions,
+  ii) the syntax for mulitple variable follows that of `map`, iii) both the
+  multiengine and task controller now have a parallel function implementation.
+
+* All of the parallel computing capabilities from `ipython1-dev` have been
+  merged into IPython proper.  This resulted in the following new subpackages:
+  :mod:`IPython.kernel`, :mod:`IPython.kernel.core`, :mod:`IPython.config`,
+  :mod:`IPython.tools` and :mod:`IPython.testing`.
+
+* As part of merging in the `ipython1-dev` stuff, the `setup.py` script and
+  friends have been completely refactored.  Now we are checking for
+  dependencies using the approach that matplotlib uses.
+
+* The documentation has been completely reorganized to accept the 
+  documentation from `ipython1-dev`.
+
+* We have switched to using Foolscap for all of our network protocols in
+  :mod:`IPython.kernel`.  This gives us secure connections that are both
+  encrypted and authenticated.
+
+* We have a brand new `COPYING.txt` files that describes the IPython license
+  and copyright. The biggest change is that we are putting "The IPython
+  Development Team" as the copyright holder. We give more details about
+  exactly what this means in this file. All developer should read this and use
+  the new banner in all IPython source code files.
+
+* sh profile: ./foo runs foo as system command, no need to do !./foo anymore
+
+* String lists now support ``sort(field, nums = True)`` method (to easily sort
+  system command output). Try it with ``a = !ls -l ; a.sort(1, nums=1)``.
+
+* '%cpaste foo' now assigns the pasted block as string list, instead of string
+
+* The ipcluster script now run by default with no security.  This is done
+  because the main usage of the script is for starting things on localhost.
+  Eventually when ipcluster is able to start things on other hosts, we will put
+  security back.
+
+* 'cd --foo' searches directory history for string foo, and jumps to that dir.
+  Last part of dir name is checked first. If no matches for that are found,
+  look at the whole path.
+
+  
+Bug fixes
+---------
+
+* The Windows installer has been fixed.  Now all IPython scripts have ``.bat``
+  versions created.  Also, the Start Menu shortcuts have been updated.
+
+* The colors escapes in the multiengine client are now turned off on win32 as
+  they don't print correctly.
+
+* The :mod:`IPython.kernel.scripts.ipengine` script was exec'ing
+  mpi_import_statement incorrectly, which was leading the engine to crash when
+  mpi was enabled.
+
+* A few subpackages had missing ``__init__.py`` files.
+
+* The documentation is only created if Sphinx is found.  Previously, the
+  ``setup.py`` script would fail if it was missing.
+
+* Greedy ``cd`` completion has been disabled again (it was enabled in 0.8.4) as
+  it caused problems on certain platforms.
+          
+
+Backwards incompatible changes
+------------------------------
+
+* The ``clusterfile`` options of the :command:`ipcluster` command has been
+  removed as it was not working and it will be replaced soon by something much
+  more robust.
+
+* The :mod:`IPython.kernel` configuration now properly find the user's
+  IPython directory.
+
+* In ipapi, the :func:`make_user_ns` function has been replaced with
+  :func:`make_user_namespaces`, to support dict subclasses in namespace
+  creation.
+
+* :class:`IPython.kernel.client.Task` has been renamed
+  :class:`IPython.kernel.client.StringTask` to make way for new task types.
+
+* The keyword argument `style` has been renamed `dist` in `scatter`, `gather`
+  and `map`.
+
+* Renamed the values that the rename `dist` keyword argument can have from
+  `'basic'` to `'b'`.
+
+* IPython has a larger set of dependencies if you want all of its capabilities.
+  See the `setup.py` script for details.
+
+* The constructors for :class:`IPython.kernel.client.MultiEngineClient` and 
+  :class:`IPython.kernel.client.TaskClient` no longer take the (ip,port) tuple.
+  Instead they take the filename of a file that contains the FURL for that
+  client.  If the FURL file is in your IPYTHONDIR, it will be found automatically
+  and the constructor can be left empty.
+
+* The asynchronous clients in :mod:`IPython.kernel.asyncclient` are now created 
+  using the factory functions :func:`get_multiengine_client` and 
+  :func:`get_task_client`.  These return a `Deferred` to the actual client.
+
+* The command line options to `ipcontroller` and `ipengine` have changed to
+  reflect the new Foolscap network protocol and the FURL files.  Please see the
+  help for these scripts for details.
+
+* The configuration files for the kernel have changed because of the Foolscap
+  stuff.  If you were using custom config files before, you should delete them
+  and regenerate new ones.
+
+Changes merged in from IPython1
+-------------------------------
+
+New features
+............
+
+* Much improved ``setup.py`` and ``setupegg.py`` scripts.  Because Twisted and
+  zope.interface are now easy installable, we can declare them as dependencies
+  in our setupegg.py script.
+
+* IPython is now compatible with Twisted 2.5.0 and 8.x.
+
+* Added a new example of how to use :mod:`ipython1.kernel.asynclient`.
+
+* Initial draft of a process daemon in :mod:`ipython1.daemon`.  This has not
+  been merged into IPython and is still in `ipython1-dev`.
+  
+* The ``TaskController`` now has methods for getting the queue status.
+
+* The ``TaskResult`` objects not have information about how long the task
+  took to run.
+  
+* We are attaching additional attributes to exceptions ``(_ipython_*)`` that 
+  we use to carry additional info around.
+  
+* New top-level module :mod:`asyncclient` that has asynchronous versions (that
+  return deferreds) of the client classes.  This is designed to users who want
+  to run their own Twisted reactor.
+  
+* All the clients in :mod:`client` are now based on Twisted.  This is done by 
+  running the Twisted reactor in a separate thread and using the
+  :func:`blockingCallFromThread` function that is in recent versions of Twisted.
+
+* Functions can now be pushed/pulled to/from engines using
+  :meth:`MultiEngineClient.push_function` and
+  :meth:`MultiEngineClient.pull_function`.
+
+* Gather/scatter are now implemented in the client to reduce the work load
+  of the controller and improve performance.
+
+* Complete rewrite of the IPython docuementation.  All of the documentation
+  from the IPython website has been moved into docs/source as restructured
+  text documents.  PDF and HTML documentation are being generated using 
+  Sphinx.
+
+* New developer oriented documentation: development guidelines and roadmap. 
+
+* Traditional ``ChangeLog`` has been changed to a more useful ``changes.txt``
+  file that is organized by release and is meant to provide something more
+  relevant for users.
+
+Bug fixes
+.........
+
+* Created a proper ``MANIFEST.in`` file to create source distributions.
+
+* Fixed a bug in the ``MultiEngine`` interface.  Previously, multi-engine 
+  actions were being collected with a :class:`DeferredList` with 
+  ``fireononeerrback=1``.  This meant that methods were returning 
+  before all engines had given their results.  This was causing extremely odd 
+  bugs in certain cases. To fix this problem, we have 1) set 
+  ``fireononeerrback=0`` to make sure all results (or exceptions) are in 
+  before returning and 2) introduced a :exc:`CompositeError` exception 
+  that wraps all of the engine exceptions.  This is a huge change as it means 
+  that users will have to catch :exc:`CompositeError` rather than the actual
+  exception.
+
+Backwards incompatible changes
+..............................
+
+* All names have been renamed to conform to the lowercase_with_underscore
+  convention.  This will require users to change references to all names like
+  ``queueStatus`` to ``queue_status``.
+
+* Previously, methods like :meth:`MultiEngineClient.push` and
+  :meth:`MultiEngineClient.push` used ``*args`` and ``**kwargs``.  This was
+  becoming a problem as we weren't able to introduce new keyword arguments into
+  the API.  Now these methods simple take a dict or sequence.  This has also
+  allowed us to get rid of the ``*All`` methods like :meth:`pushAll` and
+  :meth:`pullAll`.  These things are now handled with the ``targets`` keyword
+  argument that defaults to ``'all'``.
+
+* The :attr:`MultiEngineClient.magicTargets` has been renamed to
+  :attr:`MultiEngineClient.targets`. 
+
+* All methods in the MultiEngine interface now accept the optional keyword
+  argument ``block``.
+
+* Renamed :class:`RemoteController` to :class:`MultiEngineClient` and 
+  :class:`TaskController` to :class:`TaskClient`.
+
+* Renamed the top-level module from :mod:`api` to :mod:`client`.
+
+* Most methods in the multiengine interface now raise a :exc:`CompositeError`
+  exception that wraps the user's exceptions, rather than just raising the raw
+  user's exception.
+
+* Changed the ``setupNS`` and ``resultNames`` in the ``Task`` class to ``push`` 
+  and ``pull``.
+

docs/autogen_api.py

@@ -1,36 +1,42 b''
 #!/usr/bin/env python
 """Script to auto-generate our API docs.
 """
 # stdlib imports
 import os
 import sys
 
 # local imports
 sys.path.append(os.path.abspath('sphinxext'))
 from apigen import ApiDocWriter
 
 #*****************************************************************************
 if __name__ == '__main__':
     pjoin = os.path.join
     package = 'IPython'
     outdir = pjoin('source','api','generated')
     docwriter = ApiDocWriter(package,rst_extension='.txt')
+    # You have to escape the . here because . is a special char for regexps.
+    # You must do make clean if you change this!
     docwriter.package_skip_patterns += [r'\.fixes$',
-                                        r'\.externals$',
+                                        r'\.external$',
                                         r'\.extensions',
-                                        r'\.kernel.config',
+                                        r'\.kernel\.config',
                                         r'\.attic',
                                         r'\.quarantine',
-                                        r'\.deathrow'
+                                        r'\.deathrow',
+                                        r'\.config\.default',
+                                        r'\.config\.profile',
+                                        r'\.frontend',
+                                        r'\.gui'
                                         ]
-    docwriter.module_skip_patterns += [ r'\.core.fakemodule',
+    docwriter.module_skip_patterns += [ r'\.core\.fakemodule',
                                         r'\.cocoa',
                                         r'\.ipdoctest',
                                         r'\.Gnuplot',
-                                        r'\.frontend.process.winprocess',
+                                        r'\.frontend\.process\.winprocess',
                                         ]
     docwriter.write_api_docs(outdir)
     docwriter.write_index(outdir, 'gen',
                           relative_to = pjoin('source','api')
                           )
     print '%d files written' % len(docwriter.written_modules)

docs/source/about/credits.txt

@@ -1,207 +1,208 b''
 .. _credits:
 
 =======
 Credits
 =======
 
 IPython is led by Fernando Pérez.
 
 As of this writing, the following developers have joined the core team:
 
 * [Robert Kern] <rkern-AT-enthought.com>: co-mentored the 2005
   Google Summer of Code project to develop python interactive
   notebooks (XML documents) and graphical interface. This project
   was awarded to the students Tzanko Matev <tsanko-AT-gmail.com> and
   Toni Alatalo <antont-AT-an.org>.
 
 * [Brian Granger] <ellisonbg-AT-gmail.com>: extending IPython to allow
   support for interactive parallel computing.
 
 * [Benjamin (Min) Ragan-Kelley]: key work on IPython's parallel
   computing infrastructure.
 
 * [Ville Vainio] <vivainio-AT-gmail.com>: Ville has made many improvements
   to the core of IPython and was the maintainer of the main IPython
   trunk from version 0.7.1 to 0.8.4.
 
 * [Gael Varoquaux] <gael.varoquaux-AT-normalesup.org>: work on the merged
   architecture for the interpreter as of version 0.9, implementing a new WX GUI
   based on this system.
 
 * [Barry Wark] <barrywark-AT-gmail.com>: implementing a new Cocoa GUI, as well
   as work on the new interpreter architecture and Twisted support.
 
 * [Laurent Dufrechou] <laurent.dufrechou-AT-gmail.com>: development of the WX
   GUI support.
 
 * [Jörgen Stenarson] <jorgen.stenarson-AT-bostream.nu>: maintainer of the
   PyReadline project, necessary for IPython under windows.
 
 
 The IPython project is also very grateful to:
 
 Bill Bumgarner <bbum-AT-friday.com>: for providing the DPyGetOpt module
 which gives very powerful and convenient handling of command-line
 options (light years ahead of what Python 2.1.1's getopt module does).
 
 Ka-Ping Yee <ping-AT-lfw.org>: for providing the Itpl module for
 convenient and powerful string interpolation with a much nicer syntax
 than formatting through the '%' operator.
 
 Arnd Baecker <baecker-AT-physik.tu-dresden.de>: for his many very useful
 suggestions and comments, and lots of help with testing and
 documentation checking. Many of IPython's newer features are a result of
 discussions with him (bugs are still my fault, not his).
 
 Obviously Guido van Rossum and the whole Python development team, that
 goes without saying.
 
 IPython's website is generously hosted at http://ipython.scipy.orgby
 Enthought (http://www.enthought.com). I am very grateful to them and all
 of the SciPy team for their contribution.
 
 Fernando would also like to thank Stephen Figgins <fig-AT-monitor.net>,
 an O'Reilly Python editor. His Oct/11/2001 article about IPP and
 LazyPython, was what got this project started. You can read it at:
 http://www.onlamp.com/pub/a/python/2001/10/11/pythonnews.html.
 
 And last but not least, all the kind IPython users who have emailed new code,
 bug reports, fixes, comments and ideas. A brief list follows, please let us
 know if we have ommitted your name by accident:
 
 * Dan Milstein <danmil-AT-comcast.net>.  A bold refactoring of the 
   core prefilter stuff in the IPython interpreter.
 
 * [Jack Moffit] <jack-AT-xiph.org> Bug fixes, including the infamous
   color problem. This bug alone caused many lost hours and
   frustration, many thanks to him for the fix. I've always been a
   fan of Ogg & friends, now I have one more reason to like these folks.
   Jack is also contributing with Debian packaging and many other
   things.
  
 * [Alexander Schmolck] <a.schmolck-AT-gmx.net> Emacs work, bug
   reports, bug fixes, ideas, lots more. The ipython.el mode for
   (X)Emacs is Alex's code, providing full support for IPython under
   (X)Emacs.
 
 * [Andrea Riciputi] <andrea.riciputi-AT-libero.it> Mac OSX
   information, Fink package management.
 
 * [Gary Bishop] <gb-AT-cs.unc.edu> Bug reports, and patches to work
   around the exception handling idiosyncracies of WxPython. Readline
   and color support for Windows.
 
 * [Jeffrey Collins] <Jeff.Collins-AT-vexcel.com> Bug reports. Much
   improved readline support, including fixes for Python 2.3.
 
 * [Dryice Liu] <dryice-AT-liu.com.cn> FreeBSD port.
 
 * [Mike Heeter] <korora-AT-SDF.LONESTAR.ORG>
 
 * [Christopher Hart] <hart-AT-caltech.edu> PDB integration.
 
 * [Milan Zamazal] <pdm-AT-zamazal.org> Emacs info.
 
 * [Philip Hisley] <compsys-AT-starpower.net>
 
 * [Holger Krekel] <pyth-AT-devel.trillke.net> Tab completion, lots
   more.
 
 * [Robin Siebler] <robinsiebler-AT-starband.net>
 
 * [Ralf Ahlbrink] <ralf_ahlbrink-AT-web.de>
 
 * [Thorsten Kampe] <thorsten-AT-thorstenkampe.de>
 
 * [Fredrik Kant] <fredrik.kant-AT-front.com> Windows setup.
 
 * [Syver Enstad] <syver-en-AT-online.no> Windows setup.
 
 * [Richard] <rxe-AT-renre-europe.com> Global embedding.
 
 * [Hayden Callow] <h.callow-AT-elec.canterbury.ac.nz> Gnuplot.py 1.6
   compatibility.
   
 * [Leonardo Santagada] <retype-AT-terra.com.br> Fixes for Windows
   installation.
   
 * [Christopher Armstrong] <radix-AT-twistedmatrix.com> Bugfixes.
 
 * [Francois Pinard] <pinard-AT-iro.umontreal.ca> Code and
   documentation fixes.
   
 * [Cory Dodt] <cdodt-AT-fcoe.k12.ca.us> Bug reports and Windows
   ideas. Patches for Windows installer.
   
 * [Olivier Aubert] <oaubert-AT-bat710.univ-lyon1.fr> New magics.
 
 * [King C. Shu] <kingshu-AT-myrealbox.com> Autoindent patch.
 
 * [Chris Drexler] <chris-AT-ac-drexler.de> Readline packages for
   Win32/CygWin.
   
 * [Gustavo Cordova Avila] <gcordova-AT-sismex.com> EvalDict code for
   nice, lightweight string interpolation.
   
 * [Kasper Souren] <Kasper.Souren-AT-ircam.fr> Bug reports, ideas.
 
 * [Gever Tulley] <gever-AT-helium.com> Code contributions.
 
 * [Ralf Schmitt] <ralf-AT-brainbot.com> Bug reports & fixes.
 
 * [Oliver Sander] <osander-AT-gmx.de> Bug reports.
 
 * [Rod Holland] <rhh-AT-structurelabs.com> Bug reports and fixes to
   logging module.
 
 * [Daniel 'Dang' Griffith] <pythondev-dang-AT-lazytwinacres.net>
   Fixes, enhancement suggestions for system shell use.
 
 * [Viktor Ransmayr] <viktor.ransmayr-AT-t-online.de> Tests and
   reports on Windows installation issues. Contributed a true Windows
   binary installer.
 
 * [Mike Salib] <msalib-AT-mit.edu> Help fixing a subtle bug related
   to traceback printing.
 
 * [W.J. van der Laan] <gnufnork-AT-hetdigitalegat.nl> Bash-like
   prompt specials.
 
 * [Antoon Pardon] <Antoon.Pardon-AT-rece.vub.ac.be> Critical fix for
   the multithreaded IPython.
 
 * [John Hunter] <jdhunter-AT-nitace.bsd.uchicago.edu> Matplotlib
   author, helped with all the development of support for matplotlib
   in IPyhton, including making necessary changes to matplotlib itself.
 
 * [Matthew Arnison] <maffew-AT-cat.org.au> Bug reports, '%run -d' idea.
 
 * [Prabhu Ramachandran] <prabhu_r-AT-users.sourceforge.net> Help
   with (X)Emacs support, threading patches, ideas...
 
 * [Norbert Tretkowski] <tretkowski-AT-inittab.de> help with Debian
   packaging and distribution.
 
 * [George Sakkis] <gsakkis-AT-eden.rutgers.edu> New matcher for
   tab-completing named arguments of user-defined functions.
 
 * [Jörgen Stenarson] <jorgen.stenarson-AT-bostream.nu> Wildcard
   support implementation for searching namespaces.
 
 * [Vivian De Smedt] <vivian-AT-vdesmedt.com> Debugger enhancements,
   so that when pdb is activated from within IPython, coloring, tab
   completion and other features continue to work seamlessly.
 
 * [Scott Tsai] <scottt958-AT-yahoo.com.tw> Support for automatic
   editor invocation on syntax errors (see
   http://www.scipy.net/roundup/ipython/issue36).
 
 * [Alexander Belchenko] <bialix-AT-ukr.net> Improvements for win32
   paging system.
 
 * [Will Maier] <willmaier-AT-ml1.net> Official OpenBSD port.
 
 * [Ondrej Certik] <ondrej-AT-certik.cz>: set up the IPython docs to use the new
   Sphinx system used by Python, Matplotlib and many more projects.
 
 * [Stefan van der Walt] <stefan-AT-sun.ac.za>: support for the new config system.
+

docs/source/about/license_and_copyright.txt

@@ -1,91 +1,92 b''
 .. _license:
 
 =====================
 License and Copyright
 =====================
 
 License
 =======
 
 IPython is licensed under the terms of the new or revised BSD license, as follows::
 
     Copyright (c) 2008, IPython Development Team
 
     All rights reserved.
 
     Redistribution and use in source and binary forms, with or without
     modification, are permitted provided that the following conditions are
     met:
 
     Redistributions of source code must retain the above copyright notice,
     this list of conditions and the following disclaimer.
 
     Redistributions in binary form must reproduce the above copyright notice,
     this list of conditions and the following disclaimer in the documentation
     and/or other materials provided with the distribution.
 
     Neither the name of the IPython Development Team nor the names of its
     contributors may be used to endorse or promote products derived from this
     software without specific prior written permission.
 
     THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
     IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
     THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
     PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
     CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
     EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
     PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
     PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
     LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
     NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
     SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 About the IPython Development Team
 ==================================
 
 Fernando Perez began IPython in 2001 based on code from Janko Hauser
 <[e-mail]> and Nathaniel Gray <[e-mail]>. Fernando is still
 the project lead.
 
 The IPython Development Team is the set of all contributors to the IPython
 project. This includes all of the IPython subprojects. Here is a list of the
 currently active contributors:
 
 	* Matthieu Brucher
 	* Ondrej Certik
 	* Laurent Dufrechou
 	* Robert Kern
 	* Brian E. Granger
 	* Fernando Perez (project leader)
 	* Benjamin Ragan-Kelley
 	* Ville M. Vainio
 	* Gael Varoququx
 	* Stefan van der Walt
 	* Tech-X Corporation
 	* Barry Wark
 
 If your name is missing, please add it.  
 
 Our Copyright Policy
 ====================
 
 IPython uses a shared copyright model. Each contributor maintains copyright
 over their contributions to IPython. But, it is important to note that these
 contributions are typically only changes to the repositories. Thus, the
 IPython source code, in its entirety is not the copyright of any single person
 or institution. Instead, it is the collective copyright of the entire IPython
 Development Team. If individual contributors want to maintain a record of what
 changes/contributions they have specific copyright on, they should indicate
 their copyright in the commit message of the change, when they commit the
 change to one of the IPython repositories.
 
 Miscellaneous
 =============
 
 Some files (DPyGetOpt.py, for example) may be licensed under different
 conditions. Ultimately each file indicates clearly the conditions under which
 its author/authors have decided to publish the code.
 
 Versions of IPython up to and including 0.6.3 were released under the GNU
 Lesser General Public License (LGPL), available at
-http://www.gnu.org/copyleft/lesser.html.
  No newline at end of file
+http://www.gnu.org/copyleft/lesser.html.
+

docs/source/conf.py

@@ -1,191 +1,192 b''
 # -*- coding: utf-8 -*-
 #
 # IPython documentation build configuration file.
 
 # NOTE: This file has been edited manually from the auto-generated one from
 # sphinx.  Do NOT delete and re-generate.  If any changes from sphinx are
 # needed, generate a scratch one and merge by hand any new fields needed.
 
 #
 # This file is execfile()d with the current directory set to its containing dir.
 #
 # The contents of this file are pickled, so don't put values in the namespace
 # that aren't pickleable (module imports are okay, they're removed automatically).
 #
 # All configuration values have a default value; values that are commented out
 # serve to show the default value.
 
 import sys, os
 
 # If your extensions are in another directory, add it here. If the directory
 # is relative to the documentation root, use os.path.abspath to make it
 # absolute, like shown here.
 sys.path.append(os.path.abspath('../sphinxext'))
 
 # Import support for ipython console session syntax highlighting (lives
 # in the sphinxext directory defined above)
 import ipython_console_highlighting
 
 # We load the ipython release info into a dict by explicit execution
 iprelease = {}
 execfile('../../IPython/core/release.py',iprelease)
 
 # General configuration
 # ---------------------
 
 # Add any Sphinx extension module names here, as strings. They can be extensions
 # coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
-extensions = ['sphinx.ext.autodoc',
-              'sphinx.ext.doctest',
-              
-              'only_directives',
-              'inheritance_diagram',
-              'ipython_console_highlighting', 
-              # 'plot_directive', # disabled for now, needs matplotlib
-              'numpydoc',  # to preprocess docstrings
-              ]
+extensions = [
+    # 'matplotlib.sphinxext.mathmpl',
+    'matplotlib.sphinxext.only_directives',
+    # 'matplotlib.sphinxext.plot_directive',
+    'sphinx.ext.autodoc',
+    'sphinx.ext.doctest',
+    'inheritance_diagram',
+    'ipython_console_highlighting', 
+    'numpydoc',  # to preprocess docstrings
+]
 
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ['_templates']
 
 # The suffix of source filenames.
 source_suffix = '.txt'
 
 # The master toctree document.
 master_doc = 'index'
 
 # General substitutions.
 project = 'IPython'
 copyright = '2008, The IPython Development Team'
 
 # The default replacements for |version| and |release|, also used in various
 # other places throughout the built documents.
 #
 # The full version, including alpha/beta/rc tags.
 release = iprelease['version']
 # The short X.Y version.
 version = '.'.join(release.split('.',2)[:2])
 
 
 # There are two options for replacing |today|: either, you set today to some
 # non-false value, then it is used:
 #today = ''
 # Else, today_fmt is used as the format for a strftime call.
 today_fmt = '%B %d, %Y'
 
 # List of documents that shouldn't be included in the build.
 #unused_docs = []
 
 # List of directories, relative to source directories, that shouldn't be searched
 # for source files.
 exclude_dirs = ['attic']
 
 # If true, '()' will be appended to :func: etc. cross-reference text.
 #add_function_parentheses = True
 
 # If true, the current module name will be prepended to all description
 # unit titles (such as .. function::).
 #add_module_names = True
 
 # If true, sectionauthor and moduleauthor directives will be shown in the
 # output. They are ignored by default.
 #show_authors = False
 
 # The name of the Pygments (syntax highlighting) style to use.
 pygments_style = 'sphinx'
 
 
 # Options for HTML output
 # -----------------------
 
 # The style sheet to use for HTML and HTML Help pages. A file of that name
 # must exist either in Sphinx' static/ path, or in one of the custom paths
 # given in html_static_path.
 html_style = 'default.css'
 
 # The name for this set of Sphinx documents.  If None, it defaults to
 # "<project> v<release> documentation".
 #html_title = None
 
 # The name of an image file (within the static path) to place at the top of
 # the sidebar.
 #html_logo = None
 
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
 html_static_path = ['_static']
 
 # If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
 # using the given strftime format.
 html_last_updated_fmt = '%b %d, %Y'
 
 # If true, SmartyPants will be used to convert quotes and dashes to
 # typographically correct entities.
 #html_use_smartypants = True
 
 # Custom sidebar templates, maps document names to template names.
 #html_sidebars = {}
 
 # Additional templates that should be rendered to pages, maps page names to
 # template names.
 #html_additional_pages = {}
 
 # If false, no module index is generated.
 #html_use_modindex = True
 
 # If true, the reST sources are included in the HTML build as _sources/<name>.
 #html_copy_source = True
 
 # If true, an OpenSearch description file will be output, and all pages will
 # contain a <link> tag referring to it.  The value of this option must be the
 # base URL from which the finished HTML is served.
 #html_use_opensearch = ''
 
 # If nonempty, this is the file name suffix for HTML files (e.g. ".xhtml").
 #html_file_suffix = ''
 
 # Output file base name for HTML help builder.
 htmlhelp_basename = 'ipythondoc'
 
 
 # Options for LaTeX output
 # ------------------------
 
 # The paper size ('letter' or 'a4').
 latex_paper_size = 'letter'
 
 # The font size ('10pt', '11pt' or '12pt').
 latex_font_size = '11pt'
 
 # Grouping the document tree into LaTeX files. List of tuples
 # (source start file, target name, title, author, document class [howto/manual]).
 
 latex_documents = [ ('index', 'ipython.tex', 'IPython Documentation',
                      ur"""The IPython Development Team""",
                      'manual'),
                     ]
 
 # The name of an image file (relative to this directory) to place at the top of
 # the title page.
 #latex_logo = None
 
 # For "manual" documents, if this is true, then toplevel headings are parts,
 # not chapters.
 #latex_use_parts = False
 
 # Additional stuff for the LaTeX preamble.
 #latex_preamble = ''
 
 # Documents to append as an appendix to all manuals.
 #latex_appendices = []
 
 # If false, no module index is generated.
 #latex_use_modindex = True
 
 
 # Cleanup
 # -------
 # delete release info to avoid pickling errors from sphinx
 
 del iprelease

docs/source/config/customization.txt

@@ -1,286 +1,287 b''
 .. _customization:
 
 ========================
 Customization of IPython
 ========================
 
 There are 2 ways to configure IPython - the old way of using ipythonrc
 files (an INI-file like format), and the new way that involves editing
 your ipy_user_conf.py. Both configuration systems work at the same
 time, so you can set your options in both, but if you are hesitating
 about which alternative to choose, we recommend the ipy_user_conf.py
 approach, as it will give you more power and control in the long
 run. However, there are few options such as pylab_import_all that can
 only be specified in ipythonrc file or command line - the reason for
 this is that they are needed before IPython has been started up, and
 the IPApi object used in ipy_user_conf.py is not yet available at that
 time. A hybrid approach of specifying a few options in ipythonrc and
 doing the more advanced configuration in ipy_user_conf.py is also
 possible.
 
 .. _ipythonrc:
 
 The ipythonrc approach
 ======================
 
 As we've already mentioned, IPython reads a configuration file which can
 be specified at the command line (-rcfile) or which by default is
 assumed to be called ipythonrc. Such a file is looked for in the current
 directory where IPython is started and then in your IPYTHONDIR, which
 allows you to have local configuration files for specific projects. In
 this section we will call these types of configuration files simply
 rcfiles (short for resource configuration file).
 
 The syntax of an rcfile is one of key-value pairs separated by
 whitespace, one per line. Lines beginning with a # are ignored as
 comments, but comments can not be put on lines with data (the parser is
 fairly primitive). Note that these are not python files, and this is
 deliberate, because it allows us to do some things which would be quite
 tricky to implement if they were normal python files.
 
 First, an rcfile can contain permanent default values for almost all command
 line options (except things like -help or -Version). :ref:`This section
 <command_line_options>` contains a description of all command-line
 options. However, values you explicitly specify at the command line override
 the values defined in the rcfile.
 
 Besides command line option values, the rcfile can specify values for
 certain extra special options which are not available at the command
 line. These options are briefly described below.
 
 Each of these options may appear as many times as you need it in the file.
 
     * include <file1> <file2> ...: you can name other rcfiles you want
       to recursively load up to 15 levels (don't use the <> brackets in
       your names!). This feature allows you to define a 'base' rcfile
       with general options and special-purpose files which can be loaded
       only when needed with particular configuration options. To make
       this more convenient, IPython accepts the -profile <name> option
       (abbreviates to -p <name>) which tells it to look for an rcfile
       named ipythonrc-<name>.
     * import_mod <mod1> <mod2> ...: import modules with 'import
       <mod1>,<mod2>,...'
     * import_some <mod> <f1> <f2> ...: import functions with 'from
       <mod> import <f1>,<f2>,...'
     * import_all <mod1> <mod2> ...: for each module listed import
       functions with ``from <mod> import *``.
     * execute <python code>: give any single-line python code to be
       executed.
     * execfile <filename>: execute the python file given with an
       'execfile(filename)' command. Username expansion is performed on
       the given names. So if you need any amount of extra fancy
       customization that won't fit in any of the above 'canned' options,
       you can just put it in a separate python file and execute it.
     * alias <alias_def>: this is equivalent to calling
       '%alias <alias_def>' at the IPython command line. This way, from
       within IPython you can do common system tasks without having to
       exit it or use the ! escape. IPython isn't meant to be a shell
       replacement, but it is often very useful to be able to do things
       with files while testing code. This gives you the flexibility to
       have within IPython any aliases you may be used to under your
       normal system shell.
 
 ipy_user_conf.py
 ================
 
 There should be a simple template ipy_user_conf.py file in your
 ~/.ipython directory. It is a plain python module that is imported
 during IPython startup, so you can do pretty much what you want there
 - import modules, configure extensions, change options, define magic
 commands, put variables and functions in the IPython namespace,
 etc. You use the IPython extension api object, acquired by
 IPython.ipapi.get() and documented in the "IPython extension API"
 chapter, to interact with IPython. A sample ipy_user_conf.py is listed
 below for reference::
 
     # Most of your config files and extensions will probably start
     # with this import
 
     import IPython.ipapi
     ip = IPython.ipapi.get()
 
     # You probably want to uncomment this if you did %upgrade -nolegacy
     # import ipy_defaults    
 
     import os   
 
     def main():   
 
         #ip.dbg.debugmode = True
         ip.dbg.debug_stack()
 
         # uncomment if you want to get ipython -p sh behaviour
         # without having to use command line switches  
         import ipy_profile_sh
         import jobctrl
 
         # Configure your favourite editor?
         # Good idea e.g. for %edit os.path.isfile
 
         #import ipy_editors
 
         # Choose one of these:
 
         #ipy_editors.scite()
         #ipy_editors.scite('c:/opt/scite/scite.exe')
         #ipy_editors.komodo()
         #ipy_editors.idle()
         # ... or many others, try 'ipy_editors??' after import to see them
 
         # Or roll your own:
         #ipy_editors.install_editor("c:/opt/jed +$line $file")
 
 
         o = ip.options
         # An example on how to set options
         #o.autocall = 1
         o.system_verbose = 0
 
         #import_all("os sys")
         #execf('~/_ipython/ns.py')
 
 
         # -- prompt
         # A different, more compact set of prompts from the default ones, that
         # always show your current location in the filesystem:
 
         #o.prompt_in1 = r'\C_LightBlue[\C_LightCyan\Y2\C_LightBlue]\C_Normal\n\C_Green|\#>'
         #o.prompt_in2 = r'.\D: '
         #o.prompt_out = r'[\#] '
 
         # Try one of these color settings if you can't read the text easily
         # autoexec is a list of IPython commands to execute on startup
         #o.autoexec.append('%colors LightBG')
         #o.autoexec.append('%colors NoColor')
         o.autoexec.append('%colors Linux')
 
 
     # some config helper functions you can use 
     def import_all(modules):
         """ Usage: import_all("os sys") """ 
         for m in modules.split():
             ip.ex("from %s import *" % m)
 
     def execf(fname):
         """ Execute a file in user namespace """
         ip.ex('execfile("%s")' % os.path.expanduser(fname))
 
     main()
 
 .. _Prompts:
 
 Fine-tuning your prompt
 =======================
 
 IPython's prompts can be customized using a syntax similar to that of
 the bash shell. Many of bash's escapes are supported, as well as a few
 additional ones. We list them below::
 
     \#
         the prompt/history count number. This escape is automatically
         wrapped in the coloring codes for the currently active color scheme. 
     \N
         the 'naked' prompt/history count number: this is just the number
         itself, without any coloring applied to it. This lets you produce
         numbered prompts with your own colors. 
     \D
         the prompt/history count, with the actual digits replaced by dots.
         Used mainly in continuation prompts (prompt_in2) 
     \w
         the current working directory 
     \W
         the basename of current working directory 
     \Xn
         where $n=0\ldots5.$ The current working directory, with $HOME
         replaced by ~, and filtered out to contain only $n$ path elements 
     \Yn
         Similar to \Xn, but with the $n+1$ element included if it is ~ (this
         is similar to the behavior of the %cn escapes in tcsh) 
     \u
         the username of the current user 
     \$
         if the effective UID is 0, a #, otherwise a $ 
     \h
         the hostname up to the first '.' 
     \H
         the hostname 
     \n
         a newline 
     \r
         a carriage return 
     \v
         IPython version string 
 
 In addition to these, ANSI color escapes can be insterted into the
 prompts, as \C_ColorName. The list of valid color names is: Black, Blue,
 Brown, Cyan, DarkGray, Green, LightBlue, LightCyan, LightGray,
 LightGreen, LightPurple, LightRed, NoColor, Normal, Purple, Red, White,
 Yellow.
 
 Finally, IPython supports the evaluation of arbitrary expressions in
 your prompt string. The prompt strings are evaluated through the syntax
 of PEP 215, but basically you can use $x.y to expand the value of x.y,
 and for more complicated expressions you can use braces: ${foo()+x} will
 call function foo and add to it the value of x, before putting the
 result into your prompt. For example, using
 prompt_in1 '${commands.getoutput("uptime")}\nIn [\#]: ' 
 will print the result of the uptime command on each prompt (assuming the
 commands module has been imported in your ipythonrc file).
 
 
       Prompt examples
 
 The following options in an ipythonrc file will give you IPython's
 default prompts::
 
     prompt_in1 'In [\#]:' 
     prompt_in2 '   .\D.:' 
     prompt_out 'Out[\#]:'
 
 which look like this::
 
     In [1]: 1+2 
     Out[1]: 3
 
     In [2]: for i in (1,2,3): 
        ...:    print i, 
        ...: 
     1 2 3
 
 These will give you a very colorful prompt with path information::
 
     #prompt_in1 '\C_Red\u\C_Blue[\C_Cyan\Y1\C_Blue]\C_LightGreen\#>' 
     prompt_in2 ' ..\D>' 
     prompt_out '<\#>'
 
 which look like this::
 
     fperez[~/ipython]1> 1+2 
                     <1> 3 
     fperez[~/ipython]2> for i in (1,2,3): 
                    ...>     print i, 
                    ...> 
     1 2 3
 
 
 .. _Profiles:
 
 IPython profiles
 ================
 
 As we already mentioned, IPython supports the -profile command-line option (see
 :ref:`here <command_line_options>`).  A profile is nothing more than a
 particular configuration file like your basic ipythonrc one, but with
 particular customizations for a specific purpose. When you start IPython with
 'ipython -profile <name>', it assumes that in your IPYTHONDIR there is a file
 called ipythonrc-<name> or ipy_profile_<name>.py, and loads it instead of the
 normal ipythonrc.
 
 This system allows you to maintain multiple configurations which load
 modules, set options, define functions, etc. suitable for different
 tasks and activate them in a very simple manner. In order to avoid
 having to repeat all of your basic options (common things that don't
 change such as your color preferences, for example), any profile can
 include another configuration file. The most common way to use profiles
 is then to have each one include your basic ipythonrc file as a starting
-point, and then add further customizations.
  No newline at end of file
+point, and then add further customizations.
+

docs/source/config/index.txt

@@ -1,10 +1,11 b''
 ===============================
 Configuration and customization
 ===============================
 
 .. toctree::
    :maxdepth: 2
 
    initial_config.txt
    customization.txt
    new_config.txt
+

docs/source/config/initial_config.txt

@@ -1,249 +1,250 b''
 .. _initial config:
 
 =========================================
 Initial configuration of your environment
 =========================================
 
 This section will help you set various things in your environment for
 your IPython sessions to be as efficient as possible. All of IPython's
 configuration information, along with several example files, is stored
 in a directory named by default $HOME/.ipython. You can change this by
 defining the environment variable IPYTHONDIR, or at runtime with the
 command line option -ipythondir.
 
 If all goes well, the first time you run IPython it should automatically create
 a user copy of the config directory for you, based on its builtin defaults. You
 can look at the files it creates to learn more about configuring the
 system. The main file you will modify to configure IPython's behavior is called
 ipythonrc (with a .ini extension under Windows), included for reference
 :ref:`here <ipythonrc>`. This file is very commented and has many variables you
 can change to suit your taste, you can find more details :ref:`here
 <customization>`. Here we discuss the basic things you will want to make sure
 things are working properly from the beginning.
 
 
 .. _accessing_help:
 
 Access to the Python help system
 ================================
 
 This is true for Python in general (not just for IPython): you should have an
 environment variable called PYTHONDOCS pointing to the directory where your
 HTML Python documentation lives. In my system it's
 :file:`/usr/share/doc/python-doc/html`, check your local details or ask your
 systems administrator.
 
 This is the directory which holds the HTML version of the Python
 manuals. Unfortunately it seems that different Linux distributions
 package these files differently, so you may have to look around a bit.
 Below I show the contents of this directory on my system for reference::
 
     [html]> ls
     about.html  dist/  icons/      lib/           python2.5.devhelp.gz  whatsnew/
     acks.html   doc/   index.html  mac/           ref/
     api/        ext/   inst/       modindex.html  tut/
 
 You should really make sure this variable is correctly set so that
 Python's pydoc-based help system works. It is a powerful and convenient
 system with full access to the Python manuals and all modules accessible
 to you.
 
 Under Windows it seems that pydoc finds the documentation automatically,
 so no extra setup appears necessary.
 
 
 Editor
 ======
 
 The %edit command (and its alias %ed) will invoke the editor set in your
 environment as EDITOR. If this variable is not set, it will default to
 vi under Linux/Unix and to notepad under Windows. You may want to set
 this variable properly and to a lightweight editor which doesn't take
 too long to start (that is, something other than a new instance of
 Emacs). This way you can edit multi-line code quickly and with the power
 of a real editor right inside IPython.
 
 If you are a dedicated Emacs user, you should set up the Emacs server so
 that new requests are handled by the original process. This means that
 almost no time is spent in handling the request (assuming an Emacs
 process is already running). For this to work, you need to set your
 EDITOR environment variable to 'emacsclient'. The code below, supplied
 by Francois Pinard, can then be used in your .emacs file to enable the
 server::
 
     (defvar server-buffer-clients)
     (when (and (fboundp 'server-start) (string-equal (getenv "TERM") 'xterm))
       (server-start)
       (defun fp-kill-server-with-buffer-routine ()
         (and server-buffer-clients (server-done)))
       (add-hook 'kill-buffer-hook 'fp-kill-server-with-buffer-routine))
 
 You can also set the value of this editor via the commmand-line option
 '-editor' or in your ipythonrc file. This is useful if you wish to use
 specifically for IPython an editor different from your typical default
 (and for Windows users who tend to use fewer environment variables).
 
 
 Color
 =====
 
 The default IPython configuration has most bells and whistles turned on
 (they're pretty safe). But there's one that may cause problems on some
 systems: the use of color on screen for displaying information. This is
 very useful, since IPython can show prompts and exception tracebacks
 with various colors, display syntax-highlighted source code, and in
 general make it easier to visually parse information.
 
 The following terminals seem to handle the color sequences fine:
 
     * Linux main text console, KDE Konsole, Gnome Terminal, E-term,
       rxvt, xterm.
     * CDE terminal (tested under Solaris). This one boldfaces light colors.
     * (X)Emacs buffers. See the emacs_ section for more details on
       using IPython with (X)Emacs.
     * A Windows (XP/2k) command prompt with pyreadline_.
     * A Windows (XP/2k) CygWin shell. Although some users have reported
       problems; it is not clear whether there is an issue for everyone
       or only under specific configurations. If you have full color
       support under cygwin, please post to the IPython mailing list so
       this issue can be resolved for all users.
 
 .. _pyreadline: https://code.launchpad.net/pyreadline
       
 These have shown problems:
 
     * Windows command prompt in WinXP/2k logged into a Linux machine via
       telnet or ssh.
     * Windows native command prompt in WinXP/2k, without Gary Bishop's
       extensions. Once Gary's readline library is installed, the normal
       WinXP/2k command prompt works perfectly.
 
 Currently the following color schemes are available:
 
     * NoColor: uses no color escapes at all (all escapes are empty '' ''
       strings). This 'scheme' is thus fully safe to use in any terminal.
     * Linux: works well in Linux console type environments: dark
       background with light fonts. It uses bright colors for
       information, so it is difficult to read if you have a light
       colored background.
     * LightBG: the basic colors are similar to those in the Linux scheme
       but darker. It is easy to read in terminals with light backgrounds.
 
 IPython uses colors for two main groups of things: prompts and
 tracebacks which are directly printed to the terminal, and the object
 introspection system which passes large sets of data through a pager.
 
 
 Input/Output prompts and exception tracebacks
 =============================================
 
 You can test whether the colored prompts and tracebacks work on your
 system interactively by typing '%colors Linux' at the prompt (use
 '%colors LightBG' if your terminal has a light background). If the input
 prompt shows garbage like::
 
     [0;32mIn [[1;32m1[0;32m]: [0;00m
 
 instead of (in color) something like::
 
     In [1]:
 
 this means that your terminal doesn't properly handle color escape
 sequences. You can go to a 'no color' mode by typing '%colors NoColor'.
 
 You can try using a different terminal emulator program (Emacs users,
 see below). To permanently set your color preferences, edit the file
 $HOME/.ipython/ipythonrc and set the colors option to the desired value.
 
 
 Object details (types, docstrings, source code, etc.)
 =====================================================
 
 IPython has a set of special functions for studying the objects you are working
 with, discussed in detail :ref:`here <dynamic_object_info>`. But this system
 relies on passing information which is longer than your screen through a data
 pager, such as the common Unix less and more programs. In order to be able to
 see this information in color, your pager needs to be properly configured. I
 strongly recommend using less instead of more, as it seems that more simply can
 not understand colored text correctly.
 
 In order to configure less as your default pager, do the following:
 
    1. Set the environment PAGER variable to less.
    2. Set the environment LESS variable to -r (plus any other options
       you always want to pass to less by default). This tells less to
       properly interpret control sequences, which is how color
       information is given to your terminal.
 
 For the bash shell, add to your ~/.bashrc file the lines::
 
     export PAGER=less
     export LESS=-r
 
 For the csh or tcsh shells, add to your ~/.cshrc file the lines::
 
     setenv PAGER less
     setenv LESS -r
     
 There is similar syntax for other Unix shells, look at your system
 documentation for details.
 
 If you are on a system which lacks proper data pagers (such as Windows),
 IPython will use a very limited builtin pager.
 
 .. _emacs:
 
 (X)Emacs configuration
 ======================
 
 Thanks to the work of Alexander Schmolck and Prabhu Ramachandran,
 currently (X)Emacs and IPython get along very well.
 
 Important note: You will need to use a recent enough version of
 python-mode.el, along with the file ipython.el. You can check that the
 version you have of python-mode.el is new enough by either looking at
 the revision number in the file itself, or asking for it in (X)Emacs via
 M-x py-version. Versions 4.68 and newer contain the necessary fixes for
 proper IPython support.
 
 The file ipython.el is included with the IPython distribution, in the
 documentation directory (where this manual resides in PDF and HTML
 formats).
 
 Once you put these files in your Emacs path, all you need in your .emacs
 file is::
 
     (require 'ipython)
 
 This should give you full support for executing code snippets via
 IPython, opening IPython as your Python shell via ``C-c !``, etc.
 
 You can customize the arguments passed to the IPython instance at startup by
 setting the ``py-python-command-args`` variable.  For example, to start always
 in ``pylab`` mode with hardcoded light-background colors, you can use::
 
     (setq py-python-command-args '("-pylab" "-colors" "LightBG"))
 
 If you happen to get garbage instead of colored prompts as described in
 the previous section, you may need to set also in your .emacs file::
 
     (setq ansi-color-for-comint-mode t)
 
 Notes:
 
     * There is one caveat you should be aware of: you must start the
       IPython shell before attempting to execute any code regions via
       ``C-c |``. Simply type C-c ! to start IPython before passing any code
       regions to the interpreter, and you shouldn't experience any
       problems.
       This is due to a bug in Python itself, which has been fixed for
       Python 2.3, but exists as of Python 2.2.2 (reported as SF bug [
       737947 ]).
     * The (X)Emacs support is maintained by Alexander Schmolck, so all
       comments/requests should be directed to him through the IPython
       mailing lists.
     * This code is still somewhat experimental so it's a bit rough
       around the edges (although in practice, it works quite well).
     * Be aware that if you customize py-python-command previously, this
       value will override what ipython.el does (because loading the
       customization variables comes later).
+

docs/source/config/new_config.txt

@@ -1,27 +1,28 b''
 ========================
 New configuration system
 ========================
 
 IPython has a configuration system. When running IPython for the first time,
 reasonable defaults are used for the configuration. The configuration of IPython
 can be changed in two ways:
 
 	* Configuration files
 	* Commands line options (which override the configuration files)
 
 IPython has a separate configuration file for each subpackage. Thus, the main
 configuration files are (in your ``~/.ipython`` directory):
 
  	* ``ipython1.core.ini``
 	* ``ipython1.kernel.ini``
 	* ``ipython1.notebook.ini``
 
 To create these files for the first time, do the following::
 
 	from IPython.kernel.config import config_manager as kernel_config
 	kernel_config.write_default_config_file()
 
 But, you should only need to do this if you need to modify the defaults. If needed
 repeat this process with the ``notebook`` and ``core`` configuration as well. If you
 are running into problems with IPython, you might try deleting these configuration
-files.
  No newline at end of file
+files.
+

docs/source/development/coding_guide.txt

@@ -1,141 +1,142 b''
 ==============
  Coding guide
 ==============
 
 
 Coding conventions
 ==================
 
 In general, we'll try to follow the standard Python style conventions as
 described in Python's `PEP 8`_, the official Python Style Guide.
 
 .. _PEP 8: http://www.python.org/peps/pep-0008.html
 
 Other comments:
 
 - In a large file, top level classes and functions should be separated by 2-3
   lines to make it easier to separate them visually.
 
 - Use 4 spaces for indentation, *never* use hard tabs.
 
 - Keep the ordering of methods the same in classes that have the same methods.
   This is particularly true for classes that implement similar interfaces and
   for interfaces that are similar.
 
 Naming conventions
 ------------------
 
 In terms of naming conventions, we'll follow the guidelines of PEP 8.  Some of
 the existing code doesn't honor this perfectly, but for all new IPython code
 (and much existing code is being refactored), we'll use:
 
 - All ``lowercase`` module names.
 
 - ``CamelCase`` for class names.
 
 - ``lowercase_with_underscores`` for methods, functions, variables and
   attributes.
 
 This may be confusing as some of the existing codebase uses a different
 convention (``lowerCamelCase`` for methods and attributes).  Slowly, we will
 move IPython over to the new convention, providing shadow names for backward
 compatibility in public interfaces.
 
 There are, however, some important exceptions to these rules.  In some cases,
 IPython code will interface with packages (Twisted, Wx, Qt) that use other
 conventions.  At some level this makes it impossible to adhere to our own
 standards at all times.  In particular, when subclassing classes that use other
 naming conventions, you must follow their naming conventions.  To deal with
 cases like this, we propose the following policy:
 
 - If you are subclassing a class that uses different conventions, use its
   naming conventions throughout your subclass.  Thus, if you are creating a
   Twisted Protocol class, used Twisted's
   ``namingSchemeForMethodsAndAttributes.``
 
 - All IPython's official interfaces should use our conventions.  In some cases
   this will mean that you need to provide shadow names (first implement
   ``fooBar`` and then ``foo_bar = fooBar``).  We want to avoid this at all
   costs, but it will probably be necessary at times.  But, please use this
   sparingly!
 
 Implementation-specific *private* methods will use
 ``_single_underscore_prefix``.  Names with a leading double underscore will
 *only* be used in special cases, as they makes subclassing difficult (such
 names are not easily seen by child classes).
 
 Occasionally some run-in lowercase names are used, but mostly for very short
 names or where we are implementing methods very similar to existing ones in a
 base class (like ``runlines()`` where ``runsource()`` and ``runcode()`` had
 established precedent).
 
 The old IPython codebase has a big mix of classes and modules prefixed with an
 explicit ``IP``. In Python this is mostly unnecessary, redundant and frowned
 upon, as namespaces offer cleaner prefixing. The only case where this approach
 is justified is for classes which are expected to be imported into external
 namespaces and a very generic name (like Shell) is too likely to clash with
 something else.  We'll need to revisit this issue as we clean up and refactor
 the code, but in general we should remove as many unnecessary ``IP``/``ip``
 prefixes as possible. However, if a prefix seems absolutely necessary the more
 specific ``IPY`` or ``ipy`` are preferred.
 
 
 .. _devel-testing:
 
 Testing system
 ==============
 
 It is extremely important that all code contributed to IPython has tests. Tests
 should be written as unittests, doctests or as entities that the `Nose`_
 testing package will find. Regardless of how the tests are written, we will use
 `Nose`_ for discovering and running the tests. `Nose`_ will be required to run
 the IPython test suite, but will not be required to simply use IPython.
 
 .. _Nose: http://code.google.com/p/python-nose/
 
 Tests of `Twisted`__ using code should be written by subclassing the
 ``TestCase`` class that comes with ``twisted.trial.unittest``. When this is
 done, `Nose`_ will be able to run the tests and the twisted reactor will be
 handled correctly.
 
 .. __: http://www.twistedmatrix.com
 
 Each subpackage in IPython should have its own ``tests`` directory that
 contains all of the tests for that subpackage. This allows each subpackage to
 be self-contained. If a subpackage has any dependencies beyond the Python
 standard library, the tests for that subpackage should be skipped if the
 dependencies are not found. This is very important so users don't get tests
 failing simply because they don't have dependencies.
 
 We also need to look into use Noses ability to tag tests to allow a more
 modular approach of running tests.
 
 .. _devel-config:
 
 Configuration system
 ====================
 
 IPython uses `.ini`_ files for configuration purposes. This represents a huge
 improvement over the configuration system used in IPython. IPython works with
 these files using the `ConfigObj`_ package, which IPython includes as
 ``ipython1/external/configobj.py``.
 
 Currently, we are using raw `ConfigObj`_ objects themselves. Each subpackage of
 IPython should contain a ``config`` subdirectory that contains all of the
 configuration information for the subpackage. To see how configuration
 information is defined (along with defaults) see at the examples in
 ``ipython1/kernel/config`` and ``ipython1/core/config``. Likewise, to see how
 the configuration information is used, see examples in
 ``ipython1/kernel/scripts/ipengine.py``.
 
 Eventually, we will add a new layer on top of the raw `ConfigObj`_ objects. We
 are calling this new layer, ``tconfig``, as it will use a `Traits`_-like
 validation model.  We won't actually use `Traits`_, but will implement
 something similar in pure Python.  But, even in this new system, we will still
 use `ConfigObj`_ and `.ini`_ files underneath the hood. Talk to Fernando if you
 are interested in working on this part of IPython. The current prototype of
 ``tconfig`` is located in the IPython sandbox.
 
 .. _.ini: http://docs.python.org/lib/module-ConfigParser.html
 .. _ConfigObj: http://www.voidspace.org.uk/python/configobj.html
 .. _Traits: http://code.enthought.com/traits/
+

docs/source/development/config_blueprint.txt

@@ -1,33 +1,34 b''
 =========================================
 Notes on the IPython configuration system
 =========================================
 
 This document has some random notes on the configuration system.
 
 To start, an IPython process needs:
 
 * Configuration files
 * Command line options
 * Additional files (FURL files, extra scripts, etc.)
 
 It feeds these things into the core logic of the process, and as output,
 produces:
 
 * Log files
 * Security files
 
 There are a number of things that complicate this:
 
 * A process may need to be started on a different host that doesn't have
   any of the config files or additional files.  Those files need to be
   moved over and put in a staging area.  The process then needs to be told
   about them.
 * The location of the output files should somehow be set by config files or
   command line options.
 * Our config files are very hierarchical, but command line options are flat,
   making it difficult to relate command line options to config files.
 * Some processes (like ipcluster and the daemons) have to manage the input and
   output files for multiple different subprocesses, each possibly on a
   different host.  Ahhhh!
 * Our configurations are not singletons.  A given user will likely have
   many different configurations for different clusters.
+

docs/source/development/doc_guide.txt

@@ -1,103 +1,104 b''
 .. _documenting-ipython:
 
 =====================
  Documenting IPython
 =====================
 
 Standalone documentation
 ========================
 
 All standalone documentation should be written in plain text (``.txt``) files
 using `reStructuredText`_ for markup and formatting. All such documentation
 should be placed in the top level directory ``docs`` of the IPython source
 tree. Or, when appropriate, a suitably named subdirectory should be used. The
 documentation in this location will serve as the main source for IPython
 documentation and all existing documentation should be converted to this
 format.
 
 The actual HTML and PDF docs are built using the Sphinx_ documentation
 generation tool.  Sphinx has been adopted as the default documentation tool for
 Python itself as of version 2.6, as well as by a number of projects that
 IPython is related with, such as numpy, scipy, matplotlib, sage and nipy.
 
 .. _reStructuredText: http://docutils.sourceforge.net/rst.html
 .. _Sphinx: http://sphinx.pocoo.org/
 
 
 The rest of this document is mostly taken from the `matploblib
 documentation`__; we are using a number of Sphinx tools and extensions written
 by the matplotlib team and will mostly follow their conventions, which are
 nicely spelled out in their guide.  What follows is thus a lightly adapted
 version of the matplotlib documentation guide, taken with permission from the
 MPL team.
 
 .. __: http://matplotlib.sourceforge.net/devel/documenting_mpl.html
 
 
 A bit of Python code::
 
     for i in range(10):
         print i,
     print "A big number:",2**34
 
 An interactive Python session::
 
     >>> from IPython import genutils
     >>> genutils.get_ipython_dir()
     '/home/fperez/.ipython'
 
 
 An IPython session:
 
 .. code-block:: ipython
 
   In [7]: import IPython
 
   In [8]: print "This IPython is version:",IPython.__version__
   This IPython is version: 0.9.1
 
   In [9]: 2+4
   Out[9]: 6
 
 
 A bit of shell code:
 
 .. code-block:: bash
 
     cd /tmp
     echo "My home directory is: $HOME"
     ls
 
   
 Docstring format
 ================
 
 Good docstrings are very important.  Unfortunately, Python itself only provides
 a rather loose standard for docstrings (`PEP 257`_), and there is no universally
 accepted convention for all the different parts of a complete docstring.
 However, the NumPy project has established a very reasonable standard, and has
 developed some tools to support the smooth inclusion of such docstrings in
 Sphinx-generated manuals.  Rather than inventing yet another pseudo-standard,
 IPython will be henceforth documented using the NumPy conventions; we carry
 copies of some of the NumPy support tools to remain self-contained, but share
 back upstream with NumPy any improvements or fixes we may make to the tools.
 
 The `NumPy documentation guidelines`_ contain detailed information on this
 standard, and for a quick overview, the NumPy `example docstring`_ is a useful
 read. 
 
 As in the past IPython used epydoc, currently many docstrings still use epydoc
 conventions.  We will update them as we go, but all new code should be fully
 documented using the NumPy standard.
 
 .. _PEP 257: http://www.python.org/peps/pep-0257.html
 .. _NumPy documentation guidelines: http://projects.scipy.org/numpy/wiki/CodingStyleGuidelines
 
 .. _example docstring: http://projects.scipy.org/numpy/browser/trunk/doc/EXAMPLE_DOCSTRING.txt
 
 Additional PEPs of interest regarding documentation of code.  While both of
 these were rejected, the ideas therein form much of the basis of docutils (the
 machinery to process reStructuredText):
 
 - `Docstring Processing System Framework <http://www.python.org/peps/pep-0256.html>`_
 - `Docutils Design Specification <http://www.python.org/peps/pep-0258.html>`_
+

docs/source/development/index.txt

@@ -1,15 +1,16 b''
 ===========================
  IPython Developer's Guide
 ===========================
 
 .. toctree::
    :maxdepth: 2
 
    overview.txt
    coding_guide.txt
    doc_guide.txt
    roadmap.txt
 
    notification_blueprint.txt
    config_blueprint.txt
    reorg.txt
+

docs/source/development/notification_blueprint.txt

@@ -1,83 +1,108 b''
 .. _notification:
 
 ==========================================
 IPython.kernel.core.notification blueprint
 ==========================================
 
 Overview
 ========
 
 The :mod:`IPython.kernel.core.notification` module will provide a simple
 implementation of a notification center and support for the observer pattern
 within the :mod:`IPython.kernel.core`. The main intended use case is to
 provide notification of Interpreter events to an observing frontend during the
 execution of a single block of code.
 
 Functional Requirements
 =======================
 
 The notification center must:
 
 * Provide synchronous notification of events to all registered observers.
 
 * Provide typed or labeled notification types.
 
 * Allow observers to register callbacks for individual or all notification
   types.
 
 * Allow observers to register callbacks for events from individual or all
   notifying objects.
 
 * Notification to the observer consists of the notification type, notifying
   object and user-supplied extra information [implementation: as keyword
   parameters to the registered callback].
 
 * Perform as O(1) in the case of no registered observers.
 
 * Permit out-of-process or cross-network extension.
     
 What's not included
 ===================
 
 As written, the :mod:`IPython.kernel.core.notificaiton` module does not:
 
 * Provide out-of-process or network notifications (these should be handled by
   a separate, Twisted aware module in :mod:`IPython.kernel`).
 
 * Provide zope.interface-style interfaces for the notification system (these
   should also be provided by the :mod:`IPython.kernel` module).
     
 Use Cases
 =========
 
 The following use cases describe the main intended uses of the notificaiton module and illustrate the main success scenario for each use case:
 
     1. Dwight Schroot is writing a frontend for the IPython project. His frontend is stuck in the stone age and must communicate synchronously with an IPython.kernel.core.Interpreter instance. Because code is executed in blocks by the Interpreter, Dwight's UI freezes every time he executes a long block of code. To keep track of the progress of his long running block, Dwight adds the following code to his frontend's set-up code::
 
         from IPython.kernel.core.notification import NotificationCenter
         center = NotificationCenter.sharedNotificationCenter
         center.registerObserver(self, type=IPython.kernel.core.Interpreter.STDOUT_NOTIFICATION_TYPE, notifying_object=self.interpreter, callback=self.stdout_notification)
     
     and elsewhere in his front end::
 
         def stdout_notification(self, type, notifying_object, out_string=None):
             self.writeStdOut(out_string)
     
 If everything works, the Interpreter will (according to its published API)
 fire a notification via the
 :data:`IPython.kernel.core.notification.sharedCenter` of type
 :const:`STD_OUT_NOTIFICATION_TYPE` before writing anything to stdout [it's up
 to the Intereter implementation to figure out when to do this]. The
 notificaiton center will then call the registered callbacks for that event
 type (in this case, Dwight's frontend's stdout_notification method). Again,
 according to its API, the Interpreter provides an additional keyword argument
 when firing the notificaiton of out_string, a copy of the string it will write
 to stdout.
 
 Like magic, Dwight's frontend is able to provide output, even during
 long-running calculations. Now if Jim could just convince Dwight to use
 Twisted...
 
-    2. Boss Hog is writing a frontend for the IPython project. Because Boss Hog is stuck in the stone age, his frontend will be written in a new Fortran-like dialect of python and will run only from the command line. Because he doesn't need any fancy notification system and is used to worrying about every cycle on his rat-wheel powered mini, Boss Hog is adamant that the new notification system not produce any performance penalty. As they say in Hazard county, there's no such thing as a free lunch. If he wanted zero overhead, he should have kept using IPython 0.8. Instead, those tricky Duke boys slide in a suped-up bridge-out jumpin' awkwardly confederate-lovin' notification module that imparts only a constant (and small) performance penalty when the Interpreter (or any other object) fires an event for which there are no registered observers. Of course, the same notificaiton-enabled Interpreter can then be used in frontends that require notifications, thus saving the IPython project from a nasty civil war.
+2. Boss Hog is writing a frontend for the IPython project. Because Boss Hog is
+stuck in the stone age, his frontend will be written in a new Fortran-like
+dialect of python and will run only from the command line. Because he doesn't
+need any fancy notification system and is used to worrying about every cycle
+on his rat-wheel powered mini, Boss Hog is adamant that the new notification
+system not produce any performance penalty. As they say in Hazard county,
+there's no such thing as a free lunch. If he wanted zero overhead, he should
+have kept using IPython 0.8. Instead, those tricky Duke boys slide in a
+suped-up bridge-out jumpin' awkwardly confederate-lovin' notification module
+that imparts only a constant (and small) performance penalty when the
+Interpreter (or any other object) fires an event for which there are no
+registered observers. Of course, the same notificaiton-enabled Interpreter can
+then be used in frontends that require notifications, thus saving the IPython
+project from a nasty civil war.
     
-    3. Barry is wrting a frontend for the IPython project. Because Barry's front end is the *new hotness*, it uses an asynchronous event model to communicate with a Twisted :mod:`~IPython.kernel.engineservice` that communicates with the IPython :class:`~IPython.kernel.core.interpreter.Interpreter`. Using the :mod:`IPython.kernel.notification` module, an asynchronous wrapper on the :mod:`IPython.kernel.core.notification` module, Barry's frontend can register for notifications from the interpreter that are delivered asynchronously. Even if Barry's frontend is running on a separate process or even host from the Interpreter, the notifications are delivered, as if by dark and twisted magic. Just like Dwight's frontend, Barry's frontend can now recieve notifications of e.g. writing to stdout/stderr, opening/closing an external file, an exception in the executing code, etc.
  No newline at end of file
+3. Barry is wrting a frontend for the IPython project. Because Barry's front
+end is the *new hotness*, it uses an asynchronous event model to communicate
+with a Twisted :mod:`~IPython.kernel.engineservice` that communicates with the
+IPython :class:`~IPython.kernel.core.interpreter.Interpreter`. Using the
+:mod:`IPython.kernel.notification` module, an asynchronous wrapper on the
+:mod:`IPython.kernel.core.notification` module, Barry's frontend can register
+for notifications from the interpreter that are delivered asynchronously. Even
+if Barry's frontend is running on a separate process or even host from the
+Interpreter, the notifications are delivered, as if by dark and twisted magic.
+Just like Dwight's frontend, Barry's frontend can now recieve notifications of
+e.g. writing to stdout/stderr, opening/closing an external file, an exception
+in the executing code, etc.
+

docs/source/development/overview.txt

@@ -1,517 +1,518 b''
 .. _development:
 
 ==============================
 IPython development guidelines
 ==============================
 
 
 Overview
 ========
 
 This document describes IPython from the perspective of developers. Most
 importantly, it gives information for people who want to contribute to the
 development of IPython. So if you want to help out, read on!
 
 How to contribute to IPython
 ============================
 
 IPython development is done using Bazaar [Bazaar]_ and Launchpad [Launchpad]_.
 This makes it easy for people to contribute to the development of IPython.
 There are several ways in which you can join in.
 
 If you have a small change that you want to send to the team, you can edit your
 bazaar checkout of IPython (see below) in-place, and ask bazaar for the
 differences::
 
     $ cd /path/to/your/copy/of/ipython
     $ bzr diff > my_fixes.diff
 
 This produces a patch file with your fixes, which we can apply to the source
 tree.  This file should then be attached to a ticket in our `bug tracker
 <https://bugs.launchpad.net/ipython>`_, indicating what it does.
 
 This model of creating small, self-contained patches works very well and there
 are open source projects that do their entire development this way.  However,
 in IPython we have found that for tracking larger changes, making use of
 bazaar's full capabilities in conjunction with Launchpad's code hosting
 services makes for a much better experience.
 
 Making your own branch of IPython allows you to refine your changes over time,
 track the development of the main team, and propose your own full version of
 the code for others to use and review, with a minimum amount of fuss.  The next
 parts of this document will explain how to do this.
 
 Install Bazaar and create a Launchpad account
 ---------------------------------------------
 
 First make sure you have installed Bazaar (see their `website
 <http://bazaar-vcs.org/>`_). To see that Bazaar is installed and knows about
 you, try the following::
 
     $ bzr whoami
     Joe Coder <[e-mail]>
 
 This should display your name and email. Next, you will want to create an
 account on the `Launchpad website <http://www.launchpad.net>`_ and setup your
 ssh keys. For more information of setting up your ssh keys, see `this link
 <https://help.launchpad.net/YourAccount/CreatingAnSSHKeyPair>`_.
 
 Get the main IPython branch from Launchpad
 ------------------------------------------
 
 Now, you can get a copy of the main IPython development branch (we call this
 the "trunk")::
 
     $ bzr branch lp:ipython
 
 Create a working branch
 -----------------------
 
 When working on IPython, you won't actually make edits directly to the
 :file:`lp:ipython` branch. Instead, you will create a separate branch for your
 changes. For now, let's assume you want to do your work in a branch named
 "ipython-mybranch". Create this branch by doing::
 
     $ bzr branch ipython ipython-mybranch
 
 When you actually create a branch, you will want to give it a name that
 reflects the nature of the work that you will be doing in it, like
 "install-docs-update".
 
 Make edits in your working branch
 ---------------------------------
 
 Now you are ready to actually make edits in your :file:`ipython-mybranch`
 branch. Before doing this, it is helpful to install this branch so you can
 test your changes as you work. This is easiest if you have setuptools
 installed. Then, just do::
 
     $ cd ipython-mybranch
     $ python setupegg.py develop
 
 Now, make some changes. After a while, you will want to commit your changes.
 This let's Bazaar know that you like the changes you have made and gives you
 an opportunity to keep a nice record of what you have done. This looks like
 this::
 
     $ ...do work in ipython-mybranch...
     $ bzr commit -m "the commit message goes here"
 
 Please note that since we now don't use an old-style linear ChangeLog (that
 tends to cause problems with distributed version control systems), you should
 ensure that your log messages are reasonably detailed.  Use a docstring-like
 approach in the commit messages (including the second line being left
 *blank*)::
 
   Single line summary of  changes being committed.
 
   * more details when warranted ...
   * including crediting outside contributors if they sent the
     code/bug/idea!
 
 As you work, you will repeat this edit/commit cycle many times. If you work on
 your branch for a long time, you will also want to get the latest changes from
 the :file:`lp:ipython` branch. This can be done with the following sequence of
 commands::
 
     $ ls
     ipython
     ipython-mybranch
     
     $ cd ipython
     $ bzr pull
     $ cd ../ipython-mybranch
     $ bzr merge ../ipython
     $ bzr commit -m "Merging changes from trunk"
 
 Along the way, you should also run the IPython test suite.  You can do this
 using the :command:`iptest` command (which is basically a customized version of
 :command:`nosetests`)::
 
     $ cd
     $ iptest
 
 The :command:`iptest` command will also pick up and run any tests you have
 written. See :ref:`testing documentation <devel_testing>` for further details
 on the testing system.
 
 
 Post your branch and request a code review
 ------------------------------------------
 
 Once you are done with your edits, you should post your branch on Launchpad so
 that other IPython developers can review the changes and help you merge your
 changes into the main development branch. To post your branch on Launchpad,
 do::
 
     $ cd ipython-mybranch
     $ bzr push lp:~yourusername/ipython/ipython-mybranch
 
 Then, go to the `IPython Launchpad site <www.launchpad.net/ipython>`_, and you
 should see your branch under the "Code" tab. If you click on your branch, you
 can provide a short description of the branch as well as mark its status. Most
 importantly, you should click the link that reads "Propose for merging into
 another branch". What does this do?
 
 This let's the other IPython developers know that your branch is ready to be
 reviewed and merged into the main development branch. During this review
 process, other developers will give you feedback and help you get your code
 ready to be merged. What types of things will we be looking for:
 
 * All code is documented.
 * All code has tests.
 * The entire IPython test suite passes.
 
 Once your changes have been reviewed and approved, someone will merge them
 into the main development branch.
 
 
 Some notes for core developers when merging third-party contributions
 =====================================================================
 
 Core developers, who ultimately merge any approved branch (from themselves,
 another developer, or any third-party contribution) will typically use
 :command:`bzr merge` to merge the branch into the trunk and push it to the
 main Launcphad site. This is a short list of things to keep in mind when doing
 this process, so that the project history is easy to understand in the long
 run, and that generating release notes is as painless and accurate as
 possible.
 
 - When you merge any non-trivial functionality (from one small bug fix to a
   big feature branch), please remember to always edit the :file:`changes.txt`
   file accordingly. This file has one main section for each release, and if
   you edit it as you go, noting what new features, bug fixes or API changes
   you have made, the release notes will be almost finished when they are
   needed later. This is much easier if done when you merge the work, rather
   than weeks or months later by re-reading a massive Bazaar log.
 
 - When big merges are done, the practice of putting a summary commit message
   in the merge is *extremely* useful. It makes this kind of job much nicer,
   because that summary log message can be almost copy/pasted without changes,
   if it was well written, rather than dissecting the next-level messages from
   the individual commits.
 
 - It's important that we remember to always credit who gave us something if
   it's not the committer. In general, we have been fairly good on this front,
   this is just a reminder to keep things up. As a note, if you are ever
   committing something that is completely (or almost so) a third-party
   contribution, do the commit as::
 
     $ bzr commit --author="Someone Else"
 
   This way it will show that name separately in the log, which makes it even
   easier to spot. Obviously we often rework third party contributions
   extensively, but this is still good to keep in mind for cases when we don't
   touch the code too much.
 
 
 Documentation
 =============
 
 Standalone documentation
 ------------------------
 
 All standalone documentation should be written in plain text (``.txt``) files
 using reStructuredText [reStructuredText]_ for markup and formatting. All such
 documentation should be placed in directory :file:`docs/source` of the IPython
 source tree. The documentation in this location will serve as the main source
 for IPython documentation and all existing documentation should be converted
 to this format.
 
 To build the final documentation, we use Sphinx [Sphinx]_.  Once you have
 Sphinx installed, you can build the html docs yourself by doing::
 
     $ cd ipython-mybranch/docs
     $ make html
 
 Docstring format
 ----------------
 
 Good docstrings are very important. All new code should have docstrings that
 are formatted using reStructuredText for markup and formatting, since it is
 understood by a wide variety of tools. Details about using reStructuredText
 for docstrings can be found `here
 <http://epydoc.sourceforge.net/manual-othermarkup.html>`_.
 
 Additional PEPs of interest regarding documentation of code:
 
 * `Docstring Conventions <http://www.python.org/peps/pep-0257.html>`_
 * `Docstring Processing System Framework <http://www.python.org/peps/pep-0256.html>`_
 * `Docutils Design Specification <http://www.python.org/peps/pep-0258.html>`_
 
 
 Coding conventions
 ==================
 
 General
 -------
 
 In general, we'll try to follow the standard Python style conventions as
 described here:
 
 * `Style Guide for Python Code  <http://www.python.org/peps/pep-0008.html>`_  
 
 
 Other comments:
 
 * In a large file, top level classes and functions should be 
   separated by 2-3 lines to make it easier to separate them visually.
 * Use 4 spaces for indentation.
 * Keep the ordering of methods the same in classes that have the same 
   methods.  This is particularly true for classes that implement an interface.
 
 Naming conventions
 ------------------
 
 In terms of naming conventions, we'll follow the guidelines from the `Style
 Guide for Python Code`_.
 
 For all new IPython code (and much existing code is being refactored), we'll
 use:
 
 * All ``lowercase`` module names.
 
 * ``CamelCase`` for class names.
 
 * ``lowercase_with_underscores`` for methods, functions, variables and
   attributes.
 
 There are, however, some important exceptions to these rules. In some cases,
 IPython code will interface with packages (Twisted, Wx, Qt) that use other
 conventions. At some level this makes it impossible to adhere to our own
 standards at all times. In particular, when subclassing classes that use other
 naming conventions, you must follow their naming conventions. To deal with
 cases like this, we propose the following policy:
 
 * If you are subclassing a class that uses different conventions, use its
   naming conventions throughout your subclass.  Thus, if you are creating a
   Twisted Protocol class, used Twisted's
   ``namingSchemeForMethodsAndAttributes.``
 
 * All IPython's official interfaces should use our conventions.  In some cases
   this will mean that you need to provide shadow names (first implement
   ``fooBar`` and then ``foo_bar = fooBar``).  We want to avoid this at all
   costs, but it will probably be necessary at times.  But, please use this
   sparingly!
 
 Implementation-specific *private* methods will use
 ``_single_underscore_prefix``. Names with a leading double underscore will
 *only* be used in special cases, as they makes subclassing difficult (such
 names are not easily seen by child classes).
 
 Occasionally some run-in lowercase names are used, but mostly for very short
 names or where we are implementing methods very similar to existing ones in a
 base class (like ``runlines()`` where ``runsource()`` and ``runcode()`` had
 established precedent).
 
 The old IPython codebase has a big mix of classes and modules prefixed with an
 explicit ``IP``. In Python this is mostly unnecessary, redundant and frowned
 upon, as namespaces offer cleaner prefixing. The only case where this approach
 is justified is for classes which are expected to be imported into external
 namespaces and a very generic name (like Shell) is too likely to clash with
 something else. We'll need to revisit this issue as we clean up and refactor
 the code, but in general we should remove as many unnecessary ``IP``/``ip``
 prefixes as possible. However, if a prefix seems absolutely necessary the more
 specific ``IPY`` or ``ipy`` are preferred.
 
 .. _devel_testing:
 
 Testing system
 ==============
 
 It is extremely important that all code contributed to IPython has tests.
 Tests should be written as unittests, doctests or as entities that the Nose
 [Nose]_ testing package will find. Regardless of how the tests are written, we
 will use Nose for discovering and running the tests. Nose will be required to
 run the IPython test suite, but will not be required to simply use IPython.
 
 Tests of Twisted using code need to follow two additional guidelines:
 
 1. Twisted using tests should be written by subclassing the :class:`TestCase`
    class that comes with :mod:`twisted.trial.unittest`. 
 
 2. All :class:`Deferred` instances that are created in the test must be
    properly chained and the final one *must* be the return value of the test
    method.
 
 When these two things are done, Nose will be able to run the tests and the
 twisted reactor will be handled correctly.
 
 Each subpackage in IPython should have its own :file:`tests` directory that
 contains all of the tests for that subpackage. This allows each subpackage to
 be self-contained.  A good convention to follow is to have a file named
 :file:`test_foo.py` for each module :file:`foo.py` in the package.  This makes
 it easy to organize the tests, though like most conventions, it's OK to break
 it if logic and common sense dictate otherwise.
 
 If a subpackage has any dependencies beyond the Python standard library, the
 tests for that subpackage should be skipped if the dependencies are not
 found. This is very important so users don't get tests failing simply because
 they don't have dependencies.  We ship a set of decorators in the
 :mod:`IPython.testing` package to tag tests that may be platform-specific or
 otherwise may have restrictions; if the existing ones don't fit your needs, add
 a new decorator in that location so other tests can reuse it.
 
 To run the IPython test suite, use the :command:`iptest` command that is
 installed with IPython (if you are using IPython in-place, without installing
 it, you can find this script in the :file:`scripts` directory)::
 
     $ iptest
 
 This command colects all IPython tests into separate groups, and then calls
 either Nose with the proper options and extensions, or Twisted's
 :command:`trial`.  This ensures that tests that need the Twisted reactor
 management facilities execute separate of Nose.  If any individual test group
 fails, :command:`iptest` will print what you need to type so you can rerun that
 particular test group alone for debugging.
 
 By default, :command:`iptest` runs the entire IPython test
 suite (skipping tests that may be platform-specific or which depend on tools
 you may not have).  But you can also use it to run only one specific test file,
 or a specific test function.  For example, this will run only the
 :file:`test_magic` file from the test suite::
 
     $ iptest IPython.tests.test_magic
     ----------------------------------------------------------------------
     Ran 10 tests in 0.348s
 
     OK (SKIP=3)
     Deleting object: second_pass
 
 while the ``path:function`` syntax allows you to select a specific function in
 that file to run::
 
     $ iptest IPython.tests.test_magic:test_obj_del
     ----------------------------------------------------------------------
     Ran 1 test in 0.204s
 
     OK
 
 Since :command:`iptest` is based on nosetests, you can pass it any regular
 nosetests option.  For example, you can use ``--pdb`` or ``--pdb-failures`` to
 automatically activate the interactive Pdb debugger on errors or failures.  See
 the nosetests documentation for further details.
 
 
 A few tips for writing tests
 ----------------------------
 
 You can write tests either as normal test files, using all the conventions that
 Nose recognizes, or as doctests.  Note that *all* IPython functions should have
 at least one example that serves as a doctest, whenever technically feasible.
 However, example doctests should only be in the main docstring if they are *a
 good example*, i.e. if they convey useful information about the function.  If
 you simply would like to write a test as a doctest, put it in a separate test
 file and write a no-op function whose only purpose is its docstring.
 
 Note, however, that in a file named :file:`test_X`, functions whose only test
 is their docstring (as a doctest) and which have no test functionality of their
 own, should be called *doctest_foo* instead of *test_foo*, otherwise they get
 double-counted (the empty function call is counted as a test, which just
 inflates tests numbers artificially).  This restriction does not apply to
 functions in files with other names, due to how Nose discovers tests.
 
 You can use IPython examples in your docstrings.  Those can make full use of
 IPython functionality (magics, variable substitution, etc), but be careful to
 keep them generic enough that they run identically on all Operating Systems.
 
 The prompts in your doctests can be either of the plain Python ``>>>`` variety
 or ``In [1]:`` IPython style.  Since this is the IPython system, after all, we
 encourage you to use IPython prompts throughout, unless you are illustrating a
 specific aspect of the normal prompts (such as the ``%doctest_mode`` magic).
 
 If a test isn't safe to run inside the main nose process (e.g. because it loads
 a GUI toolkit), consider running it in a subprocess and capturing its output
 for evaluation and test decision later.  Here is an example of how to do it, by
 relying on the builtin ``_ip`` object that contains the public IPython api as
 defined in :mod:`IPython.ipapi`::
 
    def test_obj_del():
        """Test that object's __del__ methods are called on exit."""
        test_dir = os.path.dirname(__file__)
        del_file = os.path.join(test_dir,'obj_del.py')
        out = _ip.IP.getoutput('ipython %s' % del_file)
        nt.assert_equals(out,'object A deleted')
 
 
 
 If a doctest contains input whose output you don't want to verify identically
 via doctest (random output, an object id, etc), you can mark a docstring with
 ``#random``.  All of these test will have their code executed but no output
 checking will be done::
 
        >>> 1+3
        junk goes here...  # random
 
        >>> 1+2
        again,  anything goes #random
        if multiline, the random mark is only needed once.
 
        >>> 1+2
        You can also put the random marker at the end:
        # random
 
        >>> 1+2
        # random
        .. or at the beginning.
 
 In a case where you want an *entire* docstring to be executed but not verified
 (this only serves to check that the code runs without crashing, so it should be
 used very sparingly), you can put ``# all-random`` in the docstring.
        
 .. _devel_config:
 
 Release checklist
 =================
 
 Most of the release process is automated by the :file:`release` script in the
 :file:`tools` directory.  This is just a handy reminder for the release manager.
 
 #. First, run :file:`build_release`, which does all the file checking and
    building that the real release script will do.  This will let you do test
    installations, check that the build procedure runs OK, etc.  You may want to
    disable a few things like multi-version RPM building while testing, because
    otherwise the build takes really long.
 
 #. Run the release script, which makes the tar.gz, eggs and Win32 .exe
    installer.  It posts them to the site and registers the release with PyPI.
 
 #. Updating the website with announcements and links to the updated
    changes.txt in html form. Remember to put a short note both on the news
    page of the site and on Launcphad.
 
 #. Drafting a short release announcement with i) highlights and ii) a link to
    the html changes.txt.
 
 #. Make sure that the released version of the docs is live on the site.
 
 #. Celebrate!
 
 Porting to 3.0
 ==============
 
 There are no definite plans for porting of IPython to python 3. The major 
 issue is the dependency on twisted framework for the networking/threading 
 stuff. It is possible that it the traditional IPython interactive console 
 could be ported more easily since it has no such dependency. Here are a few
 things that will need to be considered when doing such a port especially
 if we want to have a codebase that works directly on both 2.x and 3.x.
 
 	1. The syntax for exceptions changed (PEP 3110). The old 
 	   `except exc, var` changed to `except exc as var`. At last
 	   count there was 78 occurences of this usage in the codebase.  This
 	   is  a particularly problematic issue, because it's not easy to
 	   implement it in a 2.5-compatible way.
 
 Because it is quite difficult to support simultaneously Python 2.5 and 3.x, we
 will likely at some point put out a release that requires strictly 2.6 and
 abandons 2.5 compatibility.  This will then allow us to port the code to using
 :func:`print` as a function, `except exc as var` syntax, etc.  But as of
 version 0.11 at least, we will retain Python 2.5 compatibility.
 
 
 .. [Bazaar] Bazaar. http://bazaar-vcs.org/
 .. [Launchpad] Launchpad. http://www.launchpad.net/ipython
 .. [reStructuredText] reStructuredText.  http://docutils.sourceforge.net/rst.html
 .. [Sphinx] Sphinx. http://sphinx.pocoo.org/
 .. [Nose] Nose: a discovery based unittest extension. http://code.google.com/p/python-nose/
+

docs/source/development/reorg.txt

@@ -1,84 +1,85 b''
 =============================
 IPython module reorganization
 =============================
 
 Currently, IPython has many top-level modules that serve many different
 purposes. The lack of organization make it very difficult for developers to
 work on IPython and understand its design. This document contains notes about
 how we will reorganize the modules into sub-packages.
 
 .. warning::
 
     This effort will possibly break third party packages that use IPython as
     a library or hack on the IPython internals.
 
 .. warning::
 
     This effort will result in the removal from IPython of certain modules
     that are not used anymore, don't currently work, are unmaintained, etc.
 
 
 Current subpackges
 ==================
 
 IPython currently has the following sub-packages:
 
 * :mod:`IPython.config`
 
 * :mod:`IPython.Extensions`
 
 * :mod:`IPython.external`
 
 * :mod:`IPython.frontend`
 
 * :mod:`IPython.gui`
 
 * :mod:`IPython.kernel`
 
 * :mod:`IPython.testing`
 
 * :mod:`IPython.tests`
 
 * :mod:`IPython.tools`
 
 * :mod:`IPython.UserConfig`
 
 New Subpackages to be created
 =============================
 
 We propose to create the following new sub-packages:
 
 * :mod:`IPython.core`. This sub-package will contain the core of the IPython
   interpreter, but none of its extended capabilities.
 
 * :mod:`IPython.lib`. IPython has many extended capabilities that are not part
   of the IPython core. These things will go here.
 
 * :mod:`IPython.utils`. This sub-package will contain anything that might
   eventually be found in the Python standard library, like things in
   :mod:`genutils`. Each sub-module in this sub-package should contain
   functions and classes that serve a single purpose.
 
 * :mod:`IPython.deathrow`. This is for code that is untested and/or rotting
   and needs to be removed from IPython. Eventually all this code will either
   i) be revived by someone willing to maintain it with tests and docs and
   re-included into IPython or 2) be removed from IPython proper, but put into
   a separate top-level (not IPython) package that we keep around. No new code
   will be allowed here.
 
 * :mod:`IPython.quarantine`.  This is for code that doesn't meet IPython's 
   standards, but that we plan on keeping.  To be moved out of this sub-package
   a module needs to have a maintainer, tests and documentation.
 
 Procedure
 =========
 
 1. Move the file to its new location with its new name.
 2. Rename all import statements to reflect the change.
 3. Run PyFlakes on each changes module.
 4. Add tests/test_imports.py to test it.
 
 Status
 ======
 
 This branch was merged into trunk in early August of 2009.
+

docs/source/development/roadmap.txt

@@ -1,96 +1,94 b''
 .. _roadmap:
 
 ===================
 Development roadmap
 ===================
 
 IPython is an ambitious project that is still under heavy development.
 However, we want IPython to become useful to as many people as possible, as
 quickly as possible. To help us accomplish this, we are laying out a roadmap
 of where we are headed and what needs to happen to get there. Hopefully, this
 will help the IPython developers figure out the best things to work on for
 each upcoming release.
 
 Work targeted to particular releases
 ====================================
 
 Release 0.11
 ------------
 
 * Full module and package reorganization (done).
 
 * Removal of the threaded shells and new implementation of GUI support
   based on ``PyOSInputHook`` (done).
 
 * Refactor the configuration system (done).
 
 * Prepare to refactor IPython's core by creating a new component and 
   application system (done).
 
 * Start to refactor IPython's core by turning everything into components
   (started).
 
 Release 0.12
 ------------
 
 * Continue to refactor IPython's core by turning everything into components.
 
 
 Major areas of work
 ===================
 
 Refactoring the main IPython core
 ---------------------------------
 
 During the summer of 2009, we began refactoring IPython's core.  The main
 thrust in this work was make the IPython core into a set of loosely coupled
 components.  The base component class for this is 
 :class:`IPython.core.component.Component`.  This section outlines the status
 of this work.
 
 Parts of the IPython core that have been turned into components:
 
 * The main :class:`InteractiveShell` class.
 * The aliases (:mod:`IPython.core.aliases`).
 * The display and builtin traps (:mod:`IPython.core.display_trap` and
   :mod:`IPython.core.builtin_trap`).
 * The prefilter machinery (:mod:`IPython.core.prefilter`).
 
 Parts of the IPythoncore that need to be turned into components:
 
 * Magics.
 * Input and output history management.
 * Prompts.
 * Completers.
 * Logging.
 * Exception handling.
 * Anything else.
 
 Process management for :mod:`IPython.kernel`
 --------------------------------------------
 
 Performance problems
 --------------------
 
 Currently, we have a number of performance issues in :mod:`IPython.kernel`:
 
 * The controller stores a large amount of state in Python dictionaries. Under
   heavy usage, these dicts with get very large, causing memory usage problems.
   We need to develop more scalable solutions to this problem. This will also
   help the controller to be more fault tolerant.
 
 * We currently don't have a good way of handling large objects in the
   controller. The biggest problem is that because we don't have any way of
   streaming objects, we get lots of temporary copies in the low-level buffers.
   We need to implement a better serialization approach and true streaming
   support.
 
 * The controller currently unpickles and repickles objects. We need to use the
   [push|pull]_serialized methods instead.
 
 * Currently the controller is a bottleneck. The best approach for this is to
   separate the controller itself into multiple processes, one for the core
   controller and one each for the controller interfaces.
 
-
-

docs/source/faq.txt

@@ -1,105 +1,96 b''
 .. _faq:
 
 ========================================
 Frequently asked questions
 ========================================
 
 General questions
 =================
 
 Questions about parallel computing with IPython
 ================================================
 
 Will IPython speed my Python code up?
 --------------------------------------
 
 Yes and no. When converting a serial code to run in parallel, there often many
 difficulty questions that need to be answered, such as:
 
 * How should data be decomposed onto the set of processors?
 
 * What are the data movement patterns?
 
 * Can the algorithm be structured to minimize data movement?
 
 * Is dynamic load balancing important?
 
 We can't answer such questions for you. This is the hard (but fun) work of parallel
 computing. But, once you understand these things IPython will make it easier for you to
 implement a good solution quickly. Most importantly, you will be able to use the
 resulting parallel code interactively.
 
 With that said, if your problem is trivial to parallelize, IPython has a number of
 different interfaces that will enable you to parallelize things is almost no time at
 all.  A good place to start is the ``map`` method of our :class:`MultiEngineClient`.
 
 What is the best way to use MPI from Python?
 --------------------------------------------
 
 What about all the other parallel computing packages in Python?
 ---------------------------------------------------------------
 
 Some of the unique characteristic of IPython are:
 
 * IPython is the only architecture that abstracts out the notion of a 
   parallel computation in such a way that new models of parallel computing
   can be explored quickly and easily.  If you don't like the models we
   provide, you can simply create your own using the capabilities we provide.
   
 * IPython is asynchronous from the ground up (we use `Twisted`_).
 
 * IPython's architecture is designed to avoid subtle problems 
   that emerge because of Python's global interpreter lock (GIL).
   
 * While IPython's architecture is designed to support a wide range
   of novel parallel computing models, it is fully interoperable with
   traditional MPI applications.
   
 * IPython has been used and tested extensively on modern supercomputers.
 
 * IPython's networking layers are completely modular.  Thus, is 
   straightforward to replace our existing network protocols with 
   high performance alternatives (ones based upon Myranet/Infiniband).
   
 * IPython is designed from the ground up to support collaborative
   parallel computing.  This enables multiple users to actively develop
   and run the *same* parallel computation.
   
 * Interactivity is a central goal for us.  While IPython does not have
   to be used interactivly, it can be.
       
 .. _Twisted: http://www.twistedmatrix.com
 
 Why The IPython controller a bottleneck in my parallel calculation?
 -------------------------------------------------------------------
 
 A golden rule in parallel computing is that you should only move data around if you
 absolutely need to. The main reason that the controller becomes a bottleneck is that
 too much data is being pushed and pulled to and from the engines. If your algorithm
 is structured in this way, you really should think about alternative ways of
 handling the data movement. Here are some ideas:
 
 1. Have the engines write data to files on the locals disks of the engines.
 
 2. Have the engines write data to files on a file system that is shared by
    the engines.
 
 3. Have the engines write data to a database that is shared by the engines.
 
 4. Simply keep data in the persistent memory of the engines and move the
    computation to the data (rather than the data to the computation).
 
 5. See if you can pass data directly between engines using MPI.
 
 Isn't Python slow to be used for high-performance parallel computing?
 ---------------------------------------------------------------------
 
-
-
-
- 
-
-
-
-
-

docs/source/index.txt

@@ -1,31 +1,34 b''
 =====================
 IPython Documentation
 =====================
 
 .. htmlonly::
 
    :Release: |release|
    :Date: |today|
 
-   Contents:
+Welcome to the official IPython documentation.  This document describes the
+various parts of IPython that are relevant to both users and developers.
+
+Contents
+========
 
 .. toctree::
-   :maxdepth: 2
+   :maxdepth: 1
 
    overview.txt
+   whatsnew/index.txt
    install/index.txt
    interactive/index.txt
    parallel/index.txt
    config/index.txt
-   faq.txt
-   history.txt
-   changes.txt
    development/index.txt
    api/index.txt
-   license_and_copyright.txt
-   credits.txt
+   faq.txt
+   about/index.txt
 
 .. htmlonly::
    * :ref:`genindex`
    * :ref:`modindex`
    * :ref:`search`
+

docs/source/install/index.txt

@@ -1,10 +1,11 b''
 .. _install_index:
 
 ============
 Installation
 ============
 
 .. toctree::
    :maxdepth: 2
 
    install.txt
+

docs/source/install/install.txt

@@ -1,293 +1,294 b''
 Overview
 ========
 
 This document describes the steps required to install IPython.  IPython is
 organized into a number of subpackages, each of which has its own dependencies.
 All of the subpackages come with IPython, so you don't need to download and
 install them separately.  However, to use a given subpackage, you will need to
 install all of its dependencies.
 
 
 Please let us know if you have problems installing IPython or any of its
 dependencies. Officially, IPython requires Python version 2.5 or 2.6.  We
 have *not* yet started to port IPython to Python 3.0.
 
 .. warning::
 
     Officially, IPython supports Python versions 2.5 and 2.6.
 
     IPython 0.10 has only been well tested with Python 2.5 and 2.6.  Parts of
     it may work with Python 2.4, but we do not officially support Python 2.4
     anymore.  If you need to use 2.4, you can still run IPython 0.9.
 
 Some of the installation approaches use the :mod:`setuptools` package and its
 :command:`easy_install` command line program.  In many scenarios, this provides
 the most simple method of installing IPython and its dependencies.  It is not
 required though.  More information about :mod:`setuptools` can be found on its
 website.
 
 More general information about installing Python packages can be found in
 Python's documentation at http://www.python.org/doc/.
 
 Quickstart
 ==========
 
 If you have :mod:`setuptools` installed and you are on OS X or Linux (not
 Windows), the following will download and install IPython *and* the main
 optional dependencies::
 
     $ easy_install ipython[kernel,security,test]
 
 This will get Twisted, zope.interface and Foolscap, which are needed for
 IPython's parallel computing features as well as the nose package, which will
 enable you to run IPython's test suite.  To run IPython's test suite, use the
 :command:`iptest` command::
 
     $ iptest
 
 Read on for more specific details and instructions for Windows.
 
 Installing IPython itself
 =========================
 
 Given a properly built Python, the basic interactive IPython shell will work
 with no external dependencies.  However, some Python distributions
 (particularly on Windows and OS X), don't come with a working :mod:`readline`
 module.  The IPython shell will work without :mod:`readline`, but will lack
 many features that users depend on, such as tab completion and command line
 editing.  See below for details of how to make sure you have a working
 :mod:`readline`.
 
 Installation using easy_install
 -------------------------------
 
 If you have :mod:`setuptools` installed, the easiest way of getting IPython is
 to simple use :command:`easy_install`::
 
 	$ easy_install ipython
 
 That's it.
 
 Installation from source
 ------------------------
 
 If you don't want to use :command:`easy_install`, or don't have it installed,
 just grab the latest stable build of IPython from `here
 <http://ipython.scipy.org/dist/>`_.  Then do the following::
 
 	$ tar -xzf ipython.tar.gz
 	$ cd ipython
 	$ python setup.py install
 
 If you are installing to a location (like ``/usr/local``) that requires higher
 permissions, you may need to run the last command with :command:`sudo`.
 
 Windows
 -------
 
 There are a few caveats for Windows users.  The main issue is that a basic
 ``python setup.py install`` approach won't create ``.bat`` file or Start Menu
 shortcuts, which most users want.  To get an installation with these, you can
 use any of the following alternatives:
 
 1. Install using :command:`easy_install`.
 
 2. Install using our binary ``.exe`` Windows installer, which can be found at
    `here <http://ipython.scipy.org/dist/>`_
 
 3. Install from source, but using :mod:`setuptools` (``python setupegg.py
    install``).
 
 IPython by default runs in a termninal window, but the normal terminal
 application supplied by Microsoft Windows is very primitive.  You may want to
 download the excellent and free Console_ application instead, which is a far
 superior tool.  You can even configure Console to give you by default an
 IPython tab, which is very convenient to create new IPython sessions directly
 from the working terminal.
 
 .. _Console:  http://sourceforge.net/projects/console
 
    
 Installing the development version
 ----------------------------------
 
 It is also possible to install the development version of IPython from our
 `Bazaar <http://bazaar-vcs.org/>`_ source code repository.  To do this you will
 need to have Bazaar installed on your system.  Then just do::
 
 	$ bzr branch lp:ipython
 	$ cd ipython
 	$ python setup.py install
 
 Again, this last step on Windows won't create ``.bat`` files or Start Menu
 shortcuts, so you will have to use one of the other approaches listed above.
 
 Some users want to be able to follow the development branch as it changes.  If
 you have :mod:`setuptools` installed, this is easy. Simply replace the last
 step by::
 
 	$ python setupegg.py develop
 
 This creates links in the right places and installs the command line script to
 the appropriate places.  Then, if you want to update your IPython at any time,
 just do::
 
 	$ bzr pull
 
 Basic optional dependencies
 ===========================
 
 There are a number of basic optional dependencies that most users will want to
 get.  These are:
 
 * readline (for command line editing, tab completion, etc.)
 * nose (to run the IPython test suite)
 * pexpect (to use things like irunner)
 
 If you are comfortable installing these things yourself, have at it, otherwise
 read on for more details.
 
 readline
 --------
 
 In principle, all Python distributions should come with a working
 :mod:`readline` module.  But, reality is not quite that simple.  There are two
 common situations where you won't have a working :mod:`readline` module:
 
 * If you are using the built-in Python on Mac OS X.
 
 * If you are running Windows, which doesn't have a :mod:`readline` module.
 
 On OS X, the built-in Python doesn't not have :mod:`readline` because of
 license issues.  Starting with OS X 10.5 (Leopard), Apple's built-in Python has
 a BSD-licensed not-quite-compatible readline replacement. As of IPython 0.9,
 many of the issues related to the differences between readline and libedit have
 been resolved.  For many users, libedit may be sufficient.
 
 Most users on OS X will want to get the full :mod:`readline` module.  To get a
 working :mod:`readline` module, just do (with :mod:`setuptools` installed)::
 
 	$ easy_install readline
 
 .. note:
 
     Other Python distributions on OS X (such as fink, MacPorts and the 
     official python.org binaries) already have readline installed so 
     you don't have to do this step.
 
 If needed, the readline egg can be build and installed from source (see the
 wiki page at http://ipython.scipy.org/moin/InstallationOSXLeopard).
 
 On Windows, you will need the PyReadline module. PyReadline is a separate,
 Windows only implementation of readline that uses native Windows calls through
 :mod:`ctypes`. The easiest way of installing PyReadline is you use the binary
 installer available `here <http://ipython.scipy.org/dist/>`_. The :mod:`ctypes`
 module, which comes with Python 2.5 and greater, is required by PyReadline. It
 is available for Python 2.4 at http://python.net/crew/theller/ctypes.
 
 nose
 ----
 
 To run the IPython test suite you will need the :mod:`nose` package.  Nose
 provides a great way of sniffing out and running all of the IPython tests.  The
 simplest way of getting nose, is to use :command:`easy_install`::
 
 	$ easy_install nose
 
 Another way of getting this is to do::
 
     $ easy_install ipython[test]
 
 For more installation options, see the `nose website
 <http://somethingaboutorange.com/mrl/projects/nose/>`_.  Once you have nose
 installed, you can run IPython's test suite using the iptest command::
 
 	$ iptest
 
 
 pexpect
 -------
 
 The `pexpect <http://www.noah.org/wiki/Pexpect>`_ package is used in IPython's
 :command:`irunner` script.  On Unix platforms (including OS X), just do::
 
 	$ easy_install pexpect
 
 Windows users are out of luck as pexpect does not run there.
 
 Dependencies for IPython.kernel (parallel computing)
 ====================================================
 
 The IPython kernel provides a nice architecture for parallel computing.  The
 main focus of this architecture is on interactive parallel computing.  These
 features require a number of additional packages:
 
 * zope.interface (yep, we use interfaces)
 * Twisted (asynchronous networking framework)
 * Foolscap (a nice, secure network protocol)
 * pyOpenSSL (security for network connections)
 
 On a Unix style platform (including OS X), if you want to use :mod:`setuptools`, you can just do::
 
     $ easy_install ipython[kernel]    # the first three
     $ easy_install ipython[security]  # pyOpenSSL
 
 zope.interface and Twisted
 --------------------------
 
 Twisted [Twisted]_ and zope.interface [ZopeInterface]_ are used for networking
 related things. On Unix style platforms (including OS X), the simplest way of
 getting the these is to use :command:`easy_install`::
 
 	$ easy_install zope.interface
 	$ easy_install Twisted
 
 Of course, you can also download the source tarballs from the `Twisted website
 <twistedmatrix.org>`_ and the `zope.interface page at PyPI
 <http://pypi.python.org/pypi/zope.interface>`_ and do the usual ``python
 setup.py install`` if you prefer.
 
 Windows is a bit different.  For zope.interface and Twisted, simply get the latest binary ``.exe`` installer from the Twisted website.  This installer includes both zope.interface and Twisted and should just work.
 
 Foolscap
 --------
 
 Foolscap [Foolscap]_ uses Twisted to provide a very nice secure RPC protocol that we use to implement our parallel computing features.
 
 On all platforms a simple::
 
     $ easy_install foolscap
 
 should work.  You can also download the source tarballs from the `Foolscap
 website <http://foolscap.lothar.com/trac>`_ and do ``python setup.py install``
 if you prefer.
 
 pyOpenSSL
 ---------
 
 IPython requires an older version of pyOpenSSL [pyOpenSSL]_ (0.6 rather than
 the current 0.7).  There are a couple of options for getting this:
 
 1.  Most Linux distributions have packages for pyOpenSSL.
 2.  The built-in Python 2.5 on OS X 10.5 already has it installed.
 3.  There are source tarballs on the pyOpenSSL website.  On Unix-like
     platforms, these can be built using ``python seutp.py install``.
 4.  There is also a binary ``.exe`` Windows installer on the `pyOpenSSL website <http://pyopenssl.sourceforge.net/>`_.
 
 Dependencies for IPython.frontend (the IPython GUI)
 ===================================================
 
 wxPython
 --------
 
 Starting with IPython 0.9, IPython has a new IPython.frontend package that has
 a nice wxPython based IPython GUI.  As you would expect, this GUI requires
 wxPython.  Most Linux distributions have wxPython packages available and the
 built-in Python on OS X comes with wxPython preinstalled.  For Windows, a
 binary installer is available on the `wxPython website
 <http://www.wxpython.org/>`_.
 
 .. [Twisted] Twisted matrix.  http://twistedmatrix.org
 .. [ZopeInterface] http://pypi.python.org/pypi/zope.interface
 .. [Foolscap] Foolscap network protocol.  http://foolscap.lothar.com/trac
 .. [pyOpenSSL] pyOpenSSL.  http://pyopenssl.sourceforge.net
+

docs/source/interactive/extension_api.txt

@@ -1,252 +1,253 b''
 =====================
 IPython extension API
 =====================
 
 IPython api (defined in IPython/ipapi.py) is the public api that
 should be used for
 
  * Configuration of user preferences (.ipython/ipy_user_conf.py)
  * Creating new profiles (.ipython/ipy_profile_PROFILENAME.py)
  * Writing extensions 
 
 Note that by using the extension api for configuration (editing
 ipy_user_conf.py instead of ipythonrc), you get better validity checks
 and get richer functionality - for example, you can import an
 extension and call functions in it to configure it for your purposes.
 
 For an example extension (the 'sh' profile), see
 IPython/extensions/ipy_profile_sh.py.
 
 For the last word on what's available, see the source code of
 IPython/ipapi.py.
 
 
 Getting started
 ===============
 
 If you want to define an extension, create a normal python module that
 can be imported. The module will access IPython functionality through
 the 'ip' object defined below.
 
 If you are creating a new profile (e.g. foobar), name the module as
 'ipy_profile_foobar.py' and put it in your ~/.ipython directory. Then,
 when you start ipython with the '-p foobar' argument, the module is
 automatically imported on ipython startup.
 
 If you are just doing some per-user configuration, you can either 
 
  * Put the commands directly into ipy_user_conf.py. 
 
  * Create a new module with your customization code and import *that*
    module in ipy_user_conf.py. This is preferable to the first approach,
    because now you can reuse and distribute your customization code.
 
 Getting a handle to the api
 ===========================
 
 Put this in the start of your module::
 
     #!python
     import IPython.ipapi
     ip = IPython.ipapi.get()
 
 The 'ip' object will then be used for accessing IPython
 functionality. 'ip' will mean this api object in all the following
 code snippets. The same 'ip' that we just acquired is always
 accessible in interactive IPython sessions by the name _ip - play with
 it like this::
 
     [~\_ipython]|81> a = 10
     [~\_ipython]|82> _ip.e
     _ip.ev           _ip.ex           _ip.expose_magic
     [~\_ipython]|82> _ip.ev('a+13')
                 <82> 23
 
 The _ip object is also used in some examples in this document - it can
 be substituted by 'ip' in non-interactive use.
 
 Changing options
 ================
 
 The ip object has 'options' attribute that can be used te get/set
 configuration options (just as in the ipythonrc file)::
 
     o = ip.options
     o.autocall = 2
     o.automagic = 1
 
 Executing statements in IPython namespace with 'ex' and 'ev'
 ============================================================
 
 Often, you want to e.g. import some module or define something that
 should be visible in IPython namespace. Use ``ip.ev`` to
 *evaluate* (calculate the value of) expression and ``ip.ex`` to
 '''execute''' a statement::
 
     # path module will be visible to the interactive session
     ip.ex("from path import path" )
 
     # define a handy function 'up' that changes the working directory
 
     ip.ex('import os')
     ip.ex("def up(): os.chdir('..')")
 
 
     # _i2 has the input history entry #2, print its value in uppercase.
     print ip.ev('_i2.upper()')
 
 Accessing the IPython namespace
 ===============================
 
 ip.user_ns attribute has a dictionary containing the IPython global
 namespace (the namespace visible in the interactive session).
 
 ::
 
     [~\_ipython]|84> tauno = 555
     [~\_ipython]|85> _ip.user_ns['tauno']
                 <85> 555
 
 Defining new magic commands
 ===========================
 
 The following example defines a new magic command, %impall. What the
 command does should be obvious::
 
     def doimp(self, arg):
         ip = self.api
         ip.ex("import %s; reload(%s); from %s import *" % (
         arg,arg,arg)
         )
 
     ip.expose_magic('impall', doimp)
 
 Things to observe in this example:
 
  * Define a function that implements the magic command using the 
    ipapi methods defined in this document
  * The first argument of the function is 'self', i.e. the 
    interpreter object. It shouldn't be used directly. however. 
    The interpreter object is probably *not* going to remain stable 
    through IPython versions.
  * Access the ipapi through 'self.api' instead of the global 'ip' object.
  * All the text following the magic command on the command line is 
    contained in the second argument
  * Expose the magic by ip.expose_magic()
 
 
 Calling magic functions and system commands
 ===========================================
 
 Use ip.magic() to execute a magic function, and ip.system() to execute
 a system command::
 
     # go to a bookmark
     ip.magic('%cd -b relfiles')
 
     # execute 'ls -F' system command. Interchangeable with os.system('ls'), really.
     ip.system('ls -F')
 
 Launching IPython instance from normal python code
 ==================================================
 
 Use ipapi.launch_new_instance() with an argument that specifies the
 namespace to use. This can be useful for trivially embedding IPython
 into your program. Here's an example of normal python program test.py
 ('''without''' an existing IPython session) that launches an IPython
 interpreter and regains control when the interpreter is exited::
 
     [ipython]|1> cat test.py
     my_ns = dict(
         kissa = 15,
         koira = 16)
     import IPython.ipapi
     print "launching IPython instance"
     IPython.ipapi.launch_new_instance(my_ns)
     print "Exited IPython instance!"
     print "New vals:",my_ns['kissa'], my_ns['koira']
 
 And here's what it looks like when run (note how we don't start it
 from an ipython session)::
 
     Q:\ipython>python test.py
     launching IPython instance
     Py 2.5 (r25:51908, Sep 19 2006, 09:52:17) [MSC v.1310 32 bit (Intel)] IPy 0.7.3b3.r1975
     [ipython]|1> kissa = 444
     [ipython]|2> koira = 555
     [ipython]|3> Exit
     Exited IPython instance!
     New vals: 444 555
 
 Accessing unexposed functionality
 =================================
 
 There are still many features that are not exposed via the ipapi. If
 you can't avoid using them, you can use the functionality in
 InteractiveShell object (central IPython session class, defined in
 iplib.py) through ip.IP.
 
 For example::
 
     [~]|7> _ip.IP.expand_aliases('np','myfile.py')
        <7> 'c:/opt/Notepad++/notepad++.exe myfile.py'
     [~]|8>
 
 Still, it's preferable that if you encounter such a feature, contact
 the IPython team and request that the functionality be exposed in a
 future version of IPython. Things not in ipapi are more likely to
 change over time.
 
 Provided extensions
 ===================
 
 You can see the list of available extensions (and profiles) by doing
 ``import ipy_<TAB>``. Some extensions don't have the ``ipy_`` prefix in
 module name, so you may need to see the contents of IPython/extensions
 folder to see what's available.
 
 You can see a brief documentation of an extension by looking at the
 module docstring::
 
     [c:p/ipython_main]|190> import ipy_fsops
     [c:p/ipython_main]|191> ipy_fsops?
 
     ...
 
     Docstring:
         File system operations
 
     Contains: Simple variants of normal unix shell commands (icp, imv, irm,
     imkdir, igrep).
 
 You can also install your own extensions - the recommended way is to
 just copy the module to ~/.ipython. Extensions are typically enabled
 by just importing them (e.g. in ipy_user_conf.py), but some extensions
 require additional steps, for example::
 
     [c:p]|192> import ipy_traits_completer
     [c:p]|193> ipy_traits_completer.activate()    
 
 Note that extensions, even if provided in the stock IPython
 installation, are not guaranteed to have the same requirements as the
 rest of IPython - an extension may require external libraries or a
 newer version of Python than what IPython officially requires. An
 extension may also be under a more restrictive license than IPython
 (e.g. ipy_bzr is under GPL).
 
 Just for reference, the list of bundled extensions at the time of
 writing is below:
 
 astyle.py clearcmd.py envpersist.py ext_rescapture.py ibrowse.py
 igrid.py InterpreterExec.py InterpreterPasteInput.py ipipe.py
 ipy_app_completers.py ipy_autoreload.py ipy_bzr.py ipy_completers.py
 ipy_constants.py ipy_defaults.py ipy_editors.py ipy_exportdb.py
 ipy_extutil.py ipy_fsops.py ipy_gnuglobal.py ipy_kitcfg.py
 ipy_legacy.py ipy_leo.py ipy_p4.py ipy_profile_doctest.py
 ipy_profile_none.py ipy_profile_scipy.py ipy_profile_sh.py
 ipy_profile_zope.py ipy_pydb.py ipy_rehashdir.py ipy_render.py
 ipy_server.py ipy_signals.py ipy_stock_completers.py
 ipy_system_conf.py ipy_traits_completer.py ipy_vimserver.py
 ipy_which.py ipy_workdir.py jobctrl.py ledit.py numeric_formats.py
 PhysicalQInput.py PhysicalQInteractive.py pickleshare.py
-pspersistence.py win32clip.py __init__.py
  No newline at end of file
+pspersistence.py win32clip.py __init__.py
+

docs/source/interactive/index.txt

@@ -1,11 +1,12 b''
 ==================================
 Using IPython for interactive work
 ==================================
 
 .. toctree::
    :maxdepth: 2
 
    tutorial.txt
    reference.txt
    shell.txt
    extension_api.txt
+

docs/source/interactive/shell.txt

@@ -1,284 +1,285 b''
 .. _ipython_as_shell:
 
 =========================
 IPython as a system shell
 =========================
 
 Overview
 ========
 
 The 'sh' profile optimizes IPython for system shell usage. Apart from
 certain job control functionality that is present in unix (ctrl+z does
 "suspend"), the sh profile should provide you with most of the
 functionality you use daily in system shell, and more. Invoke IPython
 in 'sh' profile by doing 'ipython -p sh', or (in win32) by launching
 the "pysh" shortcut in start menu.
 
 If you want to use the features of sh profile as your defaults (which
 might be a good idea if you use other profiles a lot of the time but
 still want the convenience of sh profile), add ``import ipy_profile_sh``
 to your ~/.ipython/ipy_user_conf.py.
 
 The 'sh' profile is different from the default profile in that:
 
  * Prompt shows the current directory
  * Spacing between prompts and input is more compact (no padding with
    empty lines). The startup banner is more compact as well.
  * System commands are directly available (in alias table) without
    requesting %rehashx - however, if you install new programs along
    your PATH, you might want to run %rehashx to update the persistent
    alias table
  * Macros are stored in raw format by default. That is, instead of
    '_ip.system("cat foo"), the macro will contain text 'cat foo')
  * Autocall is in full mode
  * Calling "up" does "cd .."
 
 The 'sh' profile is different from the now-obsolete (and unavailable)
 'pysh' profile in that:
 
  * '$$var = command' and '$var = command' syntax is not supported
  * anymore. Use 'var = !command' instead (incidentally, this is
  * available in all IPython profiles). Note that !!command *will*
  * work.
 
 Aliases
 =======
 
 All of your $PATH has been loaded as IPython aliases, so you should be
 able to type any normal system command and have it executed. See
 %alias?  and %unalias? for details on the alias facilities. See also
 %rehashx? for details on the mechanism used to load $PATH.
 
 
 Directory management
 ====================
 
 Since each command passed by ipython to the underlying system is executed
 in a subshell which exits immediately, you can NOT use !cd to navigate
 the filesystem.
 
 IPython provides its own builtin '%cd' magic command to move in the
 filesystem (the % is not required with automagic on). It also maintains
 a list of visited directories (use %dhist to see it) and allows direct
 switching to any of them. Type 'cd?' for more details.
 
 %pushd, %popd and %dirs are provided for directory stack handling.
 
 
 Enabled extensions
 ==================
 
 Some extensions, listed below, are enabled as default in this profile.
 
 envpersist
 ----------
 
 %env can be used to "remember" environment variable manipulations. Examples::
 
 	%env - Show all environment variables
 	%env VISUAL=jed  - set VISUAL to jed
 	%env PATH+=;/foo - append ;foo to PATH
 	%env PATH+=;/bar - also append ;bar to PATH
 	%env PATH-=/wbin; - prepend /wbin; to PATH
 	%env -d VISUAL   - forget VISUAL persistent val
 	%env -p          - print all persistent env modifications
 
 ipy_which
 ---------
 
 %which magic command. Like 'which' in unix, but knows about ipython aliases.
 
 Example::
 
  [C:/ipython]|14> %which st
  st -> start .
  [C:/ipython]|15> %which d
  d -> dir /w /og /on
  [C:/ipython]|16> %which cp
  cp -> cp
    == c:\bin\cp.exe
  c:\bin\cp.exe
 
 ipy_app_completers
 ------------------
 
 Custom tab completers for some apps like svn, hg, bzr, apt-get. Try 'apt-get install <TAB>' in debian/ubuntu.
 
 ipy_rehashdir
 -------------
 
 Allows you to add system command aliases for commands that are not along your path. Let's say that you just installed Putty and want to be able to invoke it without adding it to path, you can create the alias for it with rehashdir::
 
  [~]|22> cd c:/opt/PuTTY/
  [c:opt/PuTTY]|23> rehashdir .
               <23> ['pageant', 'plink', 'pscp', 'psftp', 'putty', 'puttygen', 'unins000']
 
 Now, you can execute any of those commams directly::
 
  [c:opt/PuTTY]|24> cd
  [~]|25> putty
 
 (the putty window opens).
 
 If you want to store the alias so that it will always be available, do '%store putty'. If you want to %store all these aliases persistently, just do it in a for loop::
 
  [~]|27> for a in _23:
     |..>     %store $a
     |..>
     |..>
  Alias stored: pageant (0, 'c:\\opt\\PuTTY\\pageant.exe')
  Alias stored: plink (0, 'c:\\opt\\PuTTY\\plink.exe')
  Alias stored: pscp (0, 'c:\\opt\\PuTTY\\pscp.exe')
  Alias stored: psftp (0, 'c:\\opt\\PuTTY\\psftp.exe')
  ...
 
 mglob
 -----
 
 Provide the magic function %mglob, which makes it easier (than the 'find' command) to collect (possibly recursive) file lists. Examples::
 
  [c:/ipython]|9> mglob *.py
  [c:/ipython]|10> mglob *.py rec:*.txt
  [c:/ipython]|19> workfiles = %mglob !.svn/ !.hg/ !*_Data/ !*.bak rec:.
 
 Note that the first 2 calls will put the file list in result history (_, _9, _10), and the last one will assign it to 'workfiles'.
 
 
 Prompt customization
 ====================
 
 The sh profile uses the following prompt configurations::
 
     o.prompt_in1= r'\C_LightBlue[\C_LightCyan\Y2\C_LightBlue]\C_Green|\#>'
     o.prompt_in2= r'\C_Green|\C_LightGreen\D\C_Green>'
 
 You can change the prompt configuration to your liking by editing
 ipy_user_conf.py.  
 
 String lists
 ============
 
 String lists (IPython.genutils.SList) are handy way to process output
 from system commands. They are produced by ``var = !cmd`` syntax.
 
 First, we acquire the output of 'ls -l'::
 
     [Q:doc/examples]|2> lines = !ls -l
      ==
     ['total 23',
      '-rw-rw-rw- 1 ville None 1163 Sep 30  2006 example-demo.py',
      '-rw-rw-rw- 1 ville None 1927 Sep 30  2006 example-embed-short.py',
      '-rwxrwxrwx 1 ville None 4606 Sep  1 17:15 example-embed.py',
      '-rwxrwxrwx 1 ville None 1017 Sep 30  2006 example-gnuplot.py',
      '-rwxrwxrwx 1 ville None  339 Jun 11 18:01 extension.py',
      '-rwxrwxrwx 1 ville None  113 Dec 20  2006 seteditor.py',
      '-rwxrwxrwx 1 ville None  245 Dec 12  2006 seteditor.pyc']
 
 Now, let's take a look at the contents of 'lines' (the first number is
 the list element number)::
 
     [Q:doc/examples]|3> lines
                     <3> SList (.p, .n, .l, .s, .grep(), .fields() available). Value:
 
     0: total 23
     1: -rw-rw-rw- 1 ville None 1163 Sep 30  2006 example-demo.py
     2: -rw-rw-rw- 1 ville None 1927 Sep 30  2006 example-embed-short.py
     3: -rwxrwxrwx 1 ville None 4606 Sep  1 17:15 example-embed.py
     4: -rwxrwxrwx 1 ville None 1017 Sep 30  2006 example-gnuplot.py
     5: -rwxrwxrwx 1 ville None  339 Jun 11 18:01 extension.py
     6: -rwxrwxrwx 1 ville None  113 Dec 20  2006 seteditor.py
     7: -rwxrwxrwx 1 ville None  245 Dec 12  2006 seteditor.pyc
 
 Now, let's filter out the 'embed' lines::
 
     [Q:doc/examples]|4> l2 = lines.grep('embed',prune=1)
     [Q:doc/examples]|5> l2
                     <5> SList (.p, .n, .l, .s, .grep(), .fields() available). Value:
 
     0: total 23
     1: -rw-rw-rw- 1 ville None 1163 Sep 30  2006 example-demo.py
     2: -rwxrwxrwx 1 ville None 1017 Sep 30  2006 example-gnuplot.py
     3: -rwxrwxrwx 1 ville None  339 Jun 11 18:01 extension.py
     4: -rwxrwxrwx 1 ville None  113 Dec 20  2006 seteditor.py
     5: -rwxrwxrwx 1 ville None  245 Dec 12  2006 seteditor.pyc
 
 Now, we want strings having just file names and permissions::
 
     [Q:doc/examples]|6> l2.fields(8,0)
                     <6> SList (.p, .n, .l, .s, .grep(), .fields() available). Value:
 
     0: total
     1: example-demo.py -rw-rw-rw-
     2: example-gnuplot.py -rwxrwxrwx
     3: extension.py -rwxrwxrwx
     4: seteditor.py -rwxrwxrwx
     5: seteditor.pyc -rwxrwxrwx
 
 Note how the line with 'total' does not raise IndexError.
 
 If you want to split these (yielding lists), call fields() without
 arguments::
 
     [Q:doc/examples]|7> _.fields()
                     <7>
     [['total'],
      ['example-demo.py', '-rw-rw-rw-'],
      ['example-gnuplot.py', '-rwxrwxrwx'],
      ['extension.py', '-rwxrwxrwx'],
      ['seteditor.py', '-rwxrwxrwx'],
      ['seteditor.pyc', '-rwxrwxrwx']]
 
 If you want to pass these separated with spaces to a command (typical
 for lists if files), use the .s property::
 
 
     [Q:doc/examples]|13> files = l2.fields(8).s
     [Q:doc/examples]|14> files
                     <14> 'example-demo.py example-gnuplot.py extension.py seteditor.py seteditor.pyc'
     [Q:doc/examples]|15> ls $files
     example-demo.py  example-gnuplot.py  extension.py  seteditor.py  seteditor.pyc
 
 SLists are inherited from normal python lists, so every list method is
 available::
 
     [Q:doc/examples]|21> lines.append('hey')
 
 
 Real world example: remove all files outside version control
 ============================================================
 
 First, capture output of "hg status"::
 
     [Q:/ipython]|28> out = !hg status
      ==
     ['M IPython\\extensions\\ipy_kitcfg.py',
      'M IPython\\extensions\\ipy_rehashdir.py',
     ...
      '? build\\lib\\IPython\\Debugger.py',
      '? build\\lib\\IPython\\extensions\\InterpreterExec.py',
      '? build\\lib\\IPython\\extensions\\InterpreterPasteInput.py',
     ...
 
 (lines starting with ? are not under version control).
 
 ::
 
     [Q:/ipython]|35> junk = out.grep(r'^\?').fields(1)
     [Q:/ipython]|36> junk
                 <36> SList (.p, .n, .l, .s, .grep(), .fields() availab
     ...
     10: build\bdist.win32\winexe\temp\_ctypes.py
     11: build\bdist.win32\winexe\temp\_hashlib.py
     12: build\bdist.win32\winexe\temp\_socket.py
 
 Now we can just remove these files by doing 'rm $junk.s'. 
 
 The .s, .n, .p properties
 =========================
 
 The '.s' property returns one string where lines are separated by
 single space (for convenient passing to system commands). The '.n'
 property return one string where the lines are separated by '\n'
 (i.e. the original output of the function). If the items in string
 list are file names, '.p' can be used to get a list of "path" objects
-for convenient file manipulation.
  No newline at end of file
+for convenient file manipulation.
+

docs/source/interactive/tutorial.txt

@@ -1,315 +1,317 b''
 .. _tutorial:
 
 ======================
 Quick IPython tutorial
 ======================
 
 IPython can be used as an improved replacement for the Python prompt,
 and for that you don't really need to read any more of this manual. But
 in this section we'll try to summarize a few tips on how to make the
 most effective use of it for everyday Python development, highlighting
 things you might miss in the rest of the manual (which is getting long).
 We'll give references to parts in the manual which provide more detail
 when appropriate.
 
 The following article by Jeremy Jones provides an introductory tutorial
 about IPython: http://www.onlamp.com/pub/a/python/2005/01/27/ipython.html
 
 Highlights
 ==========
 
 Tab completion
 --------------
 
 TAB-completion, especially for attributes, is a convenient way to explore the
 structure of any object you're dealing with. Simply type object_name.<TAB> and
 a list of the object's attributes will be printed (see :ref:`the readline
 section <readline>` for more). Tab completion also works on file and directory
 names, which combined with IPython's alias system allows you to do from within
 IPython many of the things you normally would need the system shell for.
 
 Explore your objects
 --------------------
 
 Typing object_name? will print all sorts of details about any object,
 including docstrings, function definition lines (for call arguments) and
 constructor details for classes. The magic commands %pdoc, %pdef, %psource
 and %pfile will respectively print the docstring, function definition line,
 full source code and the complete file for any object (when they can be
 found). If automagic is on (it is by default), you don't need to type the '%'
 explicitly. See :ref:`this section <dynamic_object_info>` for more.
 
 The `%run` magic command
 ------------------------
 
 The %run magic command allows you to run any python script and load all of its
 data directly into the interactive namespace. Since the file is re-read from
 disk each time, changes you make to it are reflected immediately (in contrast
 to the behavior of import). I rarely use import for code I am testing, relying
 on %run instead. See :ref:`this section <magic>` for more on this and other
 magic commands, or type the name of any magic command and ? to get details on
 it. See also :ref:`this section <dreload>` for a recursive reload command. %run
 also has special flags for timing the execution of your scripts (-t) and for
 executing them under the control of either Python's pdb debugger (-d) or
 profiler (-p). With all of these, %run can be used as the main tool for
 efficient interactive development of code which you write in your editor of
 choice.
 
 Debug a Python script
 ---------------------
 
 Use the Python debugger, pdb. The %pdb command allows you to toggle on and off
 the automatic invocation of an IPython-enhanced pdb debugger (with coloring,
 tab completion and more) at any uncaught exception. The advantage of this is
 that pdb starts inside the function where the exception occurred, with all data
 still available. You can print variables, see code, execute statements and even
 walk up and down the call stack to track down the true source of the problem
 (which often is many layers in the stack above where the exception gets
 triggered). Running programs with %run and pdb active can be an efficient to
 develop and debug code, in many cases eliminating the need for print statements
 or external debugging tools. I often simply put a 1/0 in a place where I want
 to take a look so that pdb gets called, quickly view whatever variables I need
 to or test various pieces of code and then remove the 1/0. Note also that '%run
 -d' activates pdb and automatically sets initial breakpoints for you to step
 through your code, watch variables, etc.  The :ref:`output caching section
 <output_caching>` has more details.
 
 Use the output cache
 --------------------
 
 All output results are automatically stored in a global dictionary named Out
 and variables named _1, _2, etc. alias them. For example, the result of input
 line 4 is available either as Out[4] or as _4. Additionally, three variables
 named _, __ and ___ are always kept updated with the for the last three
 results. This allows you to recall any previous result and further use it for
 new calculations. See :ref:`the output caching section <output_caching>` for
 more.
 
 Suppress output
 ---------------
 
 Put a ';' at the end of a line to suppress the printing of output. This is
 useful when doing calculations which generate long output you are not
 interested in seeing. The _* variables and the Out[] list do get updated with
 the contents of the output, even if it is not printed. You can thus still
 access the generated results this way for further processing.
 
 Input cache
 -----------
 
 A similar system exists for caching input. All input is stored in a global
 list called In , so you can re-execute lines 22 through 28 plus line 34 by
 typing 'exec In[22:29]+In[34]' (using Python slicing notation). If you need
 to execute the same set of lines often, you can assign them to a macro with
 the %macro function. See :ref:`here <input_caching>` for more.
 
 Use your input history
 ----------------------
 
 The %hist command can show you all previous input, without line numbers if
 desired (option -n) so you can directly copy and paste code either back in
 IPython or in a text editor. You can also save all your history by turning on
 logging via %logstart; these logs can later be either reloaded as IPython
 sessions or used as code for your programs.
 
 Define your own system aliases
 ------------------------------
 
 Even though IPython gives you access to your system shell via the ! prefix,
 it is convenient to have aliases to the system commands you use most often.
 This allows you to work seamlessly from inside IPython with the same commands
 you are used to in your system shell. IPython comes with some pre-defined
 aliases and a complete system for changing directories, both via a stack (see
 %pushd, %popd and %dhist) and via direct %cd. The latter keeps a history of
 visited directories and allows you to go to any previously visited one.
 
 Call system shell commands
 --------------------------
 
 Use Python to manipulate the results of system commands. The '!!' special
 syntax, and the %sc and %sx magic commands allow you to capture system output
 into Python variables.
 
 Use Python variables when calling the shell
 -------------------------------------------
 
 Expand python variables when calling the shell (either via '!' and '!!' or via
 aliases) by prepending a $ in front of them. You can also expand complete
 python expressions. See :ref:`our shell section <system_shell_access>` for
 more details.
 
 Use profiles
 ------------
 
 Use profiles to maintain different configurations (modules to load, function
 definitions, option settings) for particular tasks. You can then have
 customized versions of IPython for specific purposes. :ref:`This section
 <profiles>` has more details.
 
 
 Embed IPython in your programs
 ------------------------------
 
 A few lines of code are enough to load a complete IPython inside your own
 programs, giving you the ability to work with your data interactively after
 automatic processing has been completed. See :ref:`here <embedding>` for more.
 
 Use the Python profiler
 -----------------------
 
 When dealing with performance issues, the %run command with a -p option
 allows you to run complete programs under the control of the Python profiler.
 The %prun command does a similar job for single Python expressions (like
 function calls).
 
 Use IPython to present interactive demos
 ----------------------------------------
 
 Use the IPython.demo.Demo class to load any Python script as an interactive
 demo. With a minimal amount of simple markup, you can control the execution of
 the script, stopping as needed. See :ref:`here <interactive_demos>` for more.
 
 Run doctests
 ------------
 
 Run your doctests from within IPython for development and debugging. The
 special %doctest_mode command toggles a mode where the prompt, output and
 exceptions display matches as closely as possible that of the default Python
 interpreter. In addition, this mode allows you to directly paste in code that
 contains leading '>>>' prompts, even if they have extra leading whitespace
 (as is common in doctest files). This combined with the '%history -tn' call
 to see your translated history (with these extra prompts removed and no line
 numbers) allows for an easy doctest workflow, where you can go from doctest
 to interactive execution to pasting into valid Python code as needed.
 
 Source code handling tips
 =========================
 
 IPython is a line-oriented program, without full control of the
 terminal. Therefore, it doesn't support true multiline editing. However,
 it has a number of useful tools to help you in dealing effectively with
 more complex editing.
 
 The %edit command gives a reasonable approximation of multiline editing,
 by invoking your favorite editor on the spot. IPython will execute the
 code you type in there as if it were typed interactively. Type %edit?
 for the full details on the edit command.
 
 If you have typed various commands during a session, which you'd like to
 reuse, IPython provides you with a number of tools. Start by using %hist
 to see your input history, so you can see the line numbers of all input.
 Let us say that you'd like to reuse lines 10 through 20, plus lines 24
 and 28. All the commands below can operate on these with the syntax::
 
     %command 10-20 24 28 
 
 where the command given can be:
 
     * %macro <macroname>: this stores the lines into a variable which,
       when called at the prompt, re-executes the input. Macros can be
       edited later using '%edit macroname', and they can be stored
       persistently across sessions with '%store macroname' (the storage
       system is per-profile). The combination of quick macros,
       persistent storage and editing, allows you to easily refine
       quick-and-dirty interactive input into permanent utilities, always
       available both in IPython and as files for general reuse.
     * %edit: this will open a text editor with those lines pre-loaded
       for further modification. It will then execute the resulting
       file's contents as if you had typed it at the prompt.
     * %save <filename>: this saves the lines directly to a named file on
       disk.
 
 While %macro saves input lines into memory for interactive re-execution,
 sometimes you'd like to save your input directly to a file. The %save
 magic does this: its input sytnax is the same as %macro, but it saves
 your input directly to a Python file. Note that the %logstart command
 also saves input, but it logs all input to disk (though you can
 temporarily suspend it and reactivate it with %logoff/%logon); %save
 allows you to select which lines of input you need to save.
 
 
 Lightweight 'version control'
 =============================
 
 When you call %edit with no arguments, IPython opens an empty editor
 with a temporary file, and it returns the contents of your editing
 session as a string variable. Thanks to IPython's output caching
 mechanism, this is automatically stored::
 
     In [1]: %edit
 
     IPython will make a temporary file named: /tmp/ipython_edit_yR-HCN.py
 
     Editing... done. Executing edited code...
 
     hello - this is a temporary file
 
     Out[1]: "print 'hello - this is a temporary file'\n"
 
 Now, if you call '%edit -p', IPython tries to open an editor with the
 same data as the last time you used %edit. So if you haven't used %edit
 in the meantime, this same contents will reopen; however, it will be
 done in a new file. This means that if you make changes and you later
 want to find an old version, you can always retrieve it by using its
 output number, via '%edit _NN', where NN is the number of the output
 prompt.
 
 Continuing with the example above, this should illustrate this idea::
 
     In [2]: edit -p
 
     IPython will make a temporary file named: /tmp/ipython_edit_nA09Qk.py
 
     Editing... done. Executing edited code...
 
     hello - now I made some changes
 
     Out[2]: "print 'hello - now I made some changes'\n"
 
     In [3]: edit _1
 
     IPython will make a temporary file named: /tmp/ipython_edit_gy6-zD.py
 
     Editing... done. Executing edited code...
 
     hello - this is a temporary file
 
     IPython version control at work :)
 
     Out[3]: "print 'hello - this is a temporary file'\nprint 'IPython version control at work :)'\n"
 
 
 This section was written after a contribution by Alexander Belchenko on
 the IPython user list.
 
 
 Effective logging
 =================
 
 A very useful suggestion sent in by Robert Kern follows:
 
 I recently happened on a nifty way to keep tidy per-project log files. I
 made a profile for my project (which is called "parkfield")::
 
     include ipythonrc
 
     # cancel earlier logfile invocation:
 
     logfile ''
 
     execute import time
 
     execute __cmd = '/Users/kern/research/logfiles/parkfield-%s.log rotate'
 
     execute __IP.magic_logstart(__cmd % time.strftime('%Y-%m-%d'))
 
 I also added a shell alias for convenience::
 
     alias parkfield="ipython -pylab -profile parkfield" 
 
 Now I have a nice little directory with everything I ever type in,
 organized by project and date.
 
 Contribute your own: If you have your own favorite tip on using IPython
 efficiently for a certain task (especially things which can't be done in
 the normal Python interpreter), don't hesitate to send it!
+
+

docs/source/overview.txt

@@ -1,232 +1,234 b''
 .. _overview:
 
 ============
 Introduction
 ============
 
 Overview
 ========
 
 One of Python's most useful features is its interactive interpreter.
 This system allows very fast testing of ideas without the overhead of
 creating test files as is typical in most programming languages.
 However, the interpreter supplied with the standard Python distribution
 is somewhat limited for extended interactive use.
 
 The goal of IPython is to create a comprehensive environment for
 interactive and exploratory computing.  To support this goal, IPython
 has two main components:
 
 * An enhanced interactive Python shell.
 * An architecture for interactive parallel computing.
 
 All of IPython is open source (released under the revised BSD license).
 
 Enhanced interactive Python shell
 =================================
 
 IPython's interactive shell (:command:`ipython`), has the following goals,
 amongst others:
 
 1. Provide an interactive shell superior to Python's default. IPython
    has many features for object introspection, system shell access,
    and its own special command system for adding functionality when
    working interactively. It tries to be a very efficient environment
    both for Python code development and for exploration of problems
    using Python objects (in situations like data analysis).
   
 2. Serve as an embeddable, ready to use interpreter for your own
    programs. IPython can be started with a single call from inside
    another program, providing access to the current namespace. This
    can be very useful both for debugging purposes and for situations
    where a blend of batch-processing and interactive exploration are
    needed.  New in the 0.9 version of IPython is a reusable wxPython
    based IPython widget.
   
 3. Offer a flexible framework which can be used as the base
    environment for other systems with Python as the underlying
    language. Specifically scientific environments like Mathematica,
    IDL and Matlab inspired its design, but similar ideas can be
    useful in many fields.
   
 4. Allow interactive testing of threaded graphical toolkits. IPython
    has support for interactive, non-blocking control of GTK, Qt and
    WX applications via special threading flags. The normal Python
    shell can only do this for Tkinter applications.
 
 Main features of the interactive shell
 --------------------------------------
 
 * Dynamic object introspection. One can access docstrings, function
   definition prototypes, source code, source files and other details
   of any object accessible to the interpreter with a single
   keystroke (:samp:`?`, and using :samp:`??` provides additional detail).
   
 * Searching through modules and namespaces with :samp:`*` wildcards, both
   when using the :samp:`?` system and via the :samp:`%psearch` command.
 
 * Completion in the local namespace, by typing :kbd:`TAB` at the prompt.
   This works for keywords, modules, methods, variables and files in the
   current directory. This is supported via the readline library, and
   full access to configuring readline's behavior is provided. 
   Custom completers can be implemented easily for different purposes
   (system commands, magic arguments etc.)
 
 * Numbered input/output prompts with command history (persistent
   across sessions and tied to each profile), full searching in this
   history and caching of all input and output.
 
 * User-extensible 'magic' commands. A set of commands prefixed with
   :samp:`%` is available for controlling IPython itself and provides
   directory control, namespace information and many aliases to
   common system shell commands.
 
 * Alias facility for defining your own system aliases.
 
 * Complete system shell access. Lines starting with :samp:`!` are passed
   directly to the system shell, and using :samp:`!!` or :samp:`var = !cmd` 
   captures shell output into python variables for further use.
 
 * Background execution of Python commands in a separate thread.
   IPython has an internal job manager called jobs, and a
   convenience backgrounding magic function called :samp:`%bg`.
 
 * The ability to expand python variables when calling the system shell. In a
   shell command, any python variable prefixed with :samp:`$` is expanded. A
   double :samp:`$$` allows passing a literal :samp:`$` to the shell (for access
   to shell and environment variables like :envvar:`PATH`).
 
 * Filesystem navigation, via a magic :samp:`%cd` command, along with a
   persistent bookmark system (using :samp:`%bookmark`) for fast access to
   frequently visited directories.
 
 * A lightweight persistence framework via the :samp:`%store` command, which
   allows you to save arbitrary Python variables. These get restored
   automatically when your session restarts.
 
 * Automatic indentation (optional) of code as you type (through the
   readline library).
 
 * Macro system for quickly re-executing multiple lines of previous
   input with a single name. Macros can be stored persistently via
   :samp:`%store` and edited via :samp:`%edit`.
 
 * Session logging (you can then later use these logs as code in your
   programs). Logs can optionally timestamp all input, and also store
   session output (marked as comments, so the log remains valid
   Python source code).
 
 * Session restoring: logs can be replayed to restore a previous
   session to the state where you left it.
 
 * Verbose and colored exception traceback printouts. Easier to parse
   visually, and in verbose mode they produce a lot of useful
   debugging information (basically a terminal version of the cgitb
   module).
 
 * Auto-parentheses: callable objects can be executed without
   parentheses: :samp:`sin 3` is automatically converted to :samp:`sin(3)`.
 
 * Auto-quoting: using :samp:`,`, or :samp:`;` as the first character forces
   auto-quoting of the rest of the line: :samp:`,my_function a b` becomes
   automatically :samp:`my_function("a","b")`, while :samp:`;my_function a b`
   becomes :samp:`my_function("a b")`.
 
 * Extensible input syntax. You can define filters that pre-process
   user input to simplify input in special situations. This allows
   for example pasting multi-line code fragments which start with
   :samp:`>>>` or :samp:`...` such as those from other python sessions or the
   standard Python documentation.
 
 * Flexible configuration system. It uses a configuration file which
   allows permanent setting of all command-line options, module
   loading, code and file execution. The system allows recursive file
   inclusion, so you can have a base file with defaults and layers
   which load other customizations for particular projects.
 
 * Embeddable. You can call IPython as a python shell inside your own
   python programs. This can be used both for debugging code or for
   providing interactive abilities to your programs with knowledge
   about the local namespaces (very useful in debugging and data
   analysis situations).
 
 * Easy debugger access. You can set IPython to call up an enhanced version of
   the Python debugger (pdb) every time there is an uncaught exception. This
   drops you inside the code which triggered the exception with all the data
   live and it is possible to navigate the stack to rapidly isolate the source
   of a bug. The :samp:`%run` magic command (with the :samp:`-d` option) can run
   any script under pdb's control, automatically setting initial breakpoints for
   you.  This version of pdb has IPython-specific improvements, including
   tab-completion and traceback coloring support. For even easier debugger
   access, try :samp:`%debug` after seeing an exception. winpdb is also
   supported, see ipy_winpdb extension.
 
 * Profiler support. You can run single statements (similar to
   :samp:`profile.run()`) or complete programs under the profiler's control.
   While this is possible with standard cProfile or profile modules,
   IPython wraps this functionality with magic commands (see :samp:`%prun`
   and :samp:`%run -p`) convenient for rapid interactive work.
 
 * Doctest support. The special :samp:`%doctest_mode` command toggles a mode
   that allows you to paste existing doctests (with leading :samp:`>>>`
   prompts and whitespace) and uses doctest-compatible prompts and
   output, so you can use IPython sessions as doctest code.
 
 Interactive parallel computing
 ==============================
 
 Increasingly, parallel computer hardware, such as multicore CPUs, clusters and
 supercomputers, is becoming ubiquitous. Over the last 3 years, we have
 developed an architecture within IPython that allows such hardware to be used
 quickly and easily from Python. Moreover, this architecture is designed to
 support interactive and collaborative parallel computing.
 
 The main features of this system are:
 
 * Quickly parallelize Python code from an interactive Python/IPython session.
 
 * A flexible and dynamic process model that be deployed on anything from 
   multicore workstations to supercomputers.
 
 * An architecture that supports many different styles of parallelism, from
   message passing to task farming.  And all of these styles can be handled
   interactively.
 
 * Both blocking and fully asynchronous interfaces.
 
 * High level APIs that enable many things to be parallelized in a few lines
   of code.
 
 * Write parallel code that will run unchanged on everything from multicore
   workstations to supercomputers.
 
 * Full integration with Message Passing libraries (MPI).
 
 * Capabilities based security model with full encryption of network connections.
 
 * Share live parallel jobs with other users securely.  We call this
   collaborative parallel computing.
 
 * Dynamically load balanced task farming system.
 
 * Robust error handling.  Python exceptions raised in parallel execution are
   gathered and presented to the top-level code.
 
 For more information, see our :ref:`overview <parallel_index>` of using IPython
 for parallel computing.
 
 Portability and Python requirements
 -----------------------------------
 
-As of the 0.9 release, IPython requires Python 2.4 or greater.  We have
-not begun to test IPython on Python 2.6 or 3.0, but we expect it will
-work with some minor changes.
+As of the 0.11 release, IPython works with either Python 2.5 or 2.6.  
+Versions 0.9 and 0.10 worked with Python 2.4 as well.  We have not begun
+the test and port IPython to 3.0.  Our plan is to gradually drop Python 2.5
+support and then begin the transition to strict 2.6 and 3.0.
 
 IPython is known to work on the following operating systems:
 
 	* Linux
 	* Most other Unix-like OSs (AIX, Solaris, BSD, etc.)
 	* Mac OS X
 	* Windows (CygWin, XP, Vista, etc.)
 
-See :ref:`here <install_index>` for instructions on how to install IPython.
  No newline at end of file
+See :ref:`here <install_index>` for instructions on how to install IPython.
+

docs/source/parallel/index.txt

@@ -1,15 +1,17 b''
 .. _parallel_index:
 
 ====================================
 Using IPython for parallel computing
 ====================================
 
 .. toctree::
    :maxdepth: 2
 
    parallel_intro.txt
    parallel_process.txt
    parallel_multiengine.txt
    parallel_task.txt
    parallel_mpi.txt
    parallel_security.txt
+
+

docs/source/parallel/parallel_mpi.txt

@@ -1,176 +1,178 b''
 .. _parallelmpi:
 
 =======================
 Using MPI with IPython
 =======================
 
 Often, a parallel algorithm will require moving data between the engines. One
 way of accomplishing this is by doing a pull and then a push using the
 multiengine client. However, this will be slow as all the data has to go
 through the controller to the client and then back through the controller, to
 its final destination.
 
 A much better way of moving data between engines is to use a message passing
 library, such as the Message Passing Interface (MPI) [MPI]_. IPython's
 parallel computing architecture has been designed from the ground up to
 integrate with MPI. This document describes how to use MPI with IPython.
 
 Additional installation requirements
 ====================================
 
 If you want to use MPI with IPython, you will need to install:
 
 * A standard MPI implementation such as OpenMPI [OpenMPI]_ or MPICH.
 * The mpi4py [mpi4py]_ package.
 
 .. note::
 
     The mpi4py package is not a strict requirement. However, you need to
     have *some* way of calling MPI from Python. You also need some way of
     making sure that :func:`MPI_Init` is called when the IPython engines start
     up. There are a number of ways of doing this and a good number of
     associated subtleties. We highly recommend just using mpi4py as it
     takes care of most of these problems. If you want to do something
     different, let us know and we can help you get started.
 
 Starting the engines with MPI enabled
 =====================================
 
 To use code that calls MPI, there are typically two things that MPI requires.
 
 1. The process that wants to call MPI must be started using
    :command:`mpiexec` or a batch system (like PBS) that has MPI support.
 2. Once the process starts, it must call :func:`MPI_Init`.
 
 There are a couple of ways that you can start the IPython engines and get
 these things to happen.
 
 Automatic starting using :command:`mpiexec` and :command:`ipcluster`
 --------------------------------------------------------------------
 
 The easiest approach is to use the `mpiexec` mode of :command:`ipcluster`,
 which will first start a controller and then a set of engines using
 :command:`mpiexec`::
 
     $ ipcluster mpiexec -n 4
 
 This approach is best as interrupting :command:`ipcluster` will automatically
 stop and clean up the controller and engines.
 
 Manual starting using :command:`mpiexec`
 ----------------------------------------
 
 If you want to start the IPython engines using the :command:`mpiexec`, just
 do::
 
     $ mpiexec -n 4 ipengine --mpi=mpi4py
 
 This requires that you already have a controller running and that the FURL
 files for the engines are in place. We also have built in support for
 PyTrilinos [PyTrilinos]_, which can be used (assuming is installed) by
 starting the engines with::
 
     	mpiexec -n 4 ipengine --mpi=pytrilinos
 
 Automatic starting using PBS and :command:`ipcluster`
 -----------------------------------------------------
 
 The :command:`ipcluster` command also has built-in integration with PBS. For
 more information on this approach, see our documentation on :ref:`ipcluster
 <parallel_process>`.
 
 Actually using MPI
 ==================
 
 Once the engines are running with MPI enabled, you are ready to go. You can
 now call any code that uses MPI in the IPython engines. And, all of this can
 be done interactively. Here we show a simple example that uses mpi4py
 [mpi4py]_.
 
 First, lets define a simply function that uses MPI to calculate the sum of a
 distributed array. Save the following text in a file called :file:`psum.py`:
 
 .. sourcecode:: python
 
     from mpi4py import MPI
     import numpy as np
     
     def psum(a):
         s = np.sum(a)
         return MPI.COMM_WORLD.Allreduce(s,MPI.SUM)
 
 Now, start an IPython cluster in the same directory as :file:`psum.py`::
 
     $ ipcluster mpiexec -n 4
 
 Finally, connect to the cluster and use this function interactively. In this
 case, we create a random array on each engine and sum up all the random arrays
 using our :func:`psum` function:
 
 .. sourcecode:: ipython
 
     In [1]: from IPython.kernel import client
     
     In [2]: mec = client.MultiEngineClient()
     
     In [3]: mec.activate()
     
     In [4]: px import numpy as np
     Parallel execution on engines: all
     Out[4]: 
     <Results List>
     [0] In [13]: import numpy as np
     [1] In [13]: import numpy as np
     [2] In [13]: import numpy as np
     [3] In [13]: import numpy as np
     
     In [6]: px a = np.random.rand(100)
     Parallel execution on engines: all
     Out[6]: 
     <Results List>
     [0] In [15]: a = np.random.rand(100)
     [1] In [15]: a = np.random.rand(100)
     [2] In [15]: a = np.random.rand(100)
     [3] In [15]: a = np.random.rand(100)
     
     In [7]: px from psum import psum
     Parallel execution on engines: all
     Out[7]: 
     <Results List>
     [0] In [16]: from psum import psum
     [1] In [16]: from psum import psum
     [2] In [16]: from psum import psum
     [3] In [16]: from psum import psum
     
     In [8]: px s = psum(a)
     Parallel execution on engines: all
     Out[8]: 
     <Results List>
     [0] In [17]: s = psum(a)
     [1] In [17]: s = psum(a)
     [2] In [17]: s = psum(a)
     [3] In [17]: s = psum(a)
     
     In [9]: px print s
     Parallel execution on engines: all
     Out[9]: 
     <Results List>
     [0] In [18]: print s
     [0] Out[18]: 187.451545803
     
     [1] In [18]: print s
     [1] Out[18]: 187.451545803
     
     [2] In [18]: print s
     [2] Out[18]: 187.451545803
 
     [3] In [18]: print s
     [3] Out[18]: 187.451545803
 
 Any Python code that makes calls to MPI can be used in this manner, including
 compiled C, C++ and Fortran libraries that have been exposed to Python.
 
 .. [MPI] Message Passing Interface.  http://www-unix.mcs.anl.gov/mpi/
 .. [mpi4py] MPI for Python. mpi4py: http://mpi4py.scipy.org/
 .. [OpenMPI] Open MPI. http://www.open-mpi.org/
 .. [PyTrilinos] PyTrilinos. http://trilinos.sandia.gov/packages/pytrilinos/
+
+

docs/source/parallel/parallel_process.txt

@@ -1,387 +1,389 b''
 .. _parallel_process:
 
 ===========================================
 Starting the IPython controller and engines
 ===========================================
 
 To use IPython for parallel computing, you need to start one instance of
 the controller and one or more instances of the engine. The controller
 and each engine can run on different machines or on the same machine.
 Because of this, there are many different possibilities.
 
 Broadly speaking, there are two ways of going about starting a controller and engines:
 
 * In an automated manner using the :command:`ipcluster` command.
 * In a more manual way using the :command:`ipcontroller` and 
   :command:`ipengine` commands.
 
 This document describes both of these methods. We recommend that new users
 start with the :command:`ipcluster` command as it simplifies many common usage
 cases.
 
 General considerations
 ======================
 
 Before delving into the details about how you can start a controller and
 engines using the various methods, we outline some of the general issues that
 come up when starting the controller and engines. These things come up no
 matter which method you use to start your IPython cluster.
 
 Let's say that you want to start the controller on ``host0`` and engines on
 hosts ``host1``-``hostn``. The following steps are then required:
 
 1. Start the controller on ``host0`` by running :command:`ipcontroller` on
    ``host0``.
 2. Move the FURL file (:file:`ipcontroller-engine.furl`) created by the 
    controller from ``host0`` to hosts ``host1``-``hostn``.
 3. Start the engines on hosts ``host1``-``hostn`` by running 
    :command:`ipengine`.  This command has to be told where the FURL file 
    (:file:`ipcontroller-engine.furl`) is located.
 
 At this point, the controller and engines will be connected. By default, the
 FURL files created by the controller are put into the
 :file:`~/.ipython/security` directory. If the engines share a filesystem with
 the controller, step 2 can be skipped as the engines will automatically look
 at that location.
 
 The final step required required to actually use the running controller from a
 client is to move the FURL files :file:`ipcontroller-mec.furl` and
 :file:`ipcontroller-tc.furl` from ``host0`` to the host where the clients will
 be run. If these file are put into the :file:`~/.ipython/security` directory
 of the client's host, they will be found automatically. Otherwise, the full
 path to them has to be passed to the client's constructor.
 
 Using :command:`ipcluster`
 ==========================
 
 The :command:`ipcluster` command provides a simple way of starting a
 controller and engines in the following situations:
 
 1. When the controller and engines are all run on localhost. This is useful
    for testing or running on a multicore computer.
 2. When engines are started using the :command:`mpirun` command that comes 
    with most MPI [MPI]_ implementations
 3. When engines are started using the PBS [PBS]_ batch system.
 4. When the controller is started on localhost and the engines are started on 
    remote nodes using :command:`ssh`.
 
 .. note::
 
     It is also possible for advanced users to add support to
     :command:`ipcluster` for starting controllers and engines using other
     methods (like Sun's Grid Engine for example).
 
 .. note::
 
     Currently :command:`ipcluster` requires that the
     :file:`~/.ipython/security` directory live on a shared filesystem that is
     seen by both the controller and engines. If you don't have a shared file
     system you will need to use :command:`ipcontroller` and
     :command:`ipengine` directly.  This constraint can be relaxed if you are 
     using the :command:`ssh` method to start the cluster.
 
 Underneath the hood, :command:`ipcluster` just uses :command:`ipcontroller`
 and :command:`ipengine` to perform the steps described above.
 
 Using :command:`ipcluster` in local mode
 ----------------------------------------
 
 To start one controller and 4 engines on localhost, just do::
 
     $ ipcluster local -n 4
 
 To see other command line options for the local mode, do::
 
     $ ipcluster local -h
 
 Using :command:`ipcluster` in mpiexec/mpirun mode
 -------------------------------------------------
 
 The mpiexec/mpirun mode is useful if you:
 
 1. Have MPI installed.
 2. Your systems are configured to use the :command:`mpiexec` or
    :command:`mpirun` commands to start MPI processes.
 
 .. note::
 
     The preferred command to use is :command:`mpiexec`.  However, we also 
     support :command:`mpirun` for backwards compatibility.  The underlying
     logic used is exactly the same, the only difference being the name of the 
     command line program that is called.
 
 If these are satisfied, you can start an IPython cluster using::
 
     $ ipcluster mpiexec -n 4
 
 This does the following:
 
 1. Starts the IPython controller on current host.
 2. Uses :command:`mpiexec` to start 4 engines.
 
 On newer MPI implementations (such as OpenMPI), this will work even if you
 don't make any calls to MPI or call :func:`MPI_Init`. However, older MPI
 implementations actually require each process to call :func:`MPI_Init` upon
 starting. The easiest way of having this done is to install the mpi4py
 [mpi4py]_ package and then call ipcluster with the ``--mpi`` option::
 
     $ ipcluster mpiexec -n 4 --mpi=mpi4py
 
 Unfortunately, even this won't work for some MPI implementations. If you are
 having problems with this, you will likely have to use a custom Python
 executable that itself calls :func:`MPI_Init` at the appropriate time.
 Fortunately, mpi4py comes with such a custom Python executable that is easy to
 install and use. However, this custom Python executable approach will not work
 with :command:`ipcluster` currently.
 
 Additional command line options for this mode can be found by doing::
 
     $ ipcluster mpiexec -h
 
 More details on using MPI with IPython can be found :ref:`here <parallelmpi>`.
 
 
 Using :command:`ipcluster` in PBS mode
 --------------------------------------
 
 The PBS mode uses the Portable Batch System [PBS]_ to start the engines. To
 use this mode, you first need to create a PBS script template that will be
 used to start the engines. Here is a sample PBS script template:
 
 .. sourcecode:: bash
 
     #PBS -N ipython
     #PBS -j oe
     #PBS -l walltime=00:10:00
     #PBS -l nodes=${n/4}:ppn=4
     #PBS -q parallel
 
     cd $$PBS_O_WORKDIR
     export PATH=$$HOME/usr/local/bin
     export PYTHONPATH=$$HOME/usr/local/lib/python2.4/site-packages
     /usr/local/bin/mpiexec -n ${n} ipengine --logfile=$$PBS_O_WORKDIR/ipengine
 
 There are a few important points about this template:
 
 1. This template will be rendered at runtime using IPython's :mod:`Itpl` 
    template engine.
 
 2. Instead of putting in the actual number of engines, use the notation
    ``${n}`` to indicate the number of engines to be started. You can also uses
    expressions like ``${n/4}`` in the template to indicate the number of 
    nodes.
 
 3. Because ``$`` is a special character used by the template engine, you must  
    escape any ``$`` by using ``$$``.  This is important when referring to 
    environment variables in the template.
 
 4. Any options to :command:`ipengine` should be given in the batch script 
    template.
 
 5. Depending on the configuration of you system, you may have to set 
    environment variables in the script template.
 
 Once you have created such a script, save it with a name like
 :file:`pbs.template`. Now you are ready to start your job::
 
     $ ipcluster pbs -n 128 --pbs-script=pbs.template
 
 Additional command line options for this mode can be found by doing::
 
     $ ipcluster pbs -h
 
 Using :command:`ipcluster` in SSH mode
 --------------------------------------
 
 The SSH mode uses :command:`ssh` to execute :command:`ipengine` on remote
 nodes and the :command:`ipcontroller` on localhost.
 
 When using using this mode it highly recommended that you have set up SSH keys
 and are using ssh-agent [SSH]_ for password-less logins.
 
 To use this mode you need a python file describing the cluster, here is an
 example of such a "clusterfile":
 
 .. sourcecode:: python
     
     send_furl = True
     engines = { 'host1.example.com' : 2,
                 'host2.example.com' : 5,
                 'host3.example.com' : 1,
                 'host4.example.com' : 8 }
 
 Since this is a regular python file usual python syntax applies. Things to
 note:
 
 * The `engines` dict, where the keys is the host we want to run engines on and
   the value is the number of engines to run on that host.
 * send_furl can either be `True` or `False`, if `True` it will copy over the
   furl needed for :command:`ipengine` to each host.
 
 The ``--clusterfile`` command line option lets you specify the file to use for
 the cluster definition. Once you have your cluster file and you can
 :command:`ssh` into the remote hosts with out an password you are ready to
 start your cluster like so:
 
 .. sourcecode:: bash 
 
    $ ipcluster ssh --clusterfile /path/to/my/clusterfile.py
 
 
 Two helper shell scripts are used to start and stop :command:`ipengine` on
 remote hosts:
 
 * sshx.sh 
 * engine_killer.sh
 
 Defaults for both of these are contained in the source code for
 :command:`ipcluster`. The default scripts are written to a local file in a
 tmep directory and then copied to a temp directory on the remote host and
 executed from there. On most Unix, Linux and OS X systems this is /tmp.
 
 The default sshx.sh is the following:
 
 .. sourcecode:: bash
 
    #!/bin/sh
    "$@" &> /dev/null &
    echo $!
 
 If you want to use a custom sshx.sh script you need to use the ``--sshx``
 option and specify the file to use. Using a custom sshx.sh file could be
 helpful when you need to setup the environment on the remote host before
 executing :command:`ipengine`.
 
 For a detailed options list:
 
 .. sourcecode:: bash
 
    $ ipcluster ssh -h
 
 Current limitations of the SSH mode of :command:`ipcluster` are:
 
 * Untested on Windows. Would require a working :command:`ssh` on Windows.
   Also, we are using shell scripts to setup and execute commands on remote
   hosts.
 * :command:`ipcontroller` is started on localhost, with no option to start it
   on a remote node.
 
 Using the :command:`ipcontroller` and :command:`ipengine` commands
 ==================================================================
 
 It is also possible to use the :command:`ipcontroller` and :command:`ipengine`
 commands to start your controller and engines. This approach gives you full
 control over all aspects of the startup process.
 
 Starting the controller and engine on your local machine
 --------------------------------------------------------
 
 To use :command:`ipcontroller` and :command:`ipengine` to start things on your
 local machine, do the following.
 
 First start the controller::
 
 	$ ipcontroller
 	
 Next, start however many instances of the engine you want using (repeatedly)
 the command::
 
 	$ ipengine
 
 The engines should start and automatically connect to the controller using the
 FURL files in :file:`~./ipython/security`. You are now ready to use the
 controller and engines from IPython.
 
 .. warning:: 
 	
 	The order of the above operations is very important.  You *must*
  	start the controller before the engines, since the engines connect 
 	to the controller as they get started.
 
 .. note::
 
     On some platforms (OS X), to put the controller and engine into the
     background you may need to give these commands in the form ``(ipcontroller
     &)`` and ``(ipengine &)`` (with the parentheses) for them to work
     properly.
 
 Starting the controller and engines on different hosts
 ------------------------------------------------------
 
 When the controller and engines are running on different hosts, things are
 slightly more complicated, but the underlying ideas are the same:
 
 1. Start the controller on a host using :command:`ipcontroller`.
 2. Copy :file:`ipcontroller-engine.furl` from :file:`~./ipython/security` on
    the controller's host to the host where the engines will run.
 3. Use :command:`ipengine` on the engine's hosts to start the engines.
 
 The only thing you have to be careful of is to tell :command:`ipengine` where
 the :file:`ipcontroller-engine.furl` file is located. There are two ways you
 can do this:
 
 * Put :file:`ipcontroller-engine.furl` in the :file:`~./ipython/security`
   directory on the engine's host, where it will be found automatically.
 * Call :command:`ipengine` with the ``--furl-file=full_path_to_the_file``
   flag.
 
 The ``--furl-file`` flag works like this::
 
     $ ipengine --furl-file=/path/to/my/ipcontroller-engine.furl
 
 .. note::
     
     If the controller's and engine's hosts all have a shared file system  
     (:file:`~./ipython/security` is the same on all of them), then things
     will just work!
 
 Make FURL files persistent
 ---------------------------
 
 At fist glance it may seem that that managing the FURL files is a bit
 annoying. Going back to the house and key analogy, copying the FURL around
 each time you start the controller is like having to make a new key every time
 you want to unlock the door and enter your house. As with your house, you want
 to be able to create the key (or FURL file) once, and then simply use it at
 any point in the future.
 
 This is possible, but before you do this, you **must** remove any old FURL
 files in the :file:`~/.ipython/security` directory.
 
 .. warning::
 
     You **must** remove old FURL files before using persistent FURL files.
 
 Then, The only thing you have to do is decide what ports the controller will
 listen on for the engines and clients. This is done as follows::
 
     $ ipcontroller -r --client-port=10101 --engine-port=10102
 
 These options also work with all of the various modes of
 :command:`ipcluster`::
 
     $ ipcluster local -n 2 -r --client-port=10101 --engine-port=10102
 
 Then, just copy the furl files over the first time and you are set. You can
 start and stop the controller and engines any many times as you want in the
 future, just make sure to tell the controller to use the *same* ports.
 
 .. note::
 
     You may ask the question: what ports does the controller listen on if you
     don't tell is to use specific ones? The default is to use high random port
     numbers. We do this for two reasons: i) to increase security through
     obscurity and ii) to multiple controllers on a given host to start and
     automatically use different ports.
 
 Log files
 ---------
 
 All of the components of IPython have log files associated with them.
 These log files can be extremely useful in debugging problems with
 IPython and can be found in the directory :file:`~/.ipython/log`.  Sending
 the log files to us will often help us to debug any problems.
 
 
 .. [PBS] Portable Batch System.  http://www.openpbs.org/
 .. [SSH] SSH-Agent http://en.wikipedia.org/wiki/Ssh-agent
+
+

docs/source/parallel/parallel_security.txt

@@ -1,364 +1,366 b''
 .. _parallelsecurity:
 
 ===========================
 Security details of IPython
 ===========================
 
 IPython's :mod:`IPython.kernel` package exposes the full power of the Python
 interpreter over a TCP/IP network for the purposes of parallel computing. This
 feature brings up the important question of IPython's security model. This
 document gives details about this model and how it is implemented in IPython's
 architecture.
 
 Processs and network topology
 =============================
 
 To enable parallel computing, IPython has a number of different processes that
 run. These processes are discussed at length in the IPython documentation and
 are summarized here:
 
 * The IPython *engine*.  This process is a full blown Python
   interpreter in which user code is executed.  Multiple
   engines are started to make parallel computing possible.
 * The IPython *controller*.  This process manages a set of 
   engines, maintaining a queue for each and presenting
   an asynchronous interface to the set of engines.
 * The IPython *client*.  This process is typically an
   interactive Python process that is used to coordinate the
   engines to get a parallel computation done.
 
 Collectively, these three processes are called the IPython *kernel*.
 
 These three processes communicate over TCP/IP connections with a well defined
 topology. The IPython controller is the only process that listens on TCP/IP
 sockets. Upon starting, an engine connects to a controller and registers
 itself with the controller. These engine/controller TCP/IP connections persist
 for the lifetime of each engine.
 
 The IPython client also connects to the controller using one or more TCP/IP
 connections. These connections persist for the lifetime of the client only.
 
 A given IPython controller and set of engines typically has a relatively short
 lifetime. Typically this lifetime corresponds to the duration of a single
 parallel simulation performed by a single user. Finally, the controller,
 engines and client processes typically execute with the permissions of that
 same user. More specifically, the controller and engines are *not* executed as
 root or with any other superuser permissions.
 
 Application logic
 =================
 
 When running the IPython kernel to perform a parallel computation, a user
 utilizes the IPython client to send Python commands and data through the
 IPython controller to the IPython engines, where those commands are executed
 and the data processed. The design of IPython ensures that the client is the
 only access point for the capabilities of the engines. That is, the only way
 of addressing the engines is through a client.
 
 A user can utilize the client to instruct the IPython engines to execute
 arbitrary Python commands. These Python commands can include calls to the
 system shell, access the filesystem, etc., as required by the user's
 application code. From this perspective, when a user runs an IPython engine on
 a host, that engine has the same capabilities and permissions as the user
 themselves (as if they were logged onto the engine's host with a terminal).
 
 Secure network connections
 ==========================
 
 Overview
 --------
 
 All TCP/IP connections between the client and controller as well as the
 engines and controller are fully encrypted and authenticated. This section
 describes the details of the encryption and authentication approached used
 within IPython.
 
 IPython uses the Foolscap network protocol [Foolscap]_ for all communications
 between processes. Thus, the details of IPython's security model are directly
 related to those of Foolscap. Thus, much of the following discussion is
 actually just a discussion of the security that is built in to Foolscap.
 
 Encryption
 ----------
 
 For encryption purposes, IPython and Foolscap use the well known Secure Socket
 Layer (SSL) protocol [RFC5246]_. We use the implementation of this protocol
 provided by the OpenSSL project through the pyOpenSSL [pyOpenSSL]_ Python
 bindings to OpenSSL.
 
 Authentication
 --------------
 
 IPython clients and engines must also authenticate themselves with the
 controller. This is handled in a capabilities based security model
 [Capability]_. In this model, the controller creates a strong cryptographic
 key or token that represents each set of capability that the controller
 offers. Any party who has this key and presents it to the controller has full
 access to the corresponding capabilities of the controller. This model is
 analogous to using a physical key to gain access to physical items
 (capabilities) behind a locked door.
 
 For a capabilities based authentication system to prevent unauthorized access,
 two things must be ensured:
 
 * The keys must be cryptographically strong.  Otherwise attackers could gain 
   access by a simple brute force key guessing attack.
 * The actual keys must be distributed only to authorized parties.
 
 The keys in Foolscap are called Foolscap URL's or FURLs. The following section
 gives details about how these FURLs are created in Foolscap. The IPython
 controller creates a number of FURLs for different purposes:
 
 * One FURL that grants IPython engines access to the controller.  Also
   implicit in this access is permission to execute code sent by an
   authenticated IPython client.
 * Two or more FURLs that grant IPython clients access to the controller.
   Implicit in this access is permission to give the controller's engine code 
   to execute.
 
 Upon starting, the controller creates these different FURLS and writes them
 files in the user-read-only directory :file:`$HOME/.ipython/security`. Thus,
 only the user who starts the controller has access to the FURLs.
 
 For an IPython client or engine to authenticate with a controller, it must
 present the appropriate FURL to the controller upon connecting. If the
 FURL matches what the controller expects for a given capability, access is
 granted. If not, access is denied. The exchange of FURLs is done after
 encrypted communications channels have been established to prevent attackers
 from capturing them.
 
 .. note::
 
     The FURL is similar to an unsigned private key in SSH.
 
 Details of the Foolscap handshake
 ---------------------------------
 
 In this section we detail the precise security handshake that takes place at
 the beginning of any network connection in IPython. For the purposes of this
 discussion, the SERVER is the IPython controller process and the CLIENT is the
 IPython engine or client process.
 
 Upon starting, all IPython processes do the following:
 
 1. Create a public key x509 certificate (ISO/IEC 9594).
 2. Create a hash of the contents of the certificate using the SHA-1 algorithm.
    The base-32 encoded version of this hash is saved by the process as its 
    process id (actually in Foolscap, this is the Tub id, but here refer to
    it as the process id).
 
 Upon starting, the IPython controller also does the following:
 
 1. Save the x509 certificate to disk in a secure location. The CLIENT
    certificate is never saved to disk.
 2. Create a FURL for each capability that the controller has.  There are
    separate capabilities the controller offers for clients and engines.  The
    FURL is created using:  a) the process id of the SERVER, b) the IP 
    address and port the SERVER is listening on and c) a 160 bit,    
    cryptographically secure string that represents the capability (the
    "capability id").
 3. The FURLs are saved to disk in a secure location on the SERVER's host.
 
 For a CLIENT to be able to connect to the SERVER and access a capability of
 that SERVER, the CLIENT must have knowledge of the FURL for that SERVER's
 capability. This typically requires that the file containing the FURL be
 moved from the SERVER's host to the CLIENT's host. This is done by the end
 user who started the SERVER and wishes to have a CLIENT connect to the SERVER.
 
 When a CLIENT connects to the SERVER, the following handshake protocol takes
 place:
 
 1. The CLIENT tells the SERVER what process (or Tub) id it expects the SERVER
    to have.
 2. If the SERVER has that process id, it notifies the CLIENT that it will now 
    enter encrypted mode.  If the SERVER has a different id, the SERVER aborts.
 3. Both CLIENT and SERVER initiate the SSL handshake protocol.
 4. Both CLIENT and SERVER request the certificate of their peer and verify
    that certificate.  If this succeeds, all further communications are 
    encrypted.
 5. Both CLIENT and SERVER send a hello block containing connection parameters
    and their process id.
 6. The CLIENT and SERVER check that their peer's stated process id matches the
    hash of the x509 certificate the peer presented.  If not, the connection is
    aborted.
 7. The CLIENT verifies that the SERVER's stated id matches the id of the
    SERVER the CLIENT is intending to connect to.  If not, the connection is 
    aborted.
 8. The CLIENT and SERVER elect a master who decides on the final connection
    parameters.
 
 The public/private key pair associated with each process's x509 certificate
 are completely hidden from this handshake protocol. There are however, used
 internally by OpenSSL as part of the SSL handshake protocol. Each process
 keeps their own private key hidden and sends its peer only the public key
 (embedded in the certificate).
 
 Finally, when the CLIENT requests access to a particular SERVER capability,
 the following happens:
 
 1. The CLIENT asks the SERVER for access to a capability by presenting that 
    capabilities id.
 2. If the SERVER has a capability with that id, access is granted. If not,
    access is not granted.
 3. Once access has been gained, the CLIENT can use the capability.
 
 Specific security vulnerabilities
 =================================
 
 There are a number of potential security vulnerabilities present in IPython's
 architecture. In this section we discuss those vulnerabilities and detail how
 the security architecture described above prevents them from being exploited.
 
 Unauthorized clients
 --------------------
 
 The IPython client can instruct the IPython engines to execute arbitrary
 Python code with the permissions of the user who started the engines. If an
 attacker were able to connect their own hostile IPython client to the IPython
 controller, they could instruct the engines to execute code.
 
 This attack is prevented by the capabilities based client authentication
 performed after the encrypted channel has been established. The relevant
 authentication information is encoded into the FURL that clients must
 present to gain access to the IPython controller. By limiting the distribution
 of those FURLs, a user can grant access to only authorized persons.
 
 It is highly unlikely that a client FURL could be guessed by an attacker
 in a brute force guessing attack. A given instance of the IPython controller
 only runs for a relatively short amount of time (on the order of hours). Thus
 an attacker would have only a limited amount of time to test a search space of
 size 2**320. Furthermore, even if a controller were to run for a longer amount
 of time, this search space is quite large (larger for instance than that of
 typical username/password pair).
 
 Unauthorized engines
 --------------------
 
 If an attacker were able to connect a hostile engine to a user's controller,
 the user might unknowingly send sensitive code or data to the hostile engine.
 This attacker's engine would then have full access to that code and data.
 
 This type of attack is prevented in the same way as the unauthorized client
 attack, through the usage of the capabilities based authentication scheme.
 
 Unauthorized controllers
 ------------------------
 
 It is also possible that an attacker could try to convince a user's IPython
 client or engine to connect to a hostile IPython controller. That controller
 would then have full access to the code and data sent between the IPython
 client and the IPython engines.
 
 Again, this attack is prevented through the FURLs, which ensure that a
 client or engine connects to the correct controller. It is also important to
 note that the FURLs also encode the IP address and port that the
 controller is listening on, so there is little chance of mistakenly connecting
 to a controller running on a different IP address and port.
 
 When starting an engine or client, a user must specify which FURL to use
 for that connection. Thus, in order to introduce a hostile controller, the
 attacker must convince the user to use the FURLs associated with the
 hostile controller. As long as a user is diligent in only using FURLs from
 trusted sources, this attack is not possible.
 
 Other security measures
 =======================
 
 A number of other measures are taken to further limit the security risks
 involved in running the IPython kernel.
 
 First, by default, the IPython controller listens on random port numbers.
 While this can be overridden by the user, in the default configuration, an
 attacker would have to do a port scan to even find a controller to attack.
 When coupled with the relatively short running time of a typical controller
 (on the order of hours), an attacker would have to work extremely hard and
 extremely *fast* to even find a running controller to attack.
 
 Second, much of the time, especially when run on supercomputers or clusters,
 the controller is running behind a firewall. Thus, for engines or client to
 connect to the controller:
 
 * The different processes have to all be behind the firewall.
 
 or:
 
 * The user has to use SSH port forwarding to tunnel the
   connections through the firewall.
    
 In either case, an attacker is presented with addition barriers that prevent
 attacking or even probing the system.
 
 Summary
 =======
 
 IPython's architecture has been carefully designed with security in mind. The
 capabilities based authentication model, in conjunction with the encrypted
 TCP/IP channels, address the core potential vulnerabilities in the system,
 while still enabling user's to use the system in open networks.
 
 Other questions
 ===============
 
 About keys
 ----------
 
 Can you clarify the roles of the certificate and its keys versus the FURL,
 which is also called a key?
 
 The certificate created by IPython processes is a standard public key x509
 certificate, that is used by the SSL handshake protocol to setup encrypted
 channel between the controller and the IPython engine or client. This public
 and private key associated with this certificate are used only by the SSL
 handshake protocol in setting up this encrypted channel.
 
 The FURL serves a completely different and independent purpose from the
 key pair associated with the certificate. When we refer to a FURL as a
 key, we are using the word "key" in the capabilities based security model
 sense. This has nothing to do with "key" in the public/private key sense used
 in the SSL protocol.
 
 With that said the FURL is used as an cryptographic key, to grant
 IPython engines and clients access to particular capabilities that the
 controller offers.
 
 Self signed certificates
 ------------------------
 
 Is the controller creating a self-signed certificate? Is this created for per
 instance/session, one-time-setup or each-time the controller is started?
 
 The Foolscap network protocol, which handles the SSL protocol details, creates
 a self-signed x509 certificate using OpenSSL for each IPython process. The
 lifetime of the certificate is handled differently for the IPython controller
 and the engines/client.
 
 For the IPython engines and client, the certificate is only held in memory for
 the lifetime of its process. It is never written to disk.
 
 For the controller, the certificate can be created anew each time the
 controller starts or it can be created once and reused each time the
 controller starts. If at any point, the certificate is deleted, a new one is
 created the next time the controller starts.
 
 SSL private key
 ---------------
 
 How the private key (associated with the certificate) is distributed?
 
 In the usual implementation of the SSL protocol, the private key is never
 distributed. We follow this standard always.
 
 SSL versus Foolscap authentication
 ----------------------------------
 
 Many SSL connections only perform one sided authentication (the server to the
 client). How is the client authentication in IPython's system related to SSL
 authentication?
 
 We perform a two way SSL handshake in which both parties request and verify
 the certificate of their peer. This mutual authentication is handled by the
 SSL handshake and is separate and independent from the additional
 authentication steps that the CLIENT and SERVER perform after an encrypted
 channel is established.
 
 .. [RFC5246] <http://tools.ietf.org/html/rfc5246>
+
+

docs/source/parallel/parallel_task.txt

@@ -1,120 +1,121 b''
 .. _paralleltask:
 
 ==========================
 The IPython task interface
 ==========================
 
 The task interface to the controller presents the engines as a fault tolerant,
 dynamic load-balanced system or workers. Unlike the multiengine interface, in
 the task interface, the user have no direct access to individual engines. In
 some ways, this interface is simpler, but in other ways it is more powerful.
 
 Best of all the user can use both of these interfaces running at the same time
 to take advantage or both of their strengths. When the user can break up the
 user's work into segments that do not depend on previous execution, the task
 interface is ideal. But it also has more power and flexibility, allowing the
 user to guide the distribution of jobs, without having to assign tasks to
 engines explicitly.
 
 Starting the IPython controller and engines
 ===========================================
 
 To follow along with this tutorial, you will need to start the IPython
 controller and four IPython engines. The simplest way of doing this is to use
 the :command:`ipcluster` command::
 
 	$ ipcluster local -n 4
 	
 For more detailed information about starting the controller and engines, see
 our :ref:`introduction <ip1par>` to using IPython for parallel computing.
 
 Creating a ``TaskClient`` instance
 =========================================
 
 The first step is to import the IPython :mod:`IPython.kernel.client` module
 and then create a :class:`TaskClient` instance:
 
 .. sourcecode:: ipython
 
 	In [1]: from IPython.kernel import client
 	
 	In [2]: tc = client.TaskClient()
 
 This form assumes that the :file:`ipcontroller-tc.furl` is in the
 :file:`~./ipython/security` directory on the client's host. If not, the
 location of the FURL file must be given as an argument to the
 constructor:
 
 .. sourcecode:: ipython
 
     In [2]: mec = client.TaskClient('/path/to/my/ipcontroller-tc.furl')
 
 Quick and easy parallelism
 ==========================
 
 In many cases, you simply want to apply a Python function to a sequence of
 objects, but *in parallel*. Like the multiengine interface, the task interface
 provides two simple ways of accomplishing this: a parallel version of
 :func:`map` and ``@parallel`` function decorator. However, the verions in the
 task interface have one important difference: they are dynamically load
 balanced. Thus, if the execution time per item varies significantly, you
 should use the versions in the task interface.
 
 Parallel map
 ------------
 
 The parallel :meth:`map` in the task interface is similar to that in the
 multiengine interface:
 
 .. sourcecode:: ipython
 
 	In [63]: serial_result = map(lambda x:x**10, range(32))
 
 	In [64]: parallel_result = tc.map(lambda x:x**10, range(32))
 
 	In [65]: serial_result==parallel_result
 	Out[65]: True
 
 Parallel function decorator
 ---------------------------
 
 Parallel functions are just like normal function, but they can be called on
 sequences and *in parallel*. The multiengine interface provides a decorator
 that turns any Python function into a parallel function:
 
 .. sourcecode:: ipython
 
     In [10]: @tc.parallel()
        ....: def f(x):
        ....:     return 10.0*x**4
        ....: 
 
     In [11]: f(range(32))    # this is done in parallel
     Out[11]: 
     [0.0,10.0,160.0,...]
 
 More details
 ============
 
 The :class:`TaskClient` has many more powerful features that allow quite a bit
 of flexibility in how tasks are defined and run. The next places to look are
 in the following classes:
 
 * :class:`IPython.kernel.client.TaskClient`
 * :class:`IPython.kernel.client.StringTask`
 * :class:`IPython.kernel.client.MapTask`
 
 The following is an overview of how to use these classes together:
 
 1. Create a :class:`TaskClient`.
 2. Create one or more instances of :class:`StringTask` or :class:`MapTask`
    to define your tasks.
 3. Submit your tasks to using the :meth:`run` method of your
    :class:`TaskClient` instance.
 4. Use :meth:`TaskClient.get_task_result` to get the results of the
    tasks.
 
 We are in the process of developing more detailed information about the task
 interface. For now, the docstrings of the :class:`TaskClient`,
 :class:`StringTask` and :class:`MapTask` classes should be consulted.
 
+

docs/sphinxext/apigen.py

@@ -1,426 +1,427 b''
 """Attempt to generate templates for module reference with Sphinx
 
 XXX - we exclude extension modules
 
 To include extension modules, first identify them as valid in the
 ``_uri2path`` method, then handle them in the ``_parse_module`` script.
 
 We get functions and classes by parsing the text of .py files.
 Alternatively we could import the modules for discovery, and we'd have
 to do that for extension modules.  This would involve changing the
 ``_parse_module`` method to work via import and introspection, and
 might involve changing ``discover_modules`` (which determines which
 files are modules, and therefore which module URIs will be passed to
 ``_parse_module``).
 
 NOTE: this is a modified version of a script originally shipped with the
 PyMVPA project, which we've adapted for NIPY use.  PyMVPA is an MIT-licensed
 project."""
 
 # Stdlib imports
 import os
 import re
 
 # Functions and classes
 class ApiDocWriter(object):
     ''' Class for automatic detection and parsing of API docs
     to Sphinx-parsable reST format'''
 
     # only separating first two levels
     rst_section_levels = ['*', '=', '-', '~', '^']
 
     def __init__(self,
                  package_name,
                  rst_extension='.rst',
                  package_skip_patterns=None,
                  module_skip_patterns=None,
                  ):
         ''' Initialize package for parsing
 
         Parameters
         ----------
         package_name : string
             Name of the top-level package.  *package_name* must be the
             name of an importable package
         rst_extension : string, optional
             Extension for reST files, default '.rst'
         package_skip_patterns : None or sequence of {strings, regexps}
             Sequence of strings giving URIs of packages to be excluded
             Operates on the package path, starting at (including) the
             first dot in the package path, after *package_name* - so,
             if *package_name* is ``sphinx``, then ``sphinx.util`` will
             result in ``.util`` being passed for earching by these
             regexps.  If is None, gives default. Default is:
             ['\.tests$']
         module_skip_patterns : None or sequence
             Sequence of strings giving URIs of modules to be excluded
             Operates on the module name including preceding URI path,
             back to the first dot after *package_name*.  For example
             ``sphinx.util.console`` results in the string to search of
             ``.util.console``
             If is None, gives default. Default is:
             ['\.setup$', '\._']
         '''
         if package_skip_patterns is None:
             package_skip_patterns = ['\\.tests$']
         if module_skip_patterns is None:
             module_skip_patterns = ['\\.setup$', '\\._']
         self.package_name = package_name
         self.rst_extension = rst_extension
         self.package_skip_patterns = package_skip_patterns
         self.module_skip_patterns = module_skip_patterns
 
     def get_package_name(self):
         return self._package_name
 
     def set_package_name(self, package_name):
         ''' Set package_name
 
         >>> docwriter = ApiDocWriter('sphinx')
         >>> import sphinx
         >>> docwriter.root_path == sphinx.__path__[0]
         True
         >>> docwriter.package_name = 'docutils'
         >>> import docutils
         >>> docwriter.root_path == docutils.__path__[0]
         True
         '''
         # It's also possible to imagine caching the module parsing here
         self._package_name = package_name
         self.root_module = __import__(package_name)
         self.root_path = self.root_module.__path__[0]
         self.written_modules = None
 
     package_name = property(get_package_name, set_package_name, None,
                             'get/set package_name')
 
     def _get_object_name(self, line):
         ''' Get second token in line
         >>> docwriter = ApiDocWriter('sphinx')
         >>> docwriter._get_object_name("  def func():  ")
         'func'
         >>> docwriter._get_object_name("  class Klass(object):  ")
         'Klass'
         >>> docwriter._get_object_name("  class Klass:  ")
         'Klass'
         '''
         name = line.split()[1].split('(')[0].strip()
         # in case we have classes which are not derived from object
         # ie. old style classes
         return name.rstrip(':')
 
     def _uri2path(self, uri):
         ''' Convert uri to absolute filepath
 
         Parameters
         ----------
         uri : string
             URI of python module to return path for
 
         Returns
         -------
         path : None or string
             Returns None if there is no valid path for this URI
             Otherwise returns absolute file system path for URI
 
         Examples
         --------
         >>> docwriter = ApiDocWriter('sphinx')
         >>> import sphinx
         >>> modpath = sphinx.__path__[0]
         >>> res = docwriter._uri2path('sphinx.builder')
         >>> res == os.path.join(modpath, 'builder.py')
         True
         >>> res = docwriter._uri2path('sphinx')
         >>> res == os.path.join(modpath, '__init__.py')
         True
         >>> docwriter._uri2path('sphinx.does_not_exist')
 
         '''
         if uri == self.package_name:
             return os.path.join(self.root_path, '__init__.py')
         path = uri.replace('.', os.path.sep)
         path = path.replace(self.package_name + os.path.sep, '')
         path = os.path.join(self.root_path, path)
         # XXX maybe check for extensions as well?
         if os.path.exists(path + '.py'): # file
             path += '.py'
         elif os.path.exists(os.path.join(path, '__init__.py')):
             path = os.path.join(path, '__init__.py')
         else:
             return None
         return path
 
     def _path2uri(self, dirpath):
         ''' Convert directory path to uri '''
         relpath = dirpath.replace(self.root_path, self.package_name)
         if relpath.startswith(os.path.sep):
             relpath = relpath[1:]
         return relpath.replace(os.path.sep, '.')
 
     def _parse_module(self, uri):
         ''' Parse module defined in *uri* '''
         filename = self._uri2path(uri)
         if filename is None:
             # nothing that we could handle here.
             return ([],[])
         f = open(filename, 'rt')
         functions, classes = self._parse_lines(f)
         f.close()
         return functions, classes
     
     def _parse_lines(self, linesource):
         ''' Parse lines of text for functions and classes '''
         functions = []
         classes = []
         for line in linesource:
             if line.startswith('def ') and line.count('('):
                 # exclude private stuff
                 name = self._get_object_name(line)
                 if not name.startswith('_'):
                     functions.append(name)
             elif line.startswith('class '):
                 # exclude private stuff
                 name = self._get_object_name(line)
                 if not name.startswith('_'):
                     classes.append(name)
             else:
                 pass
         functions.sort()
         classes.sort()
         return functions, classes
 
     def generate_api_doc(self, uri):
         '''Make autodoc documentation template string for a module
 
         Parameters
         ----------
         uri : string
             python location of module - e.g 'sphinx.builder'
 
         Returns
         -------
         S : string
             Contents of API doc
         '''
         # get the names of all classes and functions
         functions, classes = self._parse_module(uri)
         if not len(functions) and not len(classes):
             print 'WARNING: Empty -',uri  # dbg
             return ''
 
         # Make a shorter version of the uri that omits the package name for
         # titles 
         uri_short = re.sub(r'^%s\.' % self.package_name,'',uri)
         
         ad = '.. AUTO-GENERATED FILE -- DO NOT EDIT!\n\n'
 
         chap_title = uri_short
         ad += (chap_title+'\n'+ self.rst_section_levels[1] * len(chap_title)
                + '\n\n')
 
         # Set the chapter title to read 'module' for all modules except for the
         # main packages
         if '.' in uri:
             title = 'Module: :mod:`' + uri_short + '`'
         else:
             title = ':mod:`' + uri_short + '`'
         ad += title + '\n' + self.rst_section_levels[2] * len(title)
 
         if len(classes):
             ad += '\nInheritance diagram for ``%s``:\n\n' % uri
             ad += '.. inheritance-diagram:: %s \n' % uri
             ad += '   :parts: 3\n'
 
         ad += '\n.. automodule:: ' + uri + '\n'
         ad += '\n.. currentmodule:: ' + uri + '\n'
         multi_class = len(classes) > 1
         multi_fx = len(functions) > 1
         if multi_class:
             ad += '\n' + 'Classes' + '\n' + \
                   self.rst_section_levels[2] * 7 + '\n'
         elif len(classes) and multi_fx:
             ad += '\n' + 'Class' + '\n' + \
                   self.rst_section_levels[2] * 5 + '\n'
         for c in classes:
             ad += '\n:class:`' + c + '`\n' \
                   + self.rst_section_levels[multi_class + 2 ] * \
                   (len(c)+9) + '\n\n'
             ad += '\n.. autoclass:: ' + c + '\n'
             # must NOT exclude from index to keep cross-refs working
             ad += '  :members:\n' \
                   '  :undoc-members:\n' \
                   '  :show-inheritance:\n' \
+                  '  :inherited-members:\n' \
                   '\n' \
                   '  .. automethod:: __init__\n'
         if multi_fx:
             ad += '\n' + 'Functions' + '\n' + \
                   self.rst_section_levels[2] * 9 + '\n\n'
         elif len(functions) and multi_class:
             ad += '\n' + 'Function' + '\n' + \
                   self.rst_section_levels[2] * 8 + '\n\n'
         for f in functions:
             # must NOT exclude from index to keep cross-refs working
             ad += '\n.. autofunction:: ' + uri + '.' + f + '\n\n'
         return ad
 
     def _survives_exclude(self, matchstr, match_type):
         ''' Returns True if *matchstr* does not match patterns
 
         ``self.package_name`` removed from front of string if present
 
         Examples
         --------
         >>> dw = ApiDocWriter('sphinx')
         >>> dw._survives_exclude('sphinx.okpkg', 'package')
         True
         >>> dw.package_skip_patterns.append('^\\.badpkg$')
         >>> dw._survives_exclude('sphinx.badpkg', 'package')
         False
         >>> dw._survives_exclude('sphinx.badpkg', 'module')
         True
         >>> dw._survives_exclude('sphinx.badmod', 'module')
         True
         >>> dw.module_skip_patterns.append('^\\.badmod$')
         >>> dw._survives_exclude('sphinx.badmod', 'module')
         False
         '''
         if match_type == 'module':
             patterns = self.module_skip_patterns
         elif match_type == 'package':
             patterns = self.package_skip_patterns
         else:
             raise ValueError('Cannot interpret match type "%s"' 
                              % match_type)
         # Match to URI without package name
         L = len(self.package_name)
         if matchstr[:L] == self.package_name:
             matchstr = matchstr[L:]
         for pat in patterns:
             try:
                 pat.search
             except AttributeError:
                 pat = re.compile(pat)
             if pat.search(matchstr):
                 return False
         return True
 
     def discover_modules(self):
         ''' Return module sequence discovered from ``self.package_name`` 
 
 
         Parameters
         ----------
         None
 
         Returns
         -------
         mods : sequence
             Sequence of module names within ``self.package_name``
 
         Examples
         --------
         >>> dw = ApiDocWriter('sphinx')
         >>> mods = dw.discover_modules()
         >>> 'sphinx.util' in mods
         True
         >>> dw.package_skip_patterns.append('\.util$')
         >>> 'sphinx.util' in dw.discover_modules()
         False
         >>> 
         '''
         modules = [self.package_name]
         # raw directory parsing
         for dirpath, dirnames, filenames in os.walk(self.root_path):
             # Check directory names for packages
             root_uri = self._path2uri(os.path.join(self.root_path,
                                                    dirpath))
             for dirname in dirnames[:]: # copy list - we modify inplace
                 package_uri = '.'.join((root_uri, dirname))
                 if (self._uri2path(package_uri) and
                     self._survives_exclude(package_uri, 'package')):
                     modules.append(package_uri)
                 else:
                     dirnames.remove(dirname)
             # Check filenames for modules
             for filename in filenames:
                 module_name = filename[:-3]
                 module_uri = '.'.join((root_uri, module_name))
                 if (self._uri2path(module_uri) and
                     self._survives_exclude(module_uri, 'module')):
                     modules.append(module_uri)
         return sorted(modules)
     
     def write_modules_api(self, modules,outdir):
         # write the list
         written_modules = []
         for m in modules:
             api_str = self.generate_api_doc(m)
             if not api_str:
                 continue
             # write out to file
             outfile = os.path.join(outdir,
                                    m + self.rst_extension)
             fileobj = open(outfile, 'wt')
             fileobj.write(api_str)
             fileobj.close()
             written_modules.append(m)
         self.written_modules = written_modules
 
     def write_api_docs(self, outdir):
         """Generate API reST files.
 
         Parameters
         ----------
         outdir : string
             Directory name in which to store files
             We create automatic filenames for each module
             
         Returns
         -------
         None
 
         Notes
         -----
         Sets self.written_modules to list of written modules
         """
         if not os.path.exists(outdir):
             os.mkdir(outdir)
         # compose list of modules
         modules = self.discover_modules()
         self.write_modules_api(modules,outdir)
         
     def write_index(self, outdir, froot='gen', relative_to=None):
         """Make a reST API index file from written files
 
         Parameters
         ----------
         path : string
             Filename to write index to
         outdir : string
             Directory to which to write generated index file
         froot : string, optional
             root (filename without extension) of filename to write to
             Defaults to 'gen'.  We add ``self.rst_extension``.
         relative_to : string
             path to which written filenames are relative.  This
             component of the written file path will be removed from
             outdir, in the generated index.  Default is None, meaning,
             leave path as it is.
         """
         if self.written_modules is None:
             raise ValueError('No modules written')
         # Get full filename path
         path = os.path.join(outdir, froot+self.rst_extension)
         # Path written into index is relative to rootpath
         if relative_to is not None:
             relpath = outdir.replace(relative_to + os.path.sep, '')
         else:
             relpath = outdir
         idx = open(path,'wt')
         w = idx.write
         w('.. AUTO-GENERATED FILE -- DO NOT EDIT!\n\n')
         w('.. toctree::\n\n')
         for f in self.written_modules:
             w('   %s\n' % os.path.join(relpath,f))
         idx.close()

docs/sphinxext/ipython_console_highlighting.py

@@ -1,98 +1,114 b''
 """reST directive for syntax-highlighting ipython interactive sessions.
+
+XXX - See what improvements can be made based on the new (as of Sept 2009)
+'pycon' lexer for the python console.  At the very least it will give better
+highlighted tracebacks.
 """
 
 #-----------------------------------------------------------------------------
 # Needed modules
 
 # Standard library
 import re
 
 # Third party
 from pygments.lexer import Lexer, do_insertions
 from pygments.lexers.agile import (PythonConsoleLexer, PythonLexer, 
                                    PythonTracebackLexer)
 from pygments.token import Comment, Generic
 
 from sphinx import highlighting
 
-
 #-----------------------------------------------------------------------------
 # Global constants
 line_re = re.compile('.*?\n')
 
 #-----------------------------------------------------------------------------
 # Code begins - classes and functions
 
 class IPythonConsoleLexer(Lexer):
     """
     For IPython console output or doctests, such as:
 
     .. sourcecode:: ipython
 
       In [1]: a = 'foo'
 
       In [2]: a
       Out[2]: 'foo'
 
       In [3]: print a
       foo
 
       In [4]: 1 / 0
 
     Notes:
 
       - Tracebacks are not currently supported.
 
       - It assumes the default IPython prompts, not customized ones.
     """
     
     name = 'IPython console session'
     aliases = ['ipython']
     mimetypes = ['text/x-ipython-console']
     input_prompt = re.compile("(In \[[0-9]+\]: )|(   \.\.\.+:)")
     output_prompt = re.compile("(Out\[[0-9]+\]: )|(   \.\.\.+:)")
     continue_prompt = re.compile("   \.\.\.+:")
     tb_start = re.compile("\-+")
 
     def get_tokens_unprocessed(self, text):
         pylexer = PythonLexer(**self.options)
         tblexer = PythonTracebackLexer(**self.options)
 
         curcode = ''
         insertions = []
         for match in line_re.finditer(text):
             line = match.group()
             input_prompt = self.input_prompt.match(line)
             continue_prompt = self.continue_prompt.match(line.rstrip())
             output_prompt = self.output_prompt.match(line)
             if line.startswith("#"):
                 insertions.append((len(curcode),
                                    [(0, Comment, line)]))
             elif input_prompt is not None:
                 insertions.append((len(curcode),
                                    [(0, Generic.Prompt, input_prompt.group())]))
                 curcode += line[input_prompt.end():]
             elif continue_prompt is not None:
                 insertions.append((len(curcode),
                                    [(0, Generic.Prompt, continue_prompt.group())]))
                 curcode += line[continue_prompt.end():]
             elif output_prompt is not None:
+                # Use the 'error' token for output.  We should probably make
+                # our own token, but error is typicaly in a bright color like
+                # red, so it works fine for our output prompts.
                 insertions.append((len(curcode),
-                                   [(0, Generic.Output, output_prompt.group())]))
+                                   [(0, Generic.Error, output_prompt.group())]))
                 curcode += line[output_prompt.end():]
             else:
                 if curcode:
                     for item in do_insertions(insertions,
                                               pylexer.get_tokens_unprocessed(curcode)):
                         yield item
                         curcode = ''
                         insertions = []
                 yield match.start(), Generic.Output, line
         if curcode:
             for item in do_insertions(insertions,
                                       pylexer.get_tokens_unprocessed(curcode)):
                 yield item
 
+
+def setup(app):
+    """Setup as a sphinx extension."""
+
+    # This is only a lexer, so adding it below to pygments appears sufficient.
+    # But if somebody knows that the right API usage should be to do that via
+    # sphinx, by all means fix it here.  At least having this setup.py
+    # suppresses the sphinx warning we'd get without it.
+    pass
+
 #-----------------------------------------------------------------------------
 # Register the extension as a valid pygments lexer
 highlighting.lexers['ipython'] = IPythonConsoleLexer()